Google Git Google Git
Sign in
llvm/llvm-project/792bf8ef030d7c2aa12abbab7662ffbcbe2d8b21/./llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
blob: 8717e2c57d24528816eaca8e787b9d11d7faf3d3 [file]
//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
// stores that can be put together into vector-stores. Next, it attempts to
// construct vectorizable tree using the use-def chains. If a profitable tree
// was found, the SLP vectorizer performs vectorization on the tree.
//
// The pass is inspired by the work described in the paper:
// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
//
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
#include "SLPVectorizer/SLPCompatibilityAnalysis.h"
#include "SLPVectorizer/SLPCostAnalysis.h"
#include "SLPVectorizer/SLPMemoryUtils.h"
#include "SLPVectorizer/SLPReductionUtils.h"
#include "SLPVectorizer/SLPShuffleAnalysis.h"
#include "SLPVectorizer/SLPTypeUtils.h"
#include "SLPVectorizer/SLPUtils.h"
#include "VectorizeOptions.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/VectorTypeUtils.h"
#ifdef EXPENSIVE_CHECKS
#include "llvm/IR/Verifier.h"
#endif
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
using namespace llvm;
using namespace llvm::PatternMatch;
using namespace slpvectorizer;
using namespace std::placeholders;
#define SV_NAME "slp-vectorizer"
#define DEBUG_TYPE "SLP"
STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
STATISTIC(NumStridedStoreChains, "Number of vectorized stride stores");
STATISTIC(NumStoreChains, "Number of vector stores created");
STATISTIC(NumVectorizedStores, "Number of vectorized stores");
DEBUG_COUNTER(VectorizedGraphs, "slp-vectorized",
"Controls which SLP graphs should be vectorized.");
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
// Another limit for the alias checks: The maximum distance between load/store
// instructions where alias checks are done.
// This limit is useful for very large basic blocks.
static const unsigned MaxMemDepDistance = 160;
/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
/// regions to be handled.
static const int MinScheduleRegionSize = 16;
/// If the reduction budget is exhausted, we allow small reduction trees to be
/// handled.
static const unsigned MinReductionTreeSize = 2048;
/// Maximum allowed number of operands in the PHI nodes.
static const unsigned MaxPHINumOperands = 128;
namespace {
/// A vectorized part of a split reduction, combined into the final reduction
/// result by the horizontal reduction emitter.
struct ReductionVectorPart {
/// The vectorized value, tracked in case it is replaced while other parts
/// are vectorized.
WeakTrackingVH Vec;
/// The number of times each lane is repeated in the reduction (emitted as a
/// multiplication by the scale for add/fadd reductions).
unsigned Scale = 1;
/// Signedness of \p Vec for reductions, operating on truncated types.
bool IsSigned = false;
/// True if the value was already reduced in-tree.
bool ReducedInTree = false;
/// True if the part contribution is subtracted from (rather than added to)
/// the final reduction result. Used for reassociated fadd reductions,
/// flattened through fsub/fneg operations.
bool Negated = false;
};
} // namespace
namespace {
class HorizontalReduction;
} // namespace
/// Bottom Up SLP Vectorizer.
class slpvectorizer::BoUpSLP {
class TreeEntry;
class ScheduleEntity;
class ScheduleData;
class ScheduleCopyableData;
class ScheduleBundle;
class ShuffleCostEstimator;
class ShuffleInstructionBuilder;
public:
/// If we decide to generate strided load / store, this struct contains all
/// the necessary info. It's fields are calculated by analyzeRtStrideCandidate
/// and analyzeConstantStrideCandidate. Note that Stride can be given either
/// as a SCEV or as a Value if it already exists. To get the stride in bytes,
/// StrideVal (or value obtained from StrideSCEV) has to by multiplied by the
/// size of element of FixedVectorType.
struct StridedPtrInfo {
Value *StrideVal = nullptr;
const SCEV *StrideSCEV = nullptr;
FixedVectorType *Ty = nullptr;
};
/// Tracks the state we can represent the loads in the given sequence.
enum class LoadsState {
Gather,
Vectorize,
ScatterVectorize,
StridedVectorize,
CompressVectorize,
BlendedLoadVectorize
};
using ValueList = SmallVector<Value *, 8>;
using InstrList = SmallVector<Instruction *, 16>;
using ValueSet = SmallPtrSet<Value *, 16>;
using StoreList = SmallVector<StoreInst *, 8>;
using ExtraValueToDebugLocsMap = SmallDenseSet<Value *, 4>;
using OrdersType = SmallVector<unsigned, 4>;
const VectorizeOptions &Opts;
BoUpSLP(const VectorizeOptions &Opts, Function *Func, ScalarEvolution *Se,
TargetTransformInfo *Tti, TargetLibraryInfo *TLi, AAResults *Aa,
LoopInfo *Li, DominatorTree *Dt, AssumptionCache *AC,
DemandedBits *DB, const DataLayout *DL,
OptimizationRemarkEmitter *ORE)
: Opts(Opts), AA(Aa), BatchAA(std::in_place, *Aa), F(Func), SE(Se),
TTI(Tti), TLI(TLi), LI(Li), DT(Dt), AC(AC), DB(DB), DL(DL), ORE(ORE),
CostKind(getSLPCostKind(Func)),
Builder(Se->getModule(), TargetFolder(*DL)) {
CodeMetrics::collectEphemeralValues(F, AC, EphValues);
// Use the vector register size specified by the target unless overridden
// by a command-line option.
// TODO: It would be better to limit the vectorization factor based on
// data type rather than just register size. For example, x86 AVX has
// 256-bit registers, but it does not support integer operations
// at that width (that requires AVX2).
MaxVecRegSize = Opts.slp_max_reg_size.value_or(
TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
.getFixedValue());
MinVecRegSize =
Opts.slp_min_reg_size.value_or(TTI->getMinVectorRegisterBitWidth());
}
/// Vectorize the tree that starts with the elements in \p VL.
/// Returns the vectorized root.
Value *vectorizeTree();
/// Vectorize the tree but with the list of externally used values \p
/// ExternallyUsedValues. Values in this MapVector can be replaced but the
/// generated extractvalue instructions.
Value *
vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot = nullptr,
ArrayRef<ReductionVectorPart> VectorValuesAndScales = {});
/// \returns the cost incurred by unwanted spills and fills, caused by
/// holding live values over call sites.
InstructionCost getSpillCost();
TargetTransformInfo::TargetCostKind getCostKind() const { return CostKind; }
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
/// final cost.
InstructionCost
calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals = {},
Instruction *RdxRoot = nullptr);
/// Finds i1 and/or nodes whose operand nodes are booleanized wide leaves
/// (one-use truncs of wide values or zero-tests of values from [0, 1]) and
/// registers them in MinBWs to be emitted in the wide leaf type.
void detectBooleanizedNodes();
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
InstructionCost getTreeCost(InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals = {},
InstructionCost ReductionCost = TTI::TCC_Free,
Instruction *RdxRoot = nullptr);
/// Construct a vectorizable tree that starts at \p Roots, ignoring users for
/// the purpose of scheduling and extraction in the \p UserIgnoreLst.
void buildTree(ArrayRef<Value *> Roots,
const SmallDenseSet<Value *> &UserIgnoreLst);
/// Construct a vectorizable tree that starts at \p Roots.
void buildTree(ArrayRef<Value *> Roots);
/// Sets the narrowed reduction chain instructions, dropped together with
/// the reduction.
void setNarrowedChainInsts(ArrayRef<Instruction *> Insts) {
NarrowedChainInsts.insert(Insts.begin(), Insts.end());
}
/// Returns true if the last buildTree() observed a may-alias memory
/// dependency between two distinct, range-checkable base objects, i.e. a
/// dependency that could be turned into a runtime alias check.
bool hasRuntimeCheckableBlockers() const {
return HasRuntimeCheckableBlockers;
}
/// Records whether a may-alias dependency between distinct, range-checkable
/// base objects has been observed, so the caller can decide to retry with
/// runtime alias checks enabled.
void setHasRuntimeCheckableBlockers(bool V) {
HasRuntimeCheckableBlockers = V;
}
/// Returns true if the last buildTree() kept a may-alias memory dependency
/// that is not runtime-checkable (call or a non-simple mem access). Such a
/// dependency cannot be dropped, so a runtime-checks retry cannot unblock the
/// region and would be pure overhead.
bool hasNonCheckableMemBlocker() const { return HasNonCheckableMemBlocker; }
/// Records that a non-runtime-checkable may-alias dependency was kept.
void setHasNonCheckableMemBlocker(bool V) { HasNonCheckableMemBlocker = V; }
/// Returns true if the current vectorization attempt may drop
/// runtime-checkable may-alias dependencies and guard the region with
/// runtime alias checks.
bool isTryingRuntimeAliasChecks() const { return TryRuntimeAliasChecks; }
/// Enables or disables dropping runtime-checkable may-alias dependencies in
/// favor of runtime alias checks for the current vectorization attempt.
void setTryRuntimeAliasChecks(bool V) { TryRuntimeAliasChecks = V; }
/// Splits the stored values of the root stores into halves, vectorized as
/// separate subtrees, while the returned guard is alive.
[[nodiscard]] SaveAndRestore<bool> splitStoredValues() {
return SaveAndRestore<bool>(SplitStoredValues, true);
}
/// Checks if the graph is built for the \p Stores and their stored values,
/// looking through the vectorized cast node, are vectorized as the alternate
/// node with the different opcodes in the halves.
bool canSplitStoredValues(ArrayRef<Value *> Stores) const {
if (VectorizableTree.empty() || getRootNode().isGather() ||
getRootNodeScalars() != Stores)
return false;
const TreeEntry *E = getOperandEntry(&getRootNode(), 0);
bool IsCast = E->State == TreeEntry::Vectorize && !E->isAltShuffle() &&
E->getMainOp()->isCast();
if (IsCast)
E = getOperandEntry(E, 0);
if (E->State != TreeEntry::Vectorize || !E->isAltShuffle())
return false;
auto GetOpcode = [&](Value *V) -> unsigned {
V = cast<StoreInst>(V)->getValueOperand();
if (auto *CI = dyn_cast<CastInst>(V); CI && IsCast)
V = CI->getOperand(0);
auto *I = dyn_cast<Instruction>(V);
return I ? I->getOpcode() : 0;
};
unsigned Opcode0 = GetOpcode(Stores.front());
unsigned Opcode1 = GetOpcode(Stores.back());
ArrayRef<Value *> Half = Stores.take_front(Stores.size() / 2);
return Opcode0 != Opcode1 &&
all_of(Half, [&](Value *V) { return GetOpcode(V) == Opcode0; }) &&
all_of(Stores.drop_front(Half.size()),
[&](Value *V) { return GetOpcode(V) == Opcode1; });
}
/// Resets the runtime alias check data.
void resetRuntimeAliasCheckState() {
HasRuntimeCheckableBlockers = false;
HasNonCheckableMemBlocker = false;
RTChecksFinalized = false;
RTChecks.clear();
RTOrigBodyOrder.clear();
}
/// Snapshots RTChecks.BB's body (non-PHI, non-alloca, non-terminator) into
/// RTOrigBodyOrder in program order, for the scalar fallback.
void captureRuntimeCheckBodySnapshot();
/// Returns true if \p BB satisfies the block-level preconditions for runtime
/// alias check versioning (straight-line, duplicable, not a scalar fallback,
/// function not optimized for size). These checks do not depend on the
/// collected checks, so they can gate the (expensive) optimistic retry before
/// any tree is rebuilt.
bool canVersionBlockForRuntimeChecks(BasicBlock *BB) const;
/// Returns true if the runtime alias checks can be safely emitted to guard
/// the vectorized region.
bool canVersionForRuntimeChecks();
/// Returns true if \p BB is a scalar fallback block created by runtime alias
/// check versioning.
bool isScalarFallbackBlock(BasicBlock *BB) const {
return ScalarFallbackBlocks.contains(BB);
}
/// Returns true if an optimistic runtime-checks versioning attempt already
/// failed for \p BB, so further retries in the same block can be skipped.
bool runtimeChecksFailedForBlock(BasicBlock *BB) const {
return FailedRuntimeChecksBlocks.contains(BB);
}
/// Records that an optimistic runtime-checks versioning attempt failed for
/// \p BB.
void markRuntimeChecksFailedForBlock(BasicBlock *BB) {
FailedRuntimeChecksBlocks.insert(BB);
}
/// Returns the modeled cost of the runtime alias checks collected during the
/// last (optimistic) buildTree().
InstructionCost getRuntimeChecksCost() const;
/// Returns true if the last (optimistic) buildTree() collected any runtime
/// alias checks that must guard the vectorized region.
bool hasRuntimeAliasChecks() const { return !RTChecks.BasePairs.empty(); }
/// Returns true if vectorization changed the CFG (i.e. a block was versioned
/// with runtime alias checks). When true, CFG analyses must not be preserved.
bool isCFGChanged() const { return CFGChanged; }
TreeEntry &getRootNode() {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return *VectorizableTree.front();
}
const TreeEntry &getRootNode() const {
assert(!VectorizableTree.empty() && "No graph to get the first node from");
return *VectorizableTree.front();
}
/// Returns the scalars of the root node.
ArrayRef<Value *> getRootNodeScalars() const { return getRootNode().Scalars; }
/// Returns the lane the given value is vectorized to in the root node.
unsigned findRootLaneForValue(Value *V) const {
return getRootNode().findLaneForValue(V);
}
/// Returns the type/is-signed info for the root node in the graph without
/// casting.
std::optional<std::pair<Type *, bool>> getRootNodeTypeWithNoCast() const {
const TreeEntry &Root = getRootNode();
if (Root.State != TreeEntry::Vectorize || Root.isAltShuffle() ||
!Root.Scalars.front()->getType()->isIntegerTy())
return std::nullopt;
auto It = MinBWs.find(&Root);
if (It != MinBWs.end())
return std::make_pair(IntegerType::get(Root.Scalars.front()->getContext(),
It->second.first),
It->second.second);
if (Root.getOpcode() == Instruction::ZExt ||
Root.getOpcode() == Instruction::SExt)
return std::make_pair(cast<CastInst>(Root.getMainOp())->getSrcTy(),
Root.getOpcode() == Instruction::SExt);
return std::nullopt;
}
/// Checks if the root graph node can be emitted with narrower bitwidth at
/// codegen and returns it signedness, if so.
bool isSignedMinBitwidthRootNode() const {
return MinBWs.at(&getRootNode()).second;
}
/// Returns reduction type after minbitdth analysis.
FixedVectorType *getReductionType() const {
if (ReductionBitWidth == 0 ||
!getRootNodeScalars().front()->getType()->isIntegerTy() ||
ReductionBitWidth >=
DL->getTypeSizeInBits(getRootNodeScalars().front()->getType()))
return cast<FixedVectorType>(
getWidenedType(getRootNodeScalars().front()->getType(),
getRootNode().getVectorFactor()));
return cast<FixedVectorType>(getWidenedType(
IntegerType::get(getRootNodeScalars().front()->getContext(),
ReductionBitWidth),
getRootNode().getVectorFactor()));
}
/// Returns true if the tree results in one of the reduced bitcasts variants.
bool isReducedBitcastRoot() const {
return getRootNode().hasState() &&
(getRootNode().CombinedOp == TreeEntry::ReducedBitcast ||
getRootNode().CombinedOp == TreeEntry::ReducedBitcastBSwap ||
getRootNode().CombinedOp == TreeEntry::ReducedBitcastLoads ||
getRootNode().CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
getRootNode().CombinedOp == TreeEntry::ReducedBitPack) &&
getRootNode().State == TreeEntry::Vectorize;
}
/// Returns true if the tree results in the reduced cmp bitcast root.
bool isReducedCmpBitcastRoot() const {
return getRootNode().hasState() &&
getRootNode().CombinedOp == TreeEntry::ReducedCmpBitcast &&
getRootNode().State == TreeEntry::Vectorize;
}
/// Returns true if the tree is a reduction tree.
bool isReductionTree() const { return UserIgnoreList != nullptr; }
/// Builds external uses of the vectorized scalars, i.e. the list of
/// vectorized scalars to be extracted, their lanes and their scalar users. \p
/// ExternallyUsedValues contains additional list of external uses to handle
/// vectorization of reductions.
void
buildExternalUses(const ExtraValueToDebugLocsMap &ExternallyUsedValues = {});
/// Transforms graph nodes to target specific representations, if profitable.
void transformNodes();
/// Clear the internal data structures that are created by 'buildTree'.
void deleteTree() {
VectorizableTree.clear();
ScalarToTreeEntries.clear();
DeletedNodes.clear();
TransformedToGatherNodes.clear();
OperandsToTreeEntry.clear();
ScalarsInSplitNodes.clear();
MustGather.clear();
ReassocScalarToTreeEntries.clear();
KeptReassocScalars.clear();
NonScheduledFirst.clear();
EntryToLastInstruction.clear();
LastInstructionToPos.clear();
LoadEntriesToVectorize.clear();
IsGraphTransformMode = false;
GatheredLoadsEntriesFirst.reset();
SplatGatheredScalarsRoots.clear();
NumCanonicalSplatSubtreeEntries = 0;
CompressEntryToData.clear();
ExternalUses.clear();
ExternalUsesAsOriginalScalar.clear();
ExternalUsesWithNonUsers.clear();
ExternalUseReplacements.clear();
RTChecks.clear();
HasRuntimeCheckableBlockers = false;
HasNonCheckableMemBlocker = false;
RTChecksFinalized = false;
for (auto &Iter : BlocksSchedules) {
BlockScheduling *BS = Iter.second.get();
BS->clear();
}
MinBWs.clear();
ReductionBitWidth = 0;
BaseGraphSize = 1;
CastMaxMinBWSizes.reset();
ExtraBitWidthNodes.clear();
InstrElementSize.clear();
UserIgnoreList = nullptr;
NarrowedChainInsts.clear();
PostponedGathers.clear();
ValueToGatherNodes.clear();
TreeEntryToStridedPtrInfoMap.clear();
CurrentLoopNest.clear();
MergedLoopBTCs.clear();
}
unsigned getTreeSize() const { return VectorizableTree.size(); }
/// Returns the size of the current tree, stopping at gathers
/// For instance, even though we continue adding tree entries after
/// a split node, this function stops counting at the split node
/// since it is a gather. This matches the behavior of MinBWs
/// computation.
unsigned getTreeSizeExcludingGathers() const {
unsigned Cnt = 0;
SmallBitVector GatherTreeNodes(VectorizableTree.size(), false);
for (unsigned NodeIdx : seq<unsigned>(VectorizableTree.size())) {
auto &TE = VectorizableTree[NodeIdx];
if (DeletedNodes.contains(TE.get()))
continue;
auto IsGather = [&](TreeEntry *TE) {
return TE->isGather() || TransformedToGatherNodes.contains(TE) ||
GatherTreeNodes.test(TE->Idx);
};
if (IsGather(TE.get()) || (!TE->UserTreeIndex.UserTE && NodeIdx != 0) ||
(TE->UserTreeIndex.UserTE && (IsGather(TE->UserTreeIndex.UserTE)))) {
GatherTreeNodes.set(NodeIdx);
continue;
}
++Cnt;
}
return Cnt;
}
/// Returns true if the tree gathers any scalars other than constants.
bool hasNonConstantGathers() const {
return any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return !DeletedNodes.contains(TE.get()) &&
(TE->isGather() || TransformedToGatherNodes.contains(TE.get())) &&
!allConstant(TE->Scalars);
});
}
/// Returns the base graph size, before any transformations.
unsigned getCanonicalGraphSize() const { return BaseGraphSize; }
/// Number of tree entries that form the splat gather subtrees.
unsigned getNumSplatSubtreeEntries() const {
return NumCanonicalSplatSubtreeEntries;
}
/// Perform LICM and CSE on the newly generated gather sequences.
void optimizeGatherSequence();
/// Does this non-empty order represent an identity order? Identity
/// should be represented as an empty order, so this is used to
/// decide if we can canonicalize a computed order. Undef elements
/// (represented as size) are ignored.
static bool isIdentityOrder(ArrayRef<unsigned> Order) {
assert(!Order.empty() && "expected non-empty order");
const unsigned Sz = Order.size();
return all_of(enumerate(Order), [&](const auto &P) {
return P.value() == P.index() || P.value() == Sz;
});
}
/// Checks if the specified gather tree entry \p TE can be represented as a
/// shuffled vector entry + (possibly) permutation with other gathers. It
/// implements the checks only for possibly ordered scalars (Loads,
/// ExtractElement, ExtractValue), which can be part of the graph.
/// \param TopToBottom If true, used for the whole tree rotation, false - for
/// sub-tree rotations. \param IgnoreReorder true, if the order of the root
/// node might be ignored.
std::optional<OrdersType> findReusedOrderedScalars(const TreeEntry &TE,
bool TopToBottom,
bool IgnoreReorder);
/// Sort loads into increasing pointers offsets to allow greater clustering.
std::optional<OrdersType> findPartiallyOrderedLoads(const TreeEntry &TE);
/// Gets reordering data for the given tree entry. If the entry is vectorized
/// - just return ReorderIndices, otherwise check if the scalars can be
/// reordered and return the most optimal order.
/// \return std::nullopt if ordering is not important, empty order, if
/// identity order is important, or the actual order.
/// \param TopToBottom If true, include the order of vectorized stores and
/// insertelement nodes, otherwise skip them.
/// \param IgnoreReorder true, if the root node order can be ignored.
std::optional<OrdersType>
getReorderingData(const TreeEntry &TE, bool TopToBottom, bool IgnoreReorder);
/// Checks if it is profitable to reorder the current tree.
/// If the tree does not contain many profitable reordable nodes, better to
/// skip it to save compile time.
bool isProfitableToReorder() const;
/// Reorders the current graph to the most profitable order starting from the
/// root node to the leaf nodes. The best order is chosen only from the nodes
/// of the same size (vectorization factor). Smaller nodes are considered
/// parts of subgraph with smaller VF and they are reordered independently. We
/// can make it because we still need to extend smaller nodes to the wider VF
/// and we can merge reordering shuffles with the widening shuffles.
void reorderTopToBottom();
/// Reorders the current graph to the most profitable order starting from
/// leaves to the root. It allows to rotate small subgraphs and reduce the
/// number of reshuffles if the leaf nodes use the same order. In this case we
/// can merge the orders and just shuffle user node instead of shuffling its
/// operands. Plus, even the leaf nodes have different orders, it allows to
/// sink reordering in the graph closer to the root node and merge it later
/// during analysis.
void reorderBottomToTop(bool IgnoreReorder = false);
/// Marks the schedule data of the copyable-modeled operands of \p TE for
/// dependency recalculation at the next bundle scheduling.
void markCopyableDepsForRecalc(TreeEntry &TE);
/// \return The vector element size in bits to use when vectorizing the
/// expression tree ending at \p V. If V is a store, the size is the width of
/// the stored value. Otherwise, the size is the width of the largest loaded
/// value reaching V. This method is used by the vectorizer to calculate
/// vectorization factors.
unsigned getVectorElementSize(Value *V);
/// Compute the minimum type sizes required to represent the entries in a
/// vectorizable tree.
void computeMinimumValueSizes();
// \returns maximum vector register size as set by TTI or overridden by cl::opt.
unsigned getMaxVecRegSize() const {
return MaxVecRegSize;
}
/// \returns the -slp-threshold value, 0 if unset.
int getCostThreshold() const { return Opts.slp_threshold.value_or(0); }
// \returns minimum vector register size as set by cl::opt.
unsigned getMinVecRegSize() const {
return MinVecRegSize;
}
/// \returns the number of parts, the type \p VecTy is split at the codegen
/// phase. The type legalization queries are repeated for the very same types
/// during the analysis, so the results are cached for the function.
unsigned getNumberOfParts(
Type *VecTy, Type *ScalarTy,
unsigned Limit = std::numeric_limits<unsigned>::max()) const {
auto [It, Inserted] =
NumberOfPartsCache.try_emplace(std::make_tuple(VecTy, ScalarTy, Limit));
if (Inserted)
It->second = slpvectorizer::getNumberOfPartsOrRegs(
/*QueryNumParts=*/true, *TTI, VecTy, ScalarTy, Opts.slp_revec, Limit);
return It->second;
}
/// \returns the number of parts, the type \p VecTy is split at the codegen
/// phase. The type legalization queries are repeated for the very same types
/// during the analysis, so the results are cached for the function.
unsigned getRegUsageForType(
Type *VecTy, Type *ScalarTy,
unsigned Limit = std::numeric_limits<unsigned>::max()) const {
auto [It, Inserted] =
NumberOfRegsCache.try_emplace(std::make_tuple(VecTy, ScalarTy, Limit));
if (Inserted)
It->second = slpvectorizer::getNumberOfPartsOrRegs(
/*QueryNumParts=*/false, *TTI, VecTy, ScalarTy, Opts.slp_revec,
Limit);
return It->second;
}
unsigned getMinVF(unsigned Sz) const {
return std::max(2U, getMinVecRegSize() / Sz);
}
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
unsigned MaxVF =
Opts.slp_max_vf.value_or(TTI->getMaximumVF(ElemWidth, Opcode));
return MaxVF ? MaxVF : UINT_MAX;
}
/// Check if homogeneous aggregate is isomorphic to some VectorType.
/// Accepts homogeneous multidimensional aggregate of scalars/vectors like
/// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> },
/// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on.
///
/// \returns number of elements in vector if isomorphism exists, 0 otherwise.
unsigned canMapToVector(Type *T) const;
/// \returns true if the vectorized insertvalue result can be stored directly
/// as a vector, i.e. every insertvalue with an external user is consumed by a
/// single store only.
bool canVectorStoreInsertValue(const TreeEntry *E) const;
/// \returns the source vector type for an InsertElement/InsertValue
/// buildvector node \p E: the inserted vector type for insertelement, or a
/// vector of the inserted scalar type wide enough to cover the highest
/// inserted index for insertvalue.
FixedVectorType *getInsertBuildVectorSrcTy(const TreeEntry *E) const;
/// \returns True if the VectorizableTree is both tiny and not fully
/// vectorizable. We do not vectorize such trees.
bool isTreeTinyAndNotFullyVectorizable(bool ForReduction = false) const;
/// Checks if the graph and all its subgraphs cannot be better vectorized.
/// It may happen, if all gather nodes are loads and they cannot be
/// "clusterized". In this case even subgraphs cannot be vectorized more
/// effectively than the base graph.
bool isTreeNotExtendable() const;
bool isStridedLoad(ArrayRef<Value *> PointerOps, Type *ScalarTy,
Align Alignment, const int64_t Diff,
const size_t Sz) const;
/// Returns true for widened strided loads, where each lane of the strided
/// load covers a group of consecutive scalars. Reversed widened strided
/// loads must be emitted with a positive stride and a reorder shuffle.
bool isWidenedStridedLoad(const TreeEntry *E) const;
/// Return true if an array of scalar loads can be replaced with a strided
/// load (with constant stride).
///
/// It is possible that the load gets "widened". Suppose that originally each
/// load loads `k` bytes and `PointerOps` can be arranged as follows (`%s` is
/// constant): %b + 0 * %s + 0 %b + 0 * %s + 1 %b + 0 * %s + 2
/// ...
/// %b + 0 * %s + (w - 1)
///
/// %b + 1 * %s + 0
/// %b + 1 * %s + 1
/// %b + 1 * %s + 2
/// ...
/// %b + 1 * %s + (w - 1)
/// ...
///
/// %b + (n - 1) * %s + 0
/// %b + (n - 1) * %s + 1
/// %b + (n - 1) * %s + 2
/// ...
/// %b + (n - 1) * %s + (w - 1)
///
/// In this case we will generate a strided load of type `<n x (k * w)>`.
///
/// \param PointerOps list of pointer arguments of loads.
/// \param ElemTy original scalar type of loads.
/// \param Alignment alignment of the first load.
/// \param SortedIndices is the order of PointerOps as returned by
/// `sortPtrAccesses`
/// \param Diff Pointer difference between the lowest and the highes pointer
/// in `PointerOps` as returned by `getPointersDiff`.
/// \param Ptr0 first pointer in `PointersOps`.
/// \param PtrN last pointer in `PointersOps`.
/// \param SPtrInfo If the function return `true`, it also sets all the fields
/// of `SPtrInfo` necessary to generate the strided load later.
bool analyzeConstantStrideCandidate(
const ArrayRef<Value *> PointerOps, Type *ElemTy, Align Alignment,
const SmallVectorImpl<unsigned> &SortedIndices, const int64_t Diff,
Value *Ptr0, StridedPtrInfo &SPtrInfo) const;
/// Return true if an array of scalar loads can be replaced with a strided
/// load (with run-time stride).
/// \param PointerOps list of pointer arguments of loads.
/// \param ScalarTy type of loads.
/// \param CommonAlignment common alignement of loads as computed by
/// `computeCommonAlignment<LoadInst>`.
/// \param SortedIndicies is a list of indicies computed by this function such
/// that the sequence `PointerOps[SortedIndices[0]],
/// PointerOps[SortedIndicies[1]], ..., PointerOps[SortedIndices[n]]` is
/// ordered by the coefficient of the stride. For example, if PointerOps is
/// `%base + %stride, %base, %base + 2 * stride` the `SortedIndices` will be
/// `[1, 0, 2]`. We follow the convention that if `SortedIndices` has to be
/// `0, 1, 2, 3, ...` we return empty vector for `SortedIndicies`.
/// \param SPtrInfo If the function return `true`, it also sets all the fields
/// of `SPtrInfo` necessary to generate the strided load later.
/// \param IsLoad Is this a strided load (true) or strided store (false)
bool analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps, Type *ScalarTy,
Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo &SPtrInfo, bool IsLoad) const;
/// Checks if the given array of loads can be represented as a vectorized,
/// scatter or just simple gather.
/// \param VL list of loads.
/// \param VL0 main load value.
/// \param Order returned order of load instructions.
/// \param PointerOps returned list of pointer operands.
/// \param BestVF return best vector factor, if recursive check found better
/// vectorization sequences rather than masked gather.
/// \param TryRecursiveCheck used to check if long masked gather can be
/// represented as a serie of loads/insert subvector, if profitable.
LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
SmallVectorImpl<unsigned> &Order,
SmallVectorImpl<Value *> &PointerOps,
StridedPtrInfo &SPtrInfo,
unsigned *BestVF = nullptr,
bool TryRecursiveCheck = true) const;
/// Checks whether some existing tree entry has scalars equal to \p VL.
/// \p S is the common opcode of \p VL when one exists; an empty \p S means
/// the values have no common opcode (mixed buildvector/gather candidates).
bool hasSameNode(const InstructionsState &S, ArrayRef<Value *> VL) const {
auto IsSame = [&](const TreeEntry *TE) { return TE->isSame(VL); };
if (S) {
// Any vectorized or gather entry equal to VL must contain S.getMainOp()
// (the representative instruction, which is also the recorded scalar
// for copyable-elements bundles), so probing the MainOp-indexed maps
// is sufficient and avoids scanning the whole tree.
return any_of(getTreeEntries(S.getMainOp()), IsSame) ||
any_of(ValueToGatherNodes.lookup(S.getMainOp()), IsSame);
}
// No common opcode: only gather entries can match. Each non-constant
// value in VL has to be in the gather entry's scalar list and is
// therefore present in ValueToGatherNodes. Probe by VL members instead
// of scanning the whole tree (O(tree) -> O(|VL|)).
SmallPtrSet<const TreeEntry *, 4> Visited;
for (Value *V : VL) {
// Constants/poisons are not tracked in ValueToGatherNodes.
if (isConstant(V))
continue;
for (const TreeEntry *TE : ValueToGatherNodes.lookup(V)) {
if (!Visited.insert(TE).second)
continue;
if (IsSame(TE))
return true;
}
}
return false;
}
/// Registers non-vectorizable sequence of loads
template <typename T> void registerNonVectorizableLoads(ArrayRef<T *> VL) {
ListOfKnonwnNonVectorizableLoads.insert(hash_value(VL));
}
/// Checks if the given loads sequence is known as not vectorizable
template <typename T>
bool areKnownNonVectorizableLoads(ArrayRef<T *> VL) const {
return ListOfKnonwnNonVectorizableLoads.contains(hash_value(VL));
}
OptimizationRemarkEmitter *getORE() { return ORE; }
/// This structure holds any data we need about the edges being traversed
/// during buildTreeRec(). We keep track of:
/// (i) the user TreeEntry index, and
/// (ii) the index of the edge.
struct EdgeInfo {
EdgeInfo() = default;
EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx)
: UserTE(UserTE), EdgeIdx(EdgeIdx) {}
/// The user TreeEntry.
TreeEntry *UserTE = nullptr;
/// The operand index of the use.
unsigned EdgeIdx = UINT_MAX;
#ifndef NDEBUG
friend inline raw_ostream &operator<<(raw_ostream &OS,
const BoUpSLP::EdgeInfo &EI) {
EI.dump(OS);
return OS;
}
/// Debug print.
void dump(raw_ostream &OS) const {
OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null")
<< " EdgeIdx:" << EdgeIdx << "}";
}
LLVM_DUMP_METHOD void dump() const { dump(dbgs()); }
#endif
bool operator == (const EdgeInfo &Other) const {
return UserTE == Other.UserTE && EdgeIdx == Other.EdgeIdx;
}
operator bool() const { return UserTE != nullptr; }
};
friend struct DenseMapInfo<EdgeInfo>;
/// A helper class used for scoring candidates for two consecutive lanes.
class LookAheadHeuristics {
const TargetLibraryInfo &TLI;
const DataLayout &DL;
ScalarEvolution &SE;
const BoUpSLP &R;
int NumLanes; // Total number of lanes (aka vectorization factor).
int MaxLevel; // The maximum recursion depth for accumulating score.
public:
LookAheadHeuristics(const TargetLibraryInfo &TLI, const DataLayout &DL,
ScalarEvolution &SE, const BoUpSLP &R, int NumLanes,
int MaxLevel)
: TLI(TLI), DL(DL), SE(SE), R(R), NumLanes(NumLanes),
MaxLevel(MaxLevel) {}
// The hard-coded scores listed here are not very important, though it shall
// be higher for better matches to improve the resulting cost. When
// computing the scores of matching one sub-tree with another, we are
// basically counting the number of values that are matching. So even if all
// scores are set to 1, we would still get a decent matching result.
// However, sometimes we have to break ties. For example we may have to
// choose between matching loads vs matching opcodes. This is what these
// scores are helping us with: they provide the order of preference. Also,
// this is important if the scalar is externally used or used in another
// tree entry node in the different lane.
/// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
static constexpr int ScoreConsecutiveLoads = 40;
/// The same load multiple times. This should have a better score than
/// `ScoreSplat` because it in x86 for a 2-lane vector we can represent it
/// with `movddup (%reg), xmm0` which has a throughput of 0.5 versus 0.5 for
/// a vector load and 1.0 for a broadcast.
static constexpr int ScoreSplatLoads = 30;
/// Loads from reversed memory addresses, e.g. load(A[i+1]), load(A[i]).
static constexpr int ScoreReversedLoads = 30;
/// A load candidate for masked gather.
static constexpr int ScoreMaskedGatherCandidate = 10;
/// ExtractElementInst from same vector and consecutive indexes.
static constexpr int ScoreConsecutiveExtracts = 40;
/// ExtractElementInst from same vector and reversed indices.
static constexpr int ScoreReversedExtracts = 30;
/// Constants.
static constexpr int ScoreConstants = 15;
/// Same constants.
static constexpr int ScoreSameConstants = 17;
/// Instructions with the same opcode.
static constexpr int ScoreSameOpcode = 20;
/// Instructions with alt opcodes (e.g, add + sub).
static constexpr int ScoreAltOpcodes = 10;
/// Identical instructions (a.k.a. splat or broadcast).
static constexpr int ScoreSplat = 10;
/// Matching with an undef is preferable to failing.
static constexpr int ScoreUndef = 10;
/// Score for failing to find a decent match.
static constexpr int ScoreFail = 0;
/// Score if all users are vectorized.
static constexpr int ScoreAllUserVectorized = 10;
/// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
/// \p U1 and \p U2 are the users of \p V1 and \p V2.
/// Also, checks if \p V1 and \p V2 are compatible with instructions in \p
/// MainAltOps.
int getShallowScore(Value *V1, Value *V2, Instruction *U1, Instruction *U2,
ArrayRef<Value *> MainAltOps) const {
if (!isValidElementType(V1->getType(), R.Opts.slp_revec) ||
!isValidElementType(V2->getType(), R.Opts.slp_revec))
return LookAheadHeuristics::ScoreFail;
if (V1 == V2) {
if (isa<LoadInst>(V1)) {
// Retruns true if the users of V1 and V2 won't need to be extracted.
auto AllUsersAreInternal = [U1, U2, this](Value *V1, Value *V2) {
// Bail out if we have too many uses to save compilation time.
if (V1->hasNUsesOrMore(UsesLimit) || V2->hasNUsesOrMore(UsesLimit))
return false;
auto AllUsersVectorized = [U1, U2, this](Value *V) {
return llvm::all_of(V->users(), [U1, U2, this](Value *U) {
return U == U1 || U == U2 || R.isVectorized(U);
});
};
return AllUsersVectorized(V1) && AllUsersVectorized(V2);
};
// A broadcast of a load can be cheaper on some targets.
if (R.TTI->isLegalBroadcastLoad(V1->getType(),
ElementCount::getFixed(NumLanes)) &&
((int)V1->getNumUses() == NumLanes ||
AllUsersAreInternal(V1, V2)))
return LookAheadHeuristics::ScoreSplatLoads;
}
if (isa<UndefValue>(V1))
return LookAheadHeuristics::ScoreUndef;
if (isConstant(V1))
return LookAheadHeuristics::ScoreSameConstants;
return LookAheadHeuristics::ScoreSplat;
}
auto CheckSameEntryOrFail = [&]() {
if (ArrayRef<TreeEntry *> TEs1 = R.getTreeEntries(V1); !TEs1.empty()) {
SmallPtrSet<TreeEntry *, 4> Set(llvm::from_range, TEs1);
if (ArrayRef<TreeEntry *> TEs2 = R.getTreeEntries(V2);
!TEs2.empty() &&
any_of(TEs2, [&](TreeEntry *E) { return Set.contains(E); }))
return LookAheadHeuristics::ScoreSplatLoads;
}
return LookAheadHeuristics::ScoreFail;
};
auto *LI1 = dyn_cast<LoadInst>(V1);
auto *LI2 = dyn_cast<LoadInst>(V2);
if (LI1 && LI2) {
if (LI1->getParent() != LI2->getParent() || !LI1->isSimple() ||
!LI2->isSimple())
return CheckSameEntryOrFail();
std::optional<int64_t> Dist = R.getPointersDiff(
LI1->getType(), LI1->getPointerOperand(), LI2->getType(),
LI2->getPointerOperand(), DL, SE, /*StrictCheck=*/true);
if (!Dist || *Dist == 0) {
if (getUnderlyingObject(LI1->getPointerOperand()) ==
getUnderlyingObject(LI2->getPointerOperand()) &&
R.TTI->isLegalMaskedGather(
getWidenedType(LI1->getType(), NumLanes), LI1->getAlign()))
return LookAheadHeuristics::ScoreMaskedGatherCandidate;
return CheckSameEntryOrFail();
}
// The distance is too large - still may be profitable to use masked
// loads/gathers.
if (std::abs(*Dist) > NumLanes / 2)
return LookAheadHeuristics::ScoreMaskedGatherCandidate;
// This still will detect consecutive loads, but we might have "holes"
// in some cases. It is ok for non-power-2 vectorization and may produce
// better results. It should not affect current vectorization.
return (*Dist > 0) ? LookAheadHeuristics::ScoreConsecutiveLoads
: LookAheadHeuristics::ScoreReversedLoads;
}
auto *C1 = dyn_cast<Constant>(V1);
auto *C2 = dyn_cast<Constant>(V2);
if (C1 && C2)
return LookAheadHeuristics::ScoreConstants;
// Consider constants and buildvector compatible.
if ((C1 && isa<InsertElementInst>(V2)) ||
(C2 && isa<InsertElementInst>(V1)))
return LookAheadHeuristics::ScoreSameOpcode;
// Extracts from consecutive indexes of the same vector better score as
// the extracts could be optimized away.
Value *EV1;
ConstantInt *Ex1Idx;
if (match(V1, m_ExtractElt(m_Value(EV1), m_ConstantInt(Ex1Idx)))) {
// Undefs are always profitable for extractelements.
// Compiler can easily combine poison and extractelement <non-poison> or
// undef and extractelement <poison>. But combining undef +
// extractelement <non-poison-but-may-produce-poison> requires some
// extra operations.
if (isa<UndefValue>(V2))
return (isa<PoisonValue>(V2) || isUndefVector(EV1).all())
? LookAheadHeuristics::ScoreConsecutiveExtracts
: LookAheadHeuristics::ScoreSameOpcode;
Value *EV2 = nullptr;
ConstantInt *Ex2Idx = nullptr;
if (match(V2,
m_ExtractElt(m_Value(EV2), m_CombineOr(m_ConstantInt(Ex2Idx),
m_Undef())))) {
// Undefs are always profitable for extractelements.
if (!Ex2Idx)
return LookAheadHeuristics::ScoreConsecutiveExtracts;
if (isUndefVector(EV2).all() && EV2->getType() == EV1->getType())
return LookAheadHeuristics::ScoreConsecutiveExtracts;
if (EV2 == EV1) {
int Idx1 = Ex1Idx->getZExtValue();
int Idx2 = Ex2Idx->getZExtValue();
int Dist = Idx2 - Idx1;
// The distance is too large - still may be profitable to use
// shuffles.
if (std::abs(Dist) == 0)
return LookAheadHeuristics::ScoreSplat;
if (std::abs(Dist) > NumLanes / 2)
return LookAheadHeuristics::ScoreSameOpcode;
return (Dist > 0) ? LookAheadHeuristics::ScoreConsecutiveExtracts
: LookAheadHeuristics::ScoreReversedExtracts;
}
return LookAheadHeuristics::ScoreAltOpcodes;
}
return CheckSameEntryOrFail();
}
auto *I1 = dyn_cast<Instruction>(V1);
auto *I2 = dyn_cast<Instruction>(V2);
if (I1 && I2) {
if (I1->getParent() != I2->getParent())
return CheckSameEntryOrFail();
Value *V;
Value *Cond;
// ZExt i1 to something must be considered same opcode for select i1
// cmp, x, y
// Required to better match the transformation after
// BoUpSLP::matchesInversedZExtSelect analysis.
if ((match(I1, m_ZExt(m_Value(V))) &&
match(I2, m_Select(m_Value(Cond), m_Value(), m_Value())) &&
V->getType() == Cond->getType()) ||
(match(I2, m_ZExt(m_Value(V))) &&
match(I1, m_Select(m_Value(Cond), m_Value(), m_Value())) &&
V->getType() == Cond->getType()))
return LookAheadHeuristics::ScoreSameOpcode;
SmallVector<Value *, 4> Ops(MainAltOps);
Ops.push_back(I1);
Ops.push_back(I2);
InstructionsState S = getSameOpcode(Ops, TLI);
// Note: Only consider instructions with <= 2 operands to avoid
// complexity explosion.
if (S &&
(S.getMainOp()->getNumOperands() <= 2 || !MainAltOps.empty() ||
!S.isAltShuffle()) &&
all_of(make_isa_range<Instruction>(Ops), [&S](Instruction *I) {
return I->getNumOperands() == S.getMainOp()->getNumOperands() ||
(S.getOpcode() == Instruction::ICmp && isa<TruncInst>(I));
}))
return S.isAltShuffle() ? LookAheadHeuristics::ScoreAltOpcodes
: LookAheadHeuristics::ScoreSameOpcode;
}
if (I1 && isa<PoisonValue>(V2))
return LookAheadHeuristics::ScoreSameOpcode;
if (isa<UndefValue>(V2))
return LookAheadHeuristics::ScoreUndef;
return CheckSameEntryOrFail();
}
/// Go through the operands of \p LHS and \p RHS recursively until
/// MaxLevel, and return the cummulative score. \p U1 and \p U2 are
/// the users of \p LHS and \p RHS (that is \p LHS and \p RHS are operands
/// of \p U1 and \p U2), except at the beginning of the recursion where
/// these are set to nullptr.
///
/// For example:
/// \verbatim
/// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1]
/// \ / \ / \ / \ /
/// + + + +
/// G1 G2 G3 G4
/// \endverbatim
/// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at
/// each level recursively, accumulating the score. It starts from matching
/// the additions at level 0, then moves on to the loads (level 1). The
/// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and
/// {B[0],B[1]} match with LookAheadHeuristics::ScoreConsecutiveLoads, while
/// {A[0],C[0]} has a score of LookAheadHeuristics::ScoreFail.
/// Please note that the order of the operands does not matter, as we
/// evaluate the score of all profitable combinations of operands. In
/// other words the score of G1 and G4 is the same as G1 and G2. This
/// heuristic is based on ideas described in:
/// Look-ahead SLP: Auto-vectorization in the presence of commutative
/// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha,
/// Luís F. W. Góes
int getScoreAtLevelRec(Value *LHS, Value *RHS, Instruction *U1,
Instruction *U2, int CurrLevel,
ArrayRef<Value *> MainAltOps) const {
// Get the shallow score of V1 and V2.
int ShallowScoreAtThisLevel =
getShallowScore(LHS, RHS, U1, U2, MainAltOps);
// If reached MaxLevel,
// or if V1 and V2 are not instructions,
// or if they are SPLAT,
// or if they are not consecutive,
// or if profitable to vectorize loads or extractelements, early return
// the current cost.
auto *I1 = dyn_cast<Instruction>(LHS);
auto *I2 = dyn_cast<Instruction>(RHS);
if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
ShallowScoreAtThisLevel == LookAheadHeuristics::ScoreFail ||
(((isa<LoadInst>(I1) && isa<LoadInst>(I2)) ||
(I1->getNumOperands() > 2 && I2->getNumOperands() > 2) ||
(isa<ExtractElementInst>(I1) && isa<ExtractElementInst>(I2))) &&
ShallowScoreAtThisLevel))
return ShallowScoreAtThisLevel;
assert(I1 && I2 && "Should have early exited.");
// Contains the I2 operand indexes that got matched with I1 operands.
SmallSet<unsigned, 4> Op2Used;
// Recursion towards the operands of I1 and I2. We are trying all possible
// operand pairs, and keeping track of the best score.
if (I1->getNumOperands() != I2->getNumOperands())
return LookAheadHeuristics::ScoreSameOpcode;
for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
OpIdx1 != NumOperands1; ++OpIdx1) {
// Try to pair op1I with the best operand of I2.
int MaxTmpScore = 0;
unsigned MaxOpIdx2 = 0;
bool FoundBest = false;
// If I2 is commutative try all combinations.
unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1;
unsigned ToIdx = isCommutative(I2)
? I2->getNumOperands()
: std::min(I2->getNumOperands(), OpIdx1 + 1);
assert(FromIdx <= ToIdx && "Bad index");
for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) {
// Skip operands already paired with OpIdx1.
if (Op2Used.count(OpIdx2))
continue;
// Recursively calculate the cost at each level
int TmpScore =
getScoreAtLevelRec(I1->getOperand(OpIdx1), I2->getOperand(OpIdx2),
I1, I2, CurrLevel + 1, {});
// Look for the best score.
if (TmpScore > LookAheadHeuristics::ScoreFail &&
TmpScore > MaxTmpScore) {
MaxTmpScore = TmpScore;
MaxOpIdx2 = OpIdx2;
FoundBest = true;
}
}
if (FoundBest) {
// Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it.
Op2Used.insert(MaxOpIdx2);
ShallowScoreAtThisLevel += MaxTmpScore;
}
}
return ShallowScoreAtThisLevel;
}
};
/// A helper data structure to hold the operands of a vector of instructions.
/// This supports a fixed vector length for all operand vectors.
class VLOperands {
/// For each operand we need (i) the value, and (ii) the opcode that it
/// would be attached to if the expression was in a left-linearized form.
/// This is required to avoid illegal operand reordering.
/// For example:
/// \verbatim
/// 0 Op1
/// |/
/// Op1 Op2 Linearized + Op2
/// \ / ----------> |/
/// - -
///
/// Op1 - Op2 (0 + Op1) - Op2
/// \endverbatim
///
/// Value Op1 is attached to a '+' operation, and Op2 to a '-'.
///
/// Another way to think of this is to track all the operations across the
/// path from the operand all the way to the root of the tree and to
/// calculate the operation that corresponds to this path. For example, the
/// path from Op2 to the root crosses the RHS of the '-', therefore the
/// corresponding operation is a '-' (which matches the one in the
/// linearized tree, as shown above).
///
/// For lack of a better term, we refer to this operation as Accumulated
/// Path Operation (APO).
struct OperandData {
OperandData() = default;
OperandData(Value *V, bool APO, bool IsUsed)
: V(V), APO(APO), IsUsed(IsUsed) {}
/// The operand value.
Value *V = nullptr;
/// TreeEntries only allow a single opcode, or an alternate sequence of
/// them (e.g, +, -). Therefore, we can safely use a boolean value for the
/// APO. It is set to 'true' if 'V' is attached to an inverse operation
/// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise
/// (e.g., Add/Mul)
bool APO = false;
/// Helper data for the reordering function.
bool IsUsed = false;
};
/// During operand reordering, we are trying to select the operand at lane
/// that matches best with the operand at the neighboring lane. Our
/// selection is based on the type of value we are looking for. For example,
/// if the neighboring lane has a load, we need to look for a load that is
/// accessing a consecutive address. These strategies are summarized in the
/// 'ReorderingMode' enumerator.
enum class ReorderingMode {
Load, ///< Matching loads to consecutive memory addresses
Opcode, ///< Matching instructions based on opcode (same or alternate)
Constant, ///< Matching constants
Splat, ///< Matching the same instruction multiple times (broadcast)
Failed, ///< We failed to create a vectorizable group
};
using OperandDataVec = SmallVector<OperandData, 2>;
/// A vector of operand vectors.
SmallVector<OperandDataVec, 4> OpsVec;
/// When VL[0] is IntrinsicInst, ArgSize is CallBase::arg_size. When VL[0]
/// is not IntrinsicInst, ArgSize is User::getNumOperands.
unsigned ArgSize = 0;
const TargetLibraryInfo &TLI;
const DataLayout &DL;
ScalarEvolution &SE;
const BoUpSLP &R;
const Loop *L = nullptr;
/// \returns the operand data at \p OpIdx and \p Lane.
OperandData &getData(unsigned OpIdx, unsigned Lane) {
return OpsVec[OpIdx][Lane];
}
/// \returns the operand data at \p OpIdx and \p Lane. Const version.
const OperandData &getData(unsigned OpIdx, unsigned Lane) const {
return OpsVec[OpIdx][Lane];
}
/// Clears the used flag for all entries.
void clearUsed() {
for (unsigned OpIdx = 0, NumOperands = getNumOperands();
OpIdx != NumOperands; ++OpIdx)
for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
++Lane)
OpsVec[OpIdx][Lane].IsUsed = false;
}
/// Swap the operand at \p OpIdx1 with that one at \p OpIdx2.
void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) {
std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
}
/// \param Lane lane of the operands under analysis.
/// \param OpIdx operand index in \p Lane lane we're looking the best
/// candidate for.
/// \param Idx operand index of the current candidate value.
/// \returns The additional score due to possible broadcasting of the
/// elements in the lane. It is more profitable to have power-of-2 unique
/// elements in the lane, it will be vectorized with higher probability
/// after removing duplicates. Currently the SLP vectorizer supports only
/// vectorization of the power-of-2 number of unique scalars.
int getSplatScore(unsigned Lane, unsigned OpIdx, unsigned Idx,
const SmallBitVector &UsedLanes) const {
Value *IdxLaneV = getData(Idx, Lane).V;
if (!isa<Instruction>(IdxLaneV) || IdxLaneV == getData(OpIdx, Lane).V ||
isa<ExtractElementInst>(IdxLaneV))
return 0;
SmallDenseMap<Value *, unsigned, 4> Uniques;
for (unsigned Ln : seq<unsigned>(getNumLanes())) {
if (Ln == Lane)
continue;
Value *OpIdxLnV = getData(OpIdx, Ln).V;
if (!isa<Instruction>(OpIdxLnV))
return 0;
Uniques.try_emplace(OpIdxLnV, Ln);
}
unsigned UniquesCount = Uniques.size();
auto IdxIt = Uniques.find(IdxLaneV);
unsigned UniquesCntWithIdxLaneV =
IdxIt != Uniques.end() ? UniquesCount : UniquesCount + 1;
Value *OpIdxLaneV = getData(OpIdx, Lane).V;
auto OpIdxIt = Uniques.find(OpIdxLaneV);
unsigned UniquesCntWithOpIdxLaneV =
OpIdxIt != Uniques.end() ? UniquesCount : UniquesCount + 1;
if (UniquesCntWithIdxLaneV == UniquesCntWithOpIdxLaneV)
return 0;
return std::min(bit_ceil(UniquesCntWithOpIdxLaneV) -
UniquesCntWithOpIdxLaneV,
UniquesCntWithOpIdxLaneV -
bit_floor(UniquesCntWithOpIdxLaneV)) -
((IdxIt != Uniques.end() && UsedLanes.test(IdxIt->second))
? UniquesCntWithIdxLaneV - bit_floor(UniquesCntWithIdxLaneV)
: bit_ceil(UniquesCntWithIdxLaneV) - UniquesCntWithIdxLaneV);
}
/// \param Lane lane of the operands under analysis.
/// \param OpIdx operand index in \p Lane lane we're looking the best
/// candidate for.
/// \param Idx operand index of the current candidate value.
/// \returns The additional score for the scalar which users are all
/// vectorized.
int getExternalUseScore(unsigned Lane, unsigned OpIdx, unsigned Idx) const {
Value *IdxLaneV = getData(Idx, Lane).V;
Value *OpIdxLaneV = getData(OpIdx, Lane).V;
// Do not care about number of uses for vector-like instructions
// (extractelement/extractvalue with constant indices), they are extracts
// themselves and already externally used. Vectorization of such
// instructions does not add extra extractelement instruction, just may
// remove it.
if (isVectorLikeInstWithConstOps(IdxLaneV) &&
isVectorLikeInstWithConstOps(OpIdxLaneV))
return LookAheadHeuristics::ScoreAllUserVectorized;
auto *IdxLaneI = dyn_cast<Instruction>(IdxLaneV);
if (!IdxLaneI || !isa<Instruction>(OpIdxLaneV))
return 0;
return R.areAllUsersVectorized(IdxLaneI)
? LookAheadHeuristics::ScoreAllUserVectorized
: 0;
}
/// Score scaling factor for fully compatible instructions but with
/// different number of external uses. Allows better selection of the
/// instructions with less external uses.
static constexpr int ScoreScaleFactor = 10;
/// Scale factor for constants only.
static constexpr int ScoreConstantScaleFactor = 6;
/// \Returns the look-ahead score, which tells us how much the sub-trees
/// rooted at \p LHS and \p RHS match, the more they match the higher the
/// score. This helps break ties in an informed way when we cannot decide on
/// the order of the operands by just considering the immediate
/// predecessors.
int getLookAheadScore(Value *LHS, Value *RHS, ArrayRef<Value *> MainAltOps,
int Lane, unsigned OpIdx, unsigned Idx,
bool &IsUsed, const SmallBitVector &UsedLanes) {
LookAheadHeuristics LookAhead(TLI, DL, SE, R, getNumLanes(),
R.Opts.slp_max_look_ahead_depth);
// Keep track of the instruction stack as we recurse into the operands
// during the look-ahead score exploration.
int Score =
LookAhead.getScoreAtLevelRec(LHS, RHS, /*U1=*/nullptr, /*U2=*/nullptr,
/*CurrLevel=*/1, MainAltOps);
if (Score) {
int SplatScore =
getSplatScore(Lane, OpIdx, Idx, UsedLanes) * ScoreScaleFactor;
if (Score <= -SplatScore) {
// Failed score.
Score = 0;
} else {
Score += SplatScore;
// Scale score to see the difference between different operands
// and similar operands but all vectorized/not all vectorized
// uses. It does not affect actual selection of the best
// compatible operand in general, just allows to select the
// operand with all vectorized uses.
const int SF = (LHS == RHS && isConstant(LHS))
? ScoreConstantScaleFactor
: ScoreScaleFactor;
Score *= SF;
Score += getExternalUseScore(Lane, OpIdx, Idx);
IsUsed = true;
}
}
return Score;
}
/// Best defined scores per lanes between the passes. Used to choose the
/// best operand (with the highest score) between the passes.
/// The key - {Operand Index, Lane}.
/// The value - the best score between the passes for the lane and the
/// operand.
SmallDenseMap<std::pair<unsigned, unsigned>, unsigned, 8>
BestScoresPerLanes;
// Search all operands in Ops[*][Lane] for the one that matches best
// Ops[OpIdx][LastLane] and return its opreand index.
// If no good match can be found, return std::nullopt.
std::optional<unsigned>
getBestOperand(unsigned OpIdx, int Lane, int LastLane,
ArrayRef<ReorderingMode> ReorderingModes,
ArrayRef<Value *> MainAltOps,
const SmallBitVector &UsedLanes) {
unsigned NumOperands = getNumOperands();
// The operand of the previous lane at OpIdx.
Value *OpLastLane = getData(OpIdx, LastLane).V;
// Our strategy mode for OpIdx.
ReorderingMode RMode = ReorderingModes[OpIdx];
if (RMode == ReorderingMode::Failed)
return std::nullopt;
// The linearized opcode of the operand at OpIdx, Lane.
bool OpIdxAPO = getData(OpIdx, Lane).APO;
// The best operand index and its score.
// Sometimes we have more than one option (e.g., Opcode and Undefs), so we
// are using the score to differentiate between the two.
struct BestOpData {
std::optional<unsigned> Idx;
unsigned Score = 0;
} BestOp;
BestOp.Score =
BestScoresPerLanes.try_emplace(std::make_pair(OpIdx, Lane), 0)
.first->second;
// Track if the operand must be marked as used. If the operand is set to
// Score 1 explicitly (because of non power-of-2 unique scalars, we may
// want to reestimate the operands again on the following iterations).
bool IsUsed = RMode == ReorderingMode::Splat ||
RMode == ReorderingMode::Constant ||
RMode == ReorderingMode::Load;
// Iterate through all unused operands and look for the best.
for (unsigned Idx = 0; Idx != NumOperands; ++Idx) {
// Get the operand at Idx and Lane.
OperandData &OpData = getData(Idx, Lane);
Value *Op = OpData.V;
bool OpAPO = OpData.APO;
// Skip already selected operands.
if (OpData.IsUsed)
continue;
// Skip if we are trying to move the operand to a position with a
// different opcode in the linearized tree form. This would break the
// semantics.
if (OpAPO != OpIdxAPO)
continue;
// Look for an operand that matches the current mode.
switch (RMode) {
case ReorderingMode::Load:
case ReorderingMode::Opcode: {
bool LeftToRight = Lane > LastLane;
Value *OpLeft = (LeftToRight) ? OpLastLane : Op;
Value *OpRight = (LeftToRight) ? Op : OpLastLane;
int Score = getLookAheadScore(OpLeft, OpRight, MainAltOps, Lane,
OpIdx, Idx, IsUsed, UsedLanes);
if (Score > static_cast<int>(BestOp.Score) ||
(Score > 0 && Score == static_cast<int>(BestOp.Score) &&
Idx == OpIdx)) {
BestOp.Idx = Idx;
BestOp.Score = Score;
BestScoresPerLanes[std::make_pair(OpIdx, Lane)] = Score;
}
break;
}
case ReorderingMode::Constant:
if (isa<Constant>(Op) ||
(!BestOp.Score && L && L->isLoopInvariant(Op))) {
BestOp.Idx = Idx;
if (isa<Constant>(Op)) {
BestOp.Score = LookAheadHeuristics::ScoreConstants;
BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
LookAheadHeuristics::ScoreConstants;
}
if (isa<UndefValue>(Op) || !isa<Constant>(Op))
IsUsed = false;
}
break;
case ReorderingMode::Splat:
if (Op == OpLastLane || (!BestOp.Score && isa<Constant>(Op))) {
IsUsed = Op == OpLastLane;
if (Op == OpLastLane) {
BestOp.Score = LookAheadHeuristics::ScoreSplat;
BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
LookAheadHeuristics::ScoreSplat;
}
BestOp.Idx = Idx;
}
break;
case ReorderingMode::Failed:
llvm_unreachable("Not expected Failed reordering mode.");
}
}
if (BestOp.Idx) {
getData(*BestOp.Idx, Lane).IsUsed = IsUsed;
return BestOp.Idx;
}
// If we could not find a good match return std::nullopt.
return std::nullopt;
}
/// Helper for reorderOperandVecs.
/// \returns the lane that we should start reordering from. This is the one
/// which has the least number of operands that can freely move about or
/// less profitable because it already has the most optimal set of operands.
unsigned getBestLaneToStartReordering() const {
unsigned Min = UINT_MAX;
unsigned SameOpNumber = 0;
// std::pair<unsigned, unsigned> is used to implement a simple voting
// algorithm and choose the lane with the least number of operands that
// can freely move about or less profitable because it already has the
// most optimal set of operands. The first unsigned is a counter for
// voting, the second unsigned is the counter of lanes with instructions
// with same/alternate opcodes and same parent basic block.
MapVector<unsigned, std::pair<unsigned, unsigned>> HashMap;
// Try to be closer to the original results, if we have multiple lanes
// with same cost. If 2 lanes have the same cost, use the one with the
// highest index.
for (int I = getNumLanes(); I > 0; --I) {
unsigned Lane = I - 1;
OperandsOrderData NumFreeOpsHash =
getMaxNumOperandsThatCanBeReordered(Lane);
// Compare the number of operands that can move and choose the one with
// the least number.
if (NumFreeOpsHash.NumOfAPOs < Min) {
Min = NumFreeOpsHash.NumOfAPOs;
SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
HashMap.clear();
HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
} else if (NumFreeOpsHash.NumOfAPOs == Min &&
NumFreeOpsHash.NumOpsWithSameOpcodeParent < SameOpNumber) {
// Select the most optimal lane in terms of number of operands that
// should be moved around.
SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
} else if (NumFreeOpsHash.NumOfAPOs == Min &&
NumFreeOpsHash.NumOpsWithSameOpcodeParent == SameOpNumber) {
auto [It, Inserted] =
HashMap.try_emplace(NumFreeOpsHash.Hash, 1, Lane);
if (!Inserted)
++It->second.first;
}
}
// Select the lane with the minimum counter.
unsigned BestLane = 0;
unsigned CntMin = UINT_MAX;
for (const auto &Data : reverse(HashMap)) {
if (Data.second.first < CntMin) {
CntMin = Data.second.first;
BestLane = Data.second.second;
}
}
return BestLane;
}
/// Data structure that helps to reorder operands.
struct OperandsOrderData {
/// The best number of operands with the same APOs, which can be
/// reordered.
unsigned NumOfAPOs = UINT_MAX;
/// Number of operands with the same/alternate instruction opcode and
/// parent.
unsigned NumOpsWithSameOpcodeParent = 0;
/// Hash for the actual operands ordering.
/// Used to count operands, actually their position id and opcode
/// value. It is used in the voting mechanism to find the lane with the
/// least number of operands that can freely move about or less profitable
/// because it already has the most optimal set of operands. Can be
/// replaced with SmallVector<unsigned> instead but hash code is faster
/// and requires less memory.
unsigned Hash = 0;
};
/// \returns the maximum number of operands that are allowed to be reordered
/// for \p Lane and the number of compatible instructions(with the same
/// parent/opcode). This is used as a heuristic for selecting the first lane
/// to start operand reordering.
OperandsOrderData getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
unsigned CntTrue = 0;
unsigned NumOperands = getNumOperands();
// Operands with the same APO can be reordered. We therefore need to count
// how many of them we have for each APO, like this: Cnt[APO] = x.
// Since we only have two APOs, namely true and false, we can avoid using
// a map. Instead we can simply count the number of operands that
// correspond to one of them (in this case the 'true' APO), and calculate
// the other by subtracting it from the total number of operands.
// Operands with the same instruction opcode and parent are more
// profitable since we don't need to move them in many cases, with a high
// probability such lane already can be vectorized effectively.
bool AllUndefs = true;
unsigned NumOpsWithSameOpcodeParent = 0;
Instruction *OpcodeI = nullptr;
BasicBlock *Parent = nullptr;
unsigned Hash = 0;
for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
const OperandData &OpData = getData(OpIdx, Lane);
if (OpData.APO)
++CntTrue;
// Use Boyer-Moore majority voting for finding the majority opcode and
// the number of times it occurs.
if (auto *I = dyn_cast<Instruction>(OpData.V)) {
if (!OpcodeI || !getSameOpcode({OpcodeI, I}, TLI) ||
I->getParent() != Parent) {
if (NumOpsWithSameOpcodeParent == 0) {
NumOpsWithSameOpcodeParent = 1;
OpcodeI = I;
Parent = I->getParent();
} else {
--NumOpsWithSameOpcodeParent;
}
} else {
++NumOpsWithSameOpcodeParent;
}
}
Hash = hash_combine(
Hash, hash_value((OpIdx + 1) * (OpData.V->getValueID() + 1)));
AllUndefs = AllUndefs && isa<UndefValue>(OpData.V);
}
if (AllUndefs)
return {};
OperandsOrderData Data;
Data.NumOfAPOs = std::max(CntTrue, NumOperands - CntTrue);
Data.NumOpsWithSameOpcodeParent = NumOpsWithSameOpcodeParent;
Data.Hash = Hash;
return Data;
}
/// Go through the instructions in VL and append their operands.
void appendOperands(ArrayRef<Value *> VL, ArrayRef<ValueList> Operands,
const InstructionsState &S) {
assert(!Operands.empty() && !VL.empty() && "Bad list of operands");
assert((empty() || all_of(Operands,
[this](const ValueList &VL) {
return VL.size() == getNumLanes();
})) &&
"Expected same number of lanes");
assert(S.valid() && "InstructionsState is invalid.");
// IntrinsicInst::isCommutative returns true if swapping the first "two"
// arguments to the intrinsic produces the same result.
Instruction *MainOp = S.getMainOp();
ArgSize = getNumberOfPotentiallyCommutativeOps(MainOp);
OpsVec.resize(ArgSize);
unsigned NumLanes = VL.size();
for (OperandDataVec &Ops : OpsVec)
Ops.resize(NumLanes);
for (unsigned Lane : seq<unsigned>(NumLanes)) {
// Our tree has just 3 nodes: the root and two operands.
// It is therefore trivial to get the APO. We only need to check the
// opcode of V and whether the operand at OpIdx is the LHS or RHS
// operand. The LHS operand of both add and sub is never attached to an
// inversese operation in the linearized form, therefore its APO is
// false. The RHS is true only if V is an inverse operation.
// Since operand reordering is performed on groups of commutative
// operations or alternating sequences (e.g., +, -), we can safely tell
// the inverse operations by checking commutativity.
auto *I = dyn_cast<Instruction>(VL[Lane]);
if (!I && isa<PoisonValue>(VL[Lane])) {
for (unsigned OpIdx : seq<unsigned>(ArgSize))
OpsVec[OpIdx][Lane] = {Operands[OpIdx][Lane], true, false};
continue;
}
bool IsInverseOperation = false;
if (S.isCopyableElement(VL[Lane])) {
// The value is a copyable element.
IsInverseOperation =
!isCommutative(MainOp, VL[Lane], /*IsCopyable=*/true);
} else {
assert(I && "Expected instruction");
Instruction *SelectedOp = convertTo(I, S).first;
// We cannot check commutativity by the converted instruction
// (SelectedOp) because isCommutative also examines def-use
// relationships.
IsInverseOperation = !isCommutative(SelectedOp, I);
}
for (unsigned OpIdx : seq<unsigned>(ArgSize)) {
bool APO = (OpIdx == 0) ? false : IsInverseOperation;
OpsVec[OpIdx][Lane] = {Operands[OpIdx][Lane], APO, false};
}
}
}
/// \returns the number of operands.
unsigned getNumOperands() const { return ArgSize; }
/// \returns the number of lanes.
unsigned getNumLanes() const { return OpsVec[0].size(); }
/// \returns the operand value at \p OpIdx and \p Lane.
Value *getValue(unsigned OpIdx, unsigned Lane) const {
return getData(OpIdx, Lane).V;
}
/// \returns true if the data structure is empty.
bool empty() const { return OpsVec.empty(); }
/// Clears the data.
void clear() { OpsVec.clear(); }
/// \Returns true if there are enough operands identical to \p Op to fill
/// the whole vector (it is mixed with constants or loop invariant values).
/// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
assert(Op == getValue(OpIdx, Lane) &&
"Op is expected to be getValue(OpIdx, Lane).");
// Small number of loads - try load matching.
if (isa<LoadInst>(Op) && getNumLanes() == 2 && getNumOperands() == 2)
return false;
bool OpAPO = getData(OpIdx, Lane).APO;
bool IsInvariant = L && L->isLoopInvariant(Op);
unsigned Cnt = 0;
for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
if (Ln == Lane)
continue;
// This is set to true if we found a candidate for broadcast at Lane.
bool FoundCandidate = false;
for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
OperandData &Data = getData(OpI, Ln);
if (Data.APO != OpAPO || Data.IsUsed)
continue;
Value *OpILane = getValue(OpI, Lane);
bool IsConstantOp = isa<Constant>(OpILane);
// Consider the broadcast candidate if:
// 1. Same value is found in one of the operands.
if (Data.V == Op ||
// 2. The operand in the given lane is not constant but there is a
// constant operand in another lane (which can be moved to the
// given lane). In this case we can represent it as a simple
// permutation of constant and broadcast.
(!IsConstantOp &&
((Lns > 2 && isa<Constant>(Data.V)) ||
// 2.1. If we have only 2 lanes, need to check that value in the
// next lane does not build same opcode sequence.
(Lns == 2 &&
!getSameOpcode({Op, getValue((OpI + 1) % OpE, Ln)}, TLI) &&
isa<Constant>(Data.V)))) ||
// 3. The operand in the current lane is loop invariant (can be
// hoisted out) and another operand is also a loop invariant
// (though not a constant). In this case the whole vector can be
// hoisted out.
// FIXME: need to teach the cost model about this case for better
// estimation.
(IsInvariant && !isa<Constant>(Data.V) &&
!getSameOpcode({Op, Data.V}, TLI) &&
L->isLoopInvariant(Data.V))) {
FoundCandidate = true;
Data.IsUsed = Data.V == Op;
if (Data.V == Op)
++Cnt;
break;
}
}
if (!FoundCandidate)
return false;
}
return getNumLanes() == 2 || Cnt > 1;
}
/// Checks if there is at least single compatible operand in lanes other
/// than \p Lane, compatible with the operand \p Op.
bool canBeVectorized(Instruction *Op, unsigned OpIdx, unsigned Lane) const {
assert(Op == getValue(OpIdx, Lane) &&
"Op is expected to be getValue(OpIdx, Lane).");
bool OpAPO = getData(OpIdx, Lane).APO;
for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
if (Ln == Lane)
continue;
if (any_of(seq<unsigned>(getNumOperands()), [&](unsigned OpI) {
const OperandData &Data = getData(OpI, Ln);
if (Data.APO != OpAPO || Data.IsUsed)
return true;
Value *OpILn = getValue(OpI, Ln);
return (L && L->isLoopInvariant(OpILn)) ||
(getSameOpcode({Op, OpILn}, TLI) &&
allSameBlock({Op, OpILn}));
}))
return true;
}
return false;
}
public:
/// Initialize with all the operands of the instruction vector \p RootVL.
VLOperands(ArrayRef<Value *> RootVL, ArrayRef<ValueList> Operands,
const InstructionsState &S, const BoUpSLP &R)
: TLI(*R.TLI), DL(*R.DL), SE(*R.SE), R(R),
L(R.LI->getLoopFor(S.getMainOp()->getParent())) {
// Append all the operands of RootVL.
appendOperands(RootVL, Operands, S);
}
/// Initialize with flattened operand columns of an associative node.
/// ArgSize is taken from \p Operands, APO is always false.
VLOperands(ArrayRef<ValueList> Operands, const BasicBlock *BB,
const BoUpSLP &R)
: TLI(*R.TLI), DL(*R.DL), SE(*R.SE), R(R), L(R.LI->getLoopFor(BB)) {
assert(!Operands.empty() && "Expected at least one operand column");
ArgSize = Operands.size();
OpsVec.resize(ArgSize);
unsigned NumLanes = Operands.front().size();
for (auto [OpIdx, Ops] : enumerate(OpsVec)) {
Ops.resize(NumLanes);
for (unsigned Lane : seq<unsigned>(NumLanes))
Ops[Lane] = OperandData(Operands[OpIdx][Lane], /*APO=*/false,
/*IsUsed=*/false);
}
}
/// \Returns a value vector with the operands across all lanes for the
/// opearnd at \p OpIdx.
ValueList getVL(unsigned OpIdx) const {
ValueList OpVL(OpsVec[OpIdx].size());
assert(OpsVec[OpIdx].size() == getNumLanes() &&
"Expected same num of lanes across all operands");
for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
OpVL[Lane] = OpsVec[OpIdx][Lane].V;
return OpVL;
}
// Performs operand reordering for 2 or more operands.
// The original operands are in OrigOps[OpIdx][Lane].
// The reordered operands are returned in 'SortedOps[OpIdx][Lane]'.
void reorder() {
unsigned NumOperands = getNumOperands();
unsigned NumLanes = getNumLanes();
// Each operand has its own mode. We are using this mode to help us select
// the instructions for each lane, so that they match best with the ones
// we have selected so far.
SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands);
// This is a greedy single-pass algorithm. We are going over each lane
// once and deciding on the best order right away with no back-tracking.
// However, in order to increase its effectiveness, we start with the lane
// that has operands that can move the least. For example, given the
// following lanes:
// Lane 0 : A[0] = B[0] + C[0] // Visited 3rd
// Lane 1 : A[1] = C[1] - B[1] // Visited 1st
// Lane 2 : A[2] = B[2] + C[2] // Visited 2nd
// Lane 3 : A[3] = C[3] - B[3] // Visited 4th
// we will start at Lane 1, since the operands of the subtraction cannot
// be reordered. Then we will visit the rest of the lanes in a circular
// fashion. That is, Lanes 2, then Lane 0, and finally Lane 3.
// Find the first lane that we will start our search from.
unsigned FirstLane = getBestLaneToStartReordering();
// Initialize the modes.
for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
Value *OpLane0 = getValue(OpIdx, FirstLane);
// Keep track if we have instructions with all the same opcode on one
// side.
if (auto *OpILane0 = dyn_cast<Instruction>(OpLane0)) {
// Check if OpLane0 should be broadcast.
if (shouldBroadcast(OpLane0, OpIdx, FirstLane) ||
!canBeVectorized(OpILane0, OpIdx, FirstLane))
ReorderingModes[OpIdx] = ReorderingMode::Splat;
else if (isa<LoadInst>(OpILane0))
ReorderingModes[OpIdx] = ReorderingMode::Load;
else
ReorderingModes[OpIdx] = ReorderingMode::Opcode;
} else if (isa<Constant>(OpLane0)) {
ReorderingModes[OpIdx] = ReorderingMode::Constant;
} else if (isa<Argument>(OpLane0)) {
// Our best hope is a Splat. It may save some cost in some cases.
ReorderingModes[OpIdx] = ReorderingMode::Splat;
} else {
llvm_unreachable("Unexpected value kind.");
}
}
// Check that we don't have same operands. No need to reorder if operands
// are just perfect diamond or shuffled diamond match. Do not do it only
// for possible broadcasts.
auto &&SkipReordering = [this]() {
SmallPtrSet<Value *, 4> UniqueValues;
ArrayRef<OperandData> Op0 = OpsVec.front();
for (const OperandData &Data : Op0)
UniqueValues.insert(Data.V);
for (ArrayRef<OperandData> Op :
ArrayRef(OpsVec).slice(1, getNumOperands() - 1)) {
if (any_of(Op, [&UniqueValues](const OperandData &Data) {
return !UniqueValues.contains(Data.V);
}))
return false;
}
return UniqueValues.size() != 2;
};
// If the initial strategy fails for any of the operand indexes, then we
// perform reordering again in a second pass. This helps avoid assigning
// high priority to the failed strategy, and should improve reordering for
// the non-failed operand indexes.
for (int Pass = 0; Pass != 2; ++Pass) {
// Check if no need to reorder operands since they're are perfect or
// shuffled diamond match.
// Need to do it to avoid extra external use cost counting for
// shuffled matches, which may cause regressions.
if (SkipReordering())
break;
// Skip the second pass if the first pass did not fail.
bool StrategyFailed = false;
// Mark all operand data as free to use.
clearUsed();
// We keep the original operand order for the FirstLane, so reorder the
// rest of the lanes. We are visiting the nodes in a circular fashion,
// using FirstLane as the center point and increasing the radius
// distance.
SmallVector<SmallVector<Value *, 2>> MainAltOps(NumOperands);
for (unsigned I = 0; I < NumOperands; ++I)
MainAltOps[I].push_back(getData(I, FirstLane).V);
SmallBitVector UsedLanes(NumLanes);
UsedLanes.set(FirstLane);
for (unsigned Distance = 1; Distance != NumLanes; ++Distance) {
// Visit the lane on the right and then the lane on the left.
for (int Direction : {+1, -1}) {
int Lane = FirstLane + Direction * Distance;
if (Lane < 0 || Lane >= (int)NumLanes)
continue;
UsedLanes.set(Lane);
int LastLane = Lane - Direction;
assert(LastLane >= 0 && LastLane < (int)NumLanes &&
"Out of bounds");
// Look for a good match for each operand.
for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
// Search for the operand that matches SortedOps[OpIdx][Lane-1].
std::optional<unsigned> BestIdx =
getBestOperand(OpIdx, Lane, LastLane, ReorderingModes,
MainAltOps[OpIdx], UsedLanes);
// By not selecting a value, we allow the operands that follow to
// select a better matching value. We will get a non-null value in
// the next run of getBestOperand().
if (BestIdx) {
// Swap the current operand with the one returned by
// getBestOperand().
swap(OpIdx, *BestIdx, Lane);
} else {
// Enable the second pass.
StrategyFailed = true;
}
// Try to get the alternate opcode and follow it during analysis.
if (MainAltOps[OpIdx].size() != 2) {
OperandData &AltOp = getData(OpIdx, Lane);
InstructionsState OpS =
getSameOpcode({MainAltOps[OpIdx].front(), AltOp.V}, TLI);
if (OpS && OpS.isAltShuffle())
MainAltOps[OpIdx].push_back(AltOp.V);
}
}
}
}
// Skip second pass if the strategy did not fail.
if (!StrategyFailed)
break;
}
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD static StringRef getModeStr(ReorderingMode RMode) {
switch (RMode) {
case ReorderingMode::Load:
return "Load";
case ReorderingMode::Opcode:
return "Opcode";
case ReorderingMode::Constant:
return "Constant";
case ReorderingMode::Splat:
return "Splat";
case ReorderingMode::Failed:
return "Failed";
}
llvm_unreachable("Unimplemented Reordering Type");
}
LLVM_DUMP_METHOD static raw_ostream &printMode(ReorderingMode RMode,
raw_ostream &OS) {
return OS << getModeStr(RMode);
}
/// Debug print.
LLVM_DUMP_METHOD static void dumpMode(ReorderingMode RMode) {
printMode(RMode, dbgs());
}
friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) {
return printMode(RMode, OS);
}
LLVM_DUMP_METHOD raw_ostream &print(raw_ostream &OS) const {
const unsigned Indent = 2;
unsigned Cnt = 0;
for (const OperandDataVec &OpDataVec : OpsVec) {
OS << "Operand " << Cnt++ << "\n";
for (const OperandData &OpData : OpDataVec) {
OS.indent(Indent) << "{";
if (Value *V = OpData.V)
OS << *V;
else
OS << "null";
OS << ", APO:" << OpData.APO << "}\n";
}
OS << "\n";
}
return OS;
}
/// Debug print.
LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
#endif
};
/// Evaluate each pair in \p Candidates and return index into \p Candidates
/// for a pair which have highest score deemed to have best chance to form
/// root of profitable tree to vectorize. Return std::nullopt if no candidate
/// scored above the LookAheadHeuristics::ScoreFail. \param Limit Lower limit
/// of the cost, considered to be good enough score.
std::pair<std::optional<int>, int>
findBestRootPair(ArrayRef<std::pair<Value *, Value *>> Candidates,
int Limit = LookAheadHeuristics::ScoreFail) const {
LookAheadHeuristics LookAhead(*TLI, *DL, *SE, *this, /*NumLanes=*/2,
Opts.slp_max_root_look_ahead_depth);
int BestScore = Limit;
std::optional<int> Index;
for (int I : seq<int>(0, Candidates.size())) {
int Score = LookAhead.getScoreAtLevelRec(Candidates[I].first,
Candidates[I].second,
/*U1=*/nullptr, /*U2=*/nullptr,
/*CurrLevel=*/1, {});
if (Score > BestScore) {
BestScore = Score;
Index = I;
}
}
return std::make_pair(Index, BestScore);
}
/// Checks if the instruction is marked for deletion.
bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
/// Cached llvm::getPointersDiff(), see SLPAnalysisCache.
std::optional<int64_t>
getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, Value *PtrB,
const DataLayout &DL, ScalarEvolution &SE,
bool StrictCheck = false, bool CheckType = true) const {
return Cache.getPointersDiff(ElemTyA, PtrA, ElemTyB, PtrB, DL, SE,
StrictCheck, CheckType);
}
/// Returns the cached analyses results, shared by all the trees built for
/// the function.
SLPAnalysisCache &getAnalysisCache() const { return Cache; }
/// Checks if the value is used only by the assume-like intrinsics.
bool isEphemeralValue(const Value *V) const { return EphValues.contains(V); }
/// Removes an instruction from its block and eventually deletes it.
/// It's like Instruction::eraseFromParent() except that the actual deletion
/// is delayed until BoUpSLP is destructed.
void eraseInstruction(Instruction *I) {
DeletedInstructions.insert(I);
}
/// Remove instructions from the parent function and clear the operands of \p
/// DeadVals instructions, marking for deletion trivially dead operands.
template <typename T>
void removeInstructionsAndOperands(
ArrayRef<T *> DeadVals,
ArrayRef<ReductionVectorPart> VectorValuesAndScales) {
SmallVector<WeakTrackingVH> DeadInsts;
for (T *V : DeadVals) {
auto *I = cast<Instruction>(V);
eraseInstruction(I);
}
DenseSet<Value *> Processed;
for (T *V : DeadVals) {
if (!V || !Processed.insert(V).second)
continue;
auto *I = cast<Instruction>(V);
salvageDebugInfo(*I);
ArrayRef<TreeEntry *> Entries = getTreeEntries(I);
for (Use &U : I->operands()) {
if (auto *OpI = dyn_cast<Instruction>(U.get());
OpI && !DeletedInstructions.contains(OpI) && OpI->hasOneUser() &&
wouldInstructionBeTriviallyDead(OpI, TLI) &&
!ExternalUseReplacements.contains(OpI) &&
(Entries.empty() || none_of(Entries, [&](const TreeEntry *Entry) {
return Entry->VectorizedValue == OpI;
})))
DeadInsts.push_back(OpI);
}
I->dropAllReferences();
}
for (T *V : DeadVals) {
auto *I = cast<Instruction>(V);
if (!I->getParent())
continue;
assert((I->use_empty() || all_of(I->uses(),
[&](Use &U) {
return isDeleted(
cast<Instruction>(U.getUser()));
})) &&
"trying to erase instruction with users.");
I->removeFromParent();
SE->forgetValue(I);
}
// Process the dead instruction list until empty.
while (!DeadInsts.empty()) {
Value *V = DeadInsts.pop_back_val();
Instruction *VI = cast<Instruction>(V);
if (!VI || !VI->getParent())
continue;
assert(isInstructionTriviallyDead(VI, TLI) &&
"Live instruction found in dead worklist!");
assert(VI->use_empty() && "Instructions with uses are not dead.");
// Don't lose the debug info while deleting the instructions.
salvageDebugInfo(*VI);
// Null out all of the instruction's operands to see if any operand
// becomes dead as we go.
for (Use &OpU : VI->operands()) {
Value *OpV = OpU.get();
if (!OpV)
continue;
OpU.set(nullptr);
if (!OpV->use_empty())
continue;
// If the operand is an instruction that became dead as we nulled out
// the operand, and if it is 'trivially' dead, delete it in a future
// loop iteration.
if (auto *OpI = dyn_cast<Instruction>(OpV))
if (!DeletedInstructions.contains(OpI) &&
!ExternalUseReplacements.contains(OpI) &&
(!OpI->getType()->isVectorTy() ||
none_of(VectorValuesAndScales,
[&](const ReductionVectorPart &V) {
return V.Vec == OpI;
})) &&
isInstructionTriviallyDead(OpI, TLI))
DeadInsts.push_back(OpI);
}
VI->removeFromParent();
eraseInstruction(VI);
SE->forgetValue(VI);
}
}
/// Checks if the instruction was already analyzed for being possible
/// reduction root.
bool isAnalyzedReductionRoot(Instruction *I) const {
return AnalyzedReductionsRoots.count(I);
}
/// Register given instruction as already analyzed for being possible
/// reduction root.
void analyzedReductionRoot(Instruction *I) {
AnalyzedReductionsRoots.insert(I);
}
/// Checks if the provided list of reduced values was checked already for
/// vectorization.
bool areAnalyzedReductionVals(ArrayRef<Value *> VL) const {
return AnalyzedReductionVals.contains(hash_value(VL));
}
/// Adds the list of reduced values to list of already checked values for the
/// vectorization.
void analyzedReductionVals(ArrayRef<Value *> VL) {
AnalyzedReductionVals.insert(hash_value(VL));
}
/// Checks if the value was already a part of the analyzed vector node.
bool isAnalyzedScalar(const Value *V) const {
return AnalyzedScalars.contains(V);
}
/// Checks if the given bundle was already rejected as non-vectorizable.
bool isAnalyzedBundle(ArrayRef<Value *> VL) const {
return AnalyzedBundles.contains(hash_value(VL));
}
/// Registers the bundle as rejected for the vectorization.
void analyzedBundle(ArrayRef<Value *> VL) {
AnalyzedBundles.insert(hash_value(VL));
}
/// Clear the list of the analyzed reduction root instructions.
void clearReductionData() {
AnalyzedReductionsRoots.clear();
AnalyzedReductionVals.clear();
AnalyzedBundles.clear();
AnalyzedMinBWVals.clear();
}
/// Checks if the given value is gathered in one of the nodes.
bool isAnyGathered(const SmallDenseSet<Value *> &Vals) const {
return any_of(MustGather, [&](Value *V) { return Vals.contains(V); });
}
/// Checks if the given value is gathered in one of the nodes.
bool isGathered(const Value *V) const {
return MustGather.contains(V);
}
/// Checks if the specified value was not schedule.
bool isNotScheduled(const Value *V) const {
return NonScheduledFirst.contains(V);
}
/// Check if \p V is a peeled reassociated scalar still owned by a live
/// (non-deleted, non-gathered) tree entry.
bool isReassocScalarVectorized(const Value *V) const {
auto It = ReassocScalarToTreeEntries.find(V);
return It != ReassocScalarToTreeEntries.end() &&
any_of(It->second, [&](const TreeEntry *E) {
return !DeletedNodes.contains(E) &&
!TransformedToGatherNodes.contains(E);
});
}
/// Check if the value is vectorized in the tree.
bool isVectorized(const Value *V) const {
assert(V && "V cannot be nullptr.");
if (isReassocScalarVectorized(V))
return true;
return any_of(getTreeEntries(V), [&](const TreeEntry *E) {
return !DeletedNodes.contains(E) && !TransformedToGatherNodes.contains(E);
});
}
/// Returns true if the role of \p I is already decided by its user: a deleted
/// user was folded into some other vector by an earlier attempt.
bool hasResolvedUser(Instruction *I) const {
return any_of(I->users(), [&](User *U) {
auto *UI = dyn_cast<Instruction>(U);
return UI && isDeleted(UI);
});
}
/// Checks if it is legal and profitable to build SplitVectorize node for the
/// given \p VL.
/// \param Op1 first homogeneous scalars.
/// \param Op2 second homogeneous scalars.
/// \param ReorderIndices indices to reorder the scalars.
/// \returns true if the node was successfully built.
bool canBuildSplitNode(ArrayRef<Value *> VL,
const InstructionsState &LocalState,
SmallVectorImpl<Value *> &Op1,
SmallVectorImpl<Value *> &Op2,
OrdersType &ReorderIndices) const;
/// \returns Cast context for the given graph node.
TargetTransformInfo::CastContextHint
getCastContextHint(const TreeEntry &TE) const;
~BoUpSLP();
private:
/// Determine if a node \p E in can be demoted to a smaller type with a
/// truncation. We collect the entries that will be demoted in ToDemote.
/// \param E Node for analysis
/// \param ToDemote indices of the nodes to be demoted.
bool collectValuesToDemote(
const TreeEntry &E, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
SmallVectorImpl<unsigned> &ToDemote, DenseSet<const TreeEntry *> &Visited,
const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
bool &IsProfitableToDemote, bool IsTruncRoot) const;
/// Builds the list of reorderable operands on the edges \p Edges of the \p
/// UserTE, which allow reordering (i.e. the operands can be reordered because
/// they have only one user and reordarable).
/// \param ReorderableGathers List of all gather nodes that require reordering
/// (e.g., gather of extractlements or partially vectorizable loads).
/// \param GatherOps List of gather operand nodes for \p UserTE that require
/// reordering, subset of \p NonVectorized.
void buildReorderableOperands(
TreeEntry *UserTE,
SmallVectorImpl<std::pair<unsigned, TreeEntry *>> &Edges,
const SmallPtrSetImpl<const TreeEntry *> &ReorderableGathers,
SmallVectorImpl<TreeEntry *> &GatherOps);
/// Checks if the given \p TE is a gather node with clustered reused scalars
/// and reorders it per given \p Mask.
void reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const;
/// Checks if all users of \p I are the part of the vectorization tree.
bool areAllUsersVectorized(
Instruction *I,
const SmallDenseSet<Value *> *VectorizedVals = nullptr) const;
/// Estimates the number of scalar instructions in the tree, each weighted by
/// its loop-nest trip count (nest-invariant entries are dropped when
/// \p TreeLoop is non-null).
uint64_t getNumScalarInsts(bool HasTreeLoop);
/// Estimates the number of vector instructions (including buildvectors,
/// shuffles, and, if \p CountExtracts is set, extracts) the tree produces,
/// weighted like getNumScalarInsts().
uint64_t getNumVectorInsts(bool HasTreeLoop, bool CountExtracts = true);
/// Returns true if the instruction-count veto is skipped for a tree costed at
/// \p TreeCost because it contains a poor-throughput operation whose real
/// vector savings the count check does not model.
bool bypassesInstCountCheck(InstructionCost TreeCost) const;
/// Collects into \p UserOps the user operations of the gathers of the scalars
/// of \p E, for TTI::getBuildVectorContextHint(). Returns false if no user is
/// found or some user cannot be described.
bool
collectGatherUserOps(const TreeEntry &E,
SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) const;
/// \returns the fadd/fsub user of the single-use fmul \p I, which the
/// backend fuses with \p I into an fmuladd in the scalar code, and the cost
/// of that fmuladd, or {nullptr, invalid cost}. If \p InTree is set, the
/// users, whose fusion is accounted for by the tree, are not returned.
std::pair<Instruction *, InstructionCost>
getFMulFusingUser(Instruction *I, bool InTree = true) const;
/// \returns true if the fadd/fsub \p U, fused with its fmul operand by the
/// backend, is a vectorized tree scalar whose lane is priced as fmuladd.
/// Peeled reassociated scalars are priced as plain fadd/fsub.
bool isFusedVectorizedFAddSub(Instruction *U) const;
/// \returns the scalar cost of the fmul lanes of \p TE that are priced as
/// fused into their vectorized fadd/fsub users. If the node is turned into a
/// gather, these fmuls stay scalar and lose the fusion, so the gather
/// alternative must pay their full price.
InstructionCost getUnfusedFMulsPenalty(const TreeEntry &TE) const;
/// \returns the graph entry for the \p Idx operand of the \p E entry.
const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const;
TreeEntry *getOperandEntry(TreeEntry *E, unsigned Idx) {
return const_cast<TreeEntry *>(
getOperandEntry(const_cast<const TreeEntry *>(E), Idx));
}
/// Gets the root instruction for the given node. If the node is a strided
/// load/store node with the reverse order, the root instruction is the last
/// one.
Instruction *getRootEntryInstruction(const TreeEntry &Entry) const;
/// \returns the scale of the given tree entry to the loop iteration.
/// \p Scalar is the scalar value from the entry, if using the parent for the
/// external use.
/// \p U is the user of the vectorized value from the entry, if using the
/// parent for the external use.
uint64_t getScaleToLoopIterations(const TreeEntry &TE,
Value *Scalar = nullptr,
Instruction *U = nullptr);
/// \returns the product of trip counts of the loop \p L and all of its
/// enclosing loops. Unlike the state kept by getScaleToLoopIterations(),
/// this helper depends only on the loop structure and is independent of
/// per-entry operand invariance. Returns 1 when loop-aware cost modeling
/// is disabled or \p L is null.
uint64_t getLoopNestScale(const Loop *L);
/// \returns a refined execution scale for a gather/buildvector tree entry
/// \p TE. The scale is computed as the average of per-lane execution
/// scales: each lane's scale is the loop-nest scale of the loop that
/// contains the lane's defining instruction (or 1 if the lane is a
/// constant / loop-invariant non-instruction value). This models the
/// LICM hoisting that optimizeGatherSequence() performs after vectorization
/// for inserts with loop-invariant operands. Falls back to the whole-entry
/// scale when per-lane information is unavailable or the feature is off.
uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE,
Instruction *U = nullptr);
/// \returns the loop-nest execution scale of \p TE.
uint64_t getEntryEffectiveScale(const TreeEntry &TE,
Instruction *U = nullptr);
/// Get the loop nest for the given loop \p L.
ArrayRef<const Loop *> getLoopNest(const Loop *L);
/// \returns the cost of the vectorizable entry.
InstructionCost getEntryCost(const TreeEntry *E,
ArrayRef<Value *> VectorizedVals,
SmallPtrSetImpl<Value *> &CheckedExtracts);
/// Estimates spill/reload cost from vector register pressure for \p E at the
/// point of emitting its vector result type \p FinalVecTy. \p ScalarTy is the
/// scalar/slot type used to widen into \p VecTy/\p FinalVecTy and may itself
/// be a FixedVectorType in ReVec mode or an adjusted type due to MinBWs.
InstructionCost
getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy, Type *VecTy,
Type *FinalVecTy,
const TTI::TargetCostKind CostKind) const;
/// This is the recursive part of buildTree.
void buildTreeRec(ArrayRef<Value *> Roots, unsigned Depth, const EdgeInfo &EI,
unsigned InterleaveFactor = 0);
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
/// be vectorized to use the original vector (or aggregate "bitcast" to a
/// vector) and sets \p CurrentOrder to the identity permutation; otherwise
/// returns false, setting \p CurrentOrder to either an empty vector or a
/// non-identity permutation that allows to reuse extract instructions.
/// \param ResizeAllowed indicates whether it is allowed to handle subvector
/// extract order.
bool canReuseExtract(ArrayRef<Value *> VL,
SmallVectorImpl<unsigned> &CurrentOrder,
bool ResizeAllowed = false) const;
/// Vectorize a single entry in the tree.
Value *vectorizeTree(TreeEntry *E);
/// Vectorize a single entry in the tree, the \p Idx-th operand of the entry
/// \p E.
Value *vectorizeOperand(TreeEntry *E, unsigned NodeIdx);
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
template <typename BVTy, typename ResTy, typename... Args>
ResTy processBuildVector(const TreeEntry *E, Type *ScalarTy, Args &...Params);
/// Handles the gather of zero-extended sub-fields of the same wider integer
/// scalar, emitted as a bitcast to the field vector plus a permutation and
/// an extension to the lane type, if the gathered values match.
template <typename ResTy, typename BVTy>
std::optional<ResTy> processExtractedFieldsGather(
BVTy &ShuffleBuilder, const TreeEntry *E, Type *ScalarTy,
ArrayRef<Value *> VL,
ArrayRef<std::pair<const TreeEntry *, unsigned>> SubVectors);
/// Create a new vector from a list of scalar values. Produces a sequence
/// which exploits values reused across lanes, and arranges the inserts
/// for ease of later optimization.
Value *createBuildVector(const TreeEntry *E, Type *ScalarTy);
/// Returns the instruction in the bundle, which can be used as a base point
/// for scheduling. Usually it is the last instruction in the bundle, except
/// for the case when all operands are external (in this case, it is the first
/// instruction in the list).
Instruction &getLastInstructionInBundle(const TreeEntry *E);
/// Tries to find extractelement instructions with constant indices from fixed
/// vector type and gather such instructions into a bunch, which highly likely
/// might be detected as a shuffle of 1 or 2 input vectors. If this attempt
/// was successful, the matched scalars are replaced by poison values in \p VL
/// for future analysis.
std::optional<TargetTransformInfo::ShuffleKind>
tryToGatherSingleRegisterExtractElements(MutableArrayRef<Value *> VL,
SmallVectorImpl<int> &Mask) const;
/// Tries to find extractelement instructions with constant indices from fixed
/// vector type and gather such instructions into a bunch, which highly likely
/// might be detected as a shuffle of 1 or 2 input vectors. If this attempt
/// was successful, the matched scalars are replaced by poison values in \p VL
/// for future analysis.
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>>
tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &Mask,
unsigned NumParts) const;
/// Checks if the gathered \p VL can be represented as a single register
/// shuffle(s) of previous tree entries.
/// \param TE Tree entry checked for permutation.
/// \param VL List of scalars (a subset of the TE scalar), checked for
/// permutations. Must form single-register vector.
/// \param ForOrder Tries to fetch the best candidates for ordering info. Also
/// commands to build the mask using the original vector value, without
/// relying on the potential reordering.
/// \returns ShuffleKind, if gathered values can be represented as shuffles of
/// previous tree entries. \p Part of \p Mask is filled with the shuffle mask.
std::optional<TargetTransformInfo::ShuffleKind>
isGatherShuffledSingleRegisterEntry(
const TreeEntry *TE, ArrayRef<Value *> VL, MutableArrayRef<int> Mask,
SmallVectorImpl<const TreeEntry *> &Entries, unsigned Part, bool ForOrder,
unsigned SliceSize);
/// Checks if the gathered \p VL can be represented as multi-register
/// shuffle(s) of previous tree entries.
/// \param TE Tree entry checked for permutation.
/// \param VL List of scalars (a subset of the TE scalar), checked for
/// permutations.
/// \param ForOrder Tries to fetch the best candidates for ordering info. Also
/// commands to build the mask using the original vector value, without
/// relying on the potential reordering.
/// \returns per-register series of ShuffleKind, if gathered values can be
/// represented as shuffles of previous tree entries. \p Mask is filled with
/// the shuffle mask (also on per-register base).
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>>
isGatherShuffledEntry(
const TreeEntry *TE, ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
SmallVectorImpl<SmallVector<const TreeEntry *>> &Entries,
unsigned NumParts, Type *ScalarTy, bool ForOrder = false);
/// \returns the cost of gathering (inserting) the values in \p VL into a
/// vector.
/// \param ForPoisonSrc true if initial vector is poison, false otherwise.
InstructionCost getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
Type *ScalarTy) const;
/// Set the Builder insert point to one after the last instruction in
/// the bundle
void setInsertPointAfterBundle(const TreeEntry *E);
/// \returns a vector from a collection of scalars in \p VL. if \p Root is not
/// specified, the starting vector value is poison.
Value *
gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy,
function_ref<Value *(Value *, Value *, ArrayRef<int>)> CreateShuffle);
/// \returns whether the VectorizableTree is fully vectorizable and will
/// be beneficial even the tree height is tiny.
bool isFullyVectorizableTinyTree(bool ForReduction) const;
/// Run through the list of all gathered loads in the graph and try to find
/// vector loads/masked gathers instead of regular gathers. Later these loads
/// are reshufled to build final gathered nodes.
void tryToVectorizeGatheredLoads(
const SmallMapVector<
std::tuple<BasicBlock *, Value *, Type *>,
SmallVector<SmallVector<std::pair<LoadInst *, int64_t>>>, 8>
&GatheredLoads);
/// Run through the gather nodes that are splats of the same instruction and
/// try to vectorize the unique splatted values together as a separate
/// subtree. The splat gathers are then emitted as broadcasts of the
/// vectorized subtree instead of insertion sequences.
void tryToVectorizeSplatGatheredScalars();
/// Helper for `findExternalStoreUsersReorderIndices()`. It iterates over the
/// users of \p TE and collects the stores. It returns the map from the store
/// pointers to the collected stores.
SmallVector<SmallVector<StoreInst *>>
collectUserStores(const BoUpSLP::TreeEntry *TE) const;
/// Helper for `findExternalStoreUsersReorderIndices()`. It checks if the
/// stores in \p StoresVec can form a vector instruction. If so it returns
/// true and populates \p ReorderIndices with the shuffle indices of the
/// stores when compared to the sorted vector.
bool canFormVector(ArrayRef<StoreInst *> StoresVec,
OrdersType &ReorderIndices) const;
/// Iterates through the users of \p TE, looking for scalar stores that can be
/// potentially vectorized in a future SLP-tree. If found, it keeps track of
/// their order and builds an order index vector for each store bundle. It
/// returns all these order vectors found.
/// We run this after the tree has formed, otherwise we may come across user
/// instructions that are not yet in the tree.
SmallVector<OrdersType, 1>
findExternalStoreUsersReorderIndices(TreeEntry *TE) const;
/// Tries to reorder the gathering node for better vectorization
/// opportunities.
void reorderGatherNode(TreeEntry &TE);
/// Checks if the tree represents disjoint or reduction of shl(zext, (0, 8,
/// .., 56))-like pattern.
/// If the int shifts unique, also strided, but not ordered, sets \p Order.
/// If the node can be represented as a bitcast + bswap, sets \p IsBSwap.
/// If the root nodes are loads, sets \p ForLoads to true.
bool matchesShlZExt(const TreeEntry &TE, OrdersType &Order, bool &IsBSwap,
bool &ForLoads) const;
/// Computes the packing layout of an or reduction of masked shifted values
/// packing per-lane byte fields of the scalar result (and(shl(x, s), m) per
/// lane). The field analysis proves the disjointness of the fields, so the
/// reduction operations are not required to be marked disjoint.
std::optional<BitPackInfo> computeBitPackLayout(const TreeEntry &TE) const;
/// Returns the node packed for the shl operand \p LhsTE of a bitpack root.
/// If \p LhsTE is a (zext of) and(lshr/shl(x, C), mask) chain extracting
/// the same bits of every lane of x, returns the node of x and sets the
/// shift amounts of \p Info to the field offset in the lanes of x.
/// Otherwise, returns \p LhsTE.
const TreeEntry *getBitPackSource(const TreeEntry &LhsTE,
BitPackInfo &Info) const;
/// Checks if the tree entry is the root of such a reduction, the tree nodes
/// are in the expected state and it is profitable to model as a bit pack.
bool matchesBitPack(const TreeEntry &TE) const;
/// If the node's lanes are a plain zext, returns the source bit width:
/// compacting the vectorized lanes back to it is free. Copyable lanes force
/// a blend after the extend, so the vectorized value is not a plain zext.
unsigned getZExtSrcWidth(const TreeEntry &TE) const;
/// Checks if the \p SelectTE matches zext+selects, which can be inversed for
/// better codegen in case like zext (icmp ne), select (icmp eq), ....
bool matchesInversedZExtSelect(
const TreeEntry &SelectTE,
SmallVectorImpl<unsigned> &InversedCmpsIndices) const;
/// Checks if the tree is reduction or of bit selects, like select %cmp, <1,
/// 2, 4, 8, ..>, zeroinitializer, which can be reduced just to a bitcast %cmp
/// to in.
bool matchesSelectOfBits(const TreeEntry &SelectTE) const;
class TreeEntry {
public:
using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
TreeEntry(VecTreeTy &Container) : Container(Container) {}
/// \returns Common mask for reorder indices and reused scalars.
SmallVector<int> getCommonMask() const {
if (State == TreeEntry::SplitVectorize)
return {};
SmallVector<int> Mask;
inversePermutation(ReorderIndices, Mask);
addMask(Mask, ReuseShuffleIndices);
return Mask;
}
/// \returns The mask for split nodes.
SmallVector<int> getSplitMask() const {
assert(State == TreeEntry::SplitVectorize && !ReorderIndices.empty() &&
"Expected only split vectorize node.");
unsigned CommonVF = std::max<unsigned>(
CombinedEntriesWithIndices.back().second,
Scalars.size() - CombinedEntriesWithIndices.back().second);
const unsigned Scale = getNumElements(Scalars.front()->getType());
CommonVF *= Scale;
SmallVector<int> Mask(getVectorFactor() * Scale, PoisonMaskElem);
for (auto [Idx, I] : enumerate(ReorderIndices)) {
for (unsigned K : seq<unsigned>(Scale)) {
Mask[Scale * I + K] =
Scale * Idx + K +
(Idx >= CombinedEntriesWithIndices.back().second
? CommonVF - CombinedEntriesWithIndices.back().second * Scale
: 0);
}
}
return Mask;
}
/// Updates (reorders) SplitVectorize node according to the given mask \p
/// Mask and order \p MaskOrder.
void reorderSplitNode(unsigned Idx, ArrayRef<int> Mask,
ArrayRef<int> MaskOrder);
/// \returns true if the scalars in VL are equal to this entry.
bool isSame(ArrayRef<Value *> VL) const {
auto &&IsSame = [VL](ArrayRef<Value *> Scalars, ArrayRef<int> Mask) {
if (Mask.size() != VL.size() && VL.size() == Scalars.size())
return std::equal(VL.begin(), VL.end(), Scalars.begin());
return VL.size() == Mask.size() &&
std::equal(VL.begin(), VL.end(), Mask.begin(),
[Scalars](Value *V, int Idx) {
return isa<PoisonValue>(V) ||
(Idx != PoisonMaskElem && V == Scalars[Idx]);
});
};
if (!ReorderIndices.empty()) {
// TODO: implement matching if the nodes are just reordered, still can
// treat the vector as the same if the list of scalars matches VL
// directly, without reordering.
SmallVector<int> Mask;
inversePermutation(ReorderIndices, Mask);
if (VL.size() == Scalars.size())
return IsSame(Scalars, Mask);
if (VL.size() == ReuseShuffleIndices.size()) {
addMask(Mask, ReuseShuffleIndices);
return IsSame(Scalars, Mask);
}
return false;
}
return IsSame(Scalars, ReuseShuffleIndices);
}
/// \returns true if current entry has same operands as \p TE.
bool hasEqualOperands(const TreeEntry &TE) const {
if (TE.getNumOperands() != getNumOperands())
return false;
SmallBitVector Used(getNumOperands());
for (unsigned I = 0, E = getNumOperands(); I < E; ++I) {
unsigned PrevCount = Used.count();
for (unsigned K = 0; K < E; ++K) {
if (Used.test(K))
continue;
if (getOperand(K) == TE.getOperand(I)) {
Used.set(K);
break;
}
}
// Check if we actually found the matching operand.
if (PrevCount == Used.count())
return false;
}
return true;
}
/// \return Final vectorization factor for the node. Defined by the total
/// number of vectorized scalars, including those, used several times in the
/// entry and counted in the \a ReuseShuffleIndices, if any.
unsigned getVectorFactor() const {
if (!ReuseShuffleIndices.empty())
return ReuseShuffleIndices.size();
return Scalars.size();
};
/// Checks if the current node is a gather node.
bool isGather() const { return State == NeedToGather; }
/// Checks if the current node is in reverse order.
bool isReverse() const {
return !ReorderIndices.empty() && isReverseOrder(ReorderIndices);
}
/// A vector of scalars.
ValueList Scalars;
/// The Scalars are vectorized into this value. It is initialized to Null.
WeakTrackingVH VectorizedValue = nullptr;
/// Do we need to gather this sequence or vectorize it
/// (either with vector instruction or with scatter/gather
/// intrinsics for store/load)?
enum EntryState {
Vectorize, ///< The node is regularly vectorized.
ScatterVectorize, ///< Masked scatter/gather node.
StridedVectorize, ///< Strided loads (and stores)
ExpandVectorize, ///< Masked stores, the values are expanded into
///< a wider vector and vectorized with a mask.
CompressVectorize, ///< (Masked) load with compress.
BlendedLoadVectorize, ///< (Masked) loads blended via `select` from two
///< candidate base pointers.
NeedToGather, ///< Gather/buildvector node.
CombinedVectorize, ///< Vectorized node, combined with its user into more
///< complex node like select/cmp to minmax, mul/add to
///< fma, etc. Must be used for the following nodes in
///< the pattern, not the very first one.
SplitVectorize, ///< Splits the node into 2 subnodes, vectorizes them
///< independently and then combines back.
};
EntryState State;
/// List of combined opcodes supported by the vectorizer.
enum CombinedOpcode {
NotCombinedOp = -1,
MinMax = Instruction::OtherOpsEnd + 1,
FMulAdd,
ReducedBitcast,
ReducedBitcastBSwap,
ReducedBitcastLoads,
ReducedBitcastBSwapLoads,
ReducedBitPack,
ReducedCmpBitcast,
};
CombinedOpcode CombinedOp = NotCombinedOp;
/// Does this sequence require some shuffling?
SmallVector<int, 4> ReuseShuffleIndices;
/// Does this entry require reordering?
SmallVector<unsigned, 4> ReorderIndices;
/// Points back to the VectorizableTree.
///
/// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
/// to be a pointer and needs to be able to initialize the child iterator.
/// Thus we need a reference back to the container to translate the indices
/// to entries.
VecTreeTy &Container;
/// The TreeEntry index containing the user of this entry.
EdgeInfo UserTreeIndex;
/// The index of this treeEntry in VectorizableTree.
unsigned Idx = 0;
/// For gather/buildvector/alt opcode nodes, which are combined from
/// other nodes as a series of insertvector instructions.
SmallVector<std::pair<unsigned, unsigned>, 2> CombinedEntriesWithIndices;
/// For ExtractValue entries that are vectorized via the struct-call path
/// (checkEVsForVecCalls succeeded during tree building), stores the common
/// field-index path shared by all scalars in the bundle. Empty for all
/// other entry kinds.
SmallVector<unsigned, 1> StructEVIndices;
private:
/// The operands of each instruction in each lane Operands[op_index][lane].
/// Note: This helps avoid the replication of the code that performs the
/// reordering of operands during buildTreeRec() and vectorizeTree().
SmallVector<ValueList, 2> Operands;
/// Copyable elements of the entry node.
SmallPtrSet<const Value *, 4> CopyableElements;
/// Intermediate instructions peeled from an associative chain (e.g. the
/// inner add in add(add(v0,x),v1)). Not part of Scalars.
SmallVector<Value *, 4> ReassocScalars;
/// Sign of each flattened operand column of a reassociated add/sub
/// chain, parallel to the operand columns: a negated column is
/// subtracted from the positive total. Empty when no column is negated.
SmallBitVector ReassocNegatedOps;
/// MainOp and AltOp are recorded inside. S should be obtained from
/// newTreeEntry.
InstructionsState S = InstructionsState::invalid();
/// Interleaving factor for interleaved loads Vectorize nodes.
unsigned InterleaveFactor = 0;
/// True if the node does not require scheduling.
bool DoesNotNeedToSchedule = false;
/// Set this bundle's \p OpIdx'th operand to \p OpVL.
void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
if (Operands.size() < OpIdx + 1)
Operands.resize(OpIdx + 1);
assert(Operands[OpIdx].empty() && "Already resized?");
assert(OpVL.size() <= Scalars.size() &&
"Number of operands is greater than the number of scalars.");
Operands[OpIdx].resize(OpVL.size());
copy(OpVL, Operands[OpIdx].begin());
}
/// Maps values to their lanes in the node.
mutable SmallDenseMap<Value *, unsigned> ValueToLane;
public:
/// Returns interleave factor for interleave nodes.
unsigned getInterleaveFactor() const { return InterleaveFactor; }
/// Sets interleaving factor for the interleaving nodes.
void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
/// Marks the node as one that does not require scheduling.
void setDoesNotNeedToSchedule() { DoesNotNeedToSchedule = true; }
/// Returns true if the node is marked as one that does not require
/// scheduling.
bool doesNotNeedToSchedule() const { return DoesNotNeedToSchedule; }
/// Set this bundle's operands from \p Operands.
void setOperands(ArrayRef<ValueList> Operands) {
for (unsigned I : seq<unsigned>(Operands.size()))
setOperand(I, Operands[I]);
}
/// Reorders operands of the node to the given mask \p Mask.
void reorderOperands(ArrayRef<int> Mask) {
for (ValueList &Operand : Operands)
reorderScalars(Operand, Mask);
}
/// \returns the \p OpIdx operand of this TreeEntry.
ValueList &getOperand(unsigned OpIdx) {
assert(OpIdx < Operands.size() && "Off bounds");
return Operands[OpIdx];
}
/// \returns the \p OpIdx operand of this TreeEntry.
ArrayRef<Value *> getOperand(unsigned OpIdx) const {
assert(OpIdx < Operands.size() && "Off bounds");
return Operands[OpIdx];
}
/// \returns the number of operands.
unsigned getNumOperands() const { return Operands.size(); }
/// \return the single \p OpIdx operand.
Value *getSingleOperand(unsigned OpIdx) const {
assert(OpIdx < Operands.size() && "Off bounds");
assert(!Operands[OpIdx].empty() && "No operand available");
return Operands[OpIdx][0];
}
/// Some of the instructions in the list have alternate opcodes.
bool isAltShuffle() const { return S.isAltShuffle(); }
Instruction *getMatchingMainOpOrAltOp(Instruction *I) const {
return S.getMatchingMainOpOrAltOp(I);
}
void setOperations(const InstructionsState &S) {
assert(S && "InstructionsState is invalid.");
this->S = S;
}
Instruction *getMainOp() const { return S.getMainOp(); }
Instruction *getAltOp() const { return S.getAltOp(); }
/// The main/alternate opcodes for the list of instructions.
unsigned getOpcode() const { return S.getOpcode(); }
unsigned getAltOpcode() const { return S.getAltOpcode(); }
bool hasState() const { return S.valid(); }
/// Add \p V to the list of copyable elements.
void addCopyableElement(Value *V) {
assert(S.isCopyableElement(V) && "Not a copyable element.");
CopyableElements.insert(V);
}
/// Returns true if \p V is a copyable element.
bool isCopyableElement(Value *V) const {
return CopyableElements.contains(V);
}
/// Checks if the value \p V is a transformed instruction, compatible either
/// with main or alternate ops.
bool isExpandedBinOp(Value *V) const {
assert(hasState() && "InstructionsState is invalid.");
if (isCopyableElement(V))
return false;
return S.isExpandedBinOp(V);
}
/// Checks if the operand at index \p Idx of instruction \p I is an expanded
/// operand.
bool isExpandedOperand(Instruction *I, unsigned Idx) const {
assert(hasState() && "InstructionsState is invalid.");
if (isCopyableElement(I))
return false;
if (!isExpandedBinOp(I))
return false;
return S.isExpandedOperand(I, Idx);
}
/// Returns true if any scalar in the list is a copyable element.
bool hasCopyableElements() const { return !CopyableElements.empty(); }
/// Adds \p V to the peeled reassociated scalars.
void addReassocScalar(Value *V) { ReassocScalars.push_back(V); }
/// True if operands were gathered from an associative chain.
bool hasReassocScalars() const { return !ReassocScalars.empty(); }
/// Returns peeled reassociated scalars.
ArrayRef<Value *> getReassocScalars() const { return ReassocScalars; }
/// Records the signs of the flattened operand columns.
void setReassocNegatedOps(const SmallBitVector &NegatedOps) {
assert(NegatedOps.size() == getNumOperands() &&
"Signs must cover all operand columns.");
ReassocNegatedOps = NegatedOps;
}
/// True if operand column \p Idx is subtracted rather than added.
bool isReassocNegatedOp(unsigned Idx) const {
return Idx < ReassocNegatedOps.size() && ReassocNegatedOps[Idx];
}
/// Returns the state of the operations.
const InstructionsState &getOperations() const { return S; }
/// When ReuseReorderShuffleIndices is empty it just returns position of \p
/// V within vector of Scalars. Otherwise, try to remap on its reuse index.
unsigned findLaneForValue(Value *V) const {
auto Res = ValueToLane.try_emplace(V, getVectorFactor());
if (!Res.second)
return Res.first->second;
unsigned &FoundLane = Res.first->getSecond();
// Poison can take any lane, match it to the lane of the first non-poison
// scalar.
auto IsMatch = [V](Value *S) {
return isa<PoisonValue>(V) ? !isa<PoisonValue>(S) : S == V;
};
for (auto *It = find_if(Scalars, IsMatch), *End = Scalars.end();
It != End; std::advance(It, 1)) {
if (!IsMatch(*It))
continue;
FoundLane = std::distance(Scalars.begin(), It);
assert(FoundLane < Scalars.size() && "Couldn't find extract lane");
if (!ReorderIndices.empty())
FoundLane = ReorderIndices[FoundLane];
assert(FoundLane < Scalars.size() && "Couldn't find extract lane");
if (ReuseShuffleIndices.empty())
break;
if (auto *RIt = find(ReuseShuffleIndices, FoundLane);
RIt != ReuseShuffleIndices.end()) {
FoundLane = std::distance(ReuseShuffleIndices.begin(), RIt);
break;
}
}
assert(FoundLane < getVectorFactor() && "Unable to find given value.");
return FoundLane;
}
/// Build a shuffle mask for graph entry which represents a merge of main
/// and alternate operations.
void
buildAltOpShuffleMask(const function_ref<bool(Instruction *)> IsAltOp,
SmallVectorImpl<int> &Mask,
SmallVectorImpl<Value *> *OpScalars = nullptr,
SmallVectorImpl<Value *> *AltScalars = nullptr) const;
/// Return true if this is a non-power-of-2 node.
bool isNonPowOf2Vec() const {
bool IsNonPowerOf2 = !has_single_bit(Scalars.size());
return IsNonPowerOf2;
}
Value *getOrdered(unsigned Idx) const {
if (ReorderIndices.empty())
return Scalars[Idx];
SmallVector<int> Mask;
inversePermutation(ReorderIndices, Mask);
return Scalars[Mask[Idx]];
}
#ifndef NDEBUG
/// Debug printer.
LLVM_DUMP_METHOD void dump() const {
dbgs() << Idx << ".\n";
for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) {
dbgs() << "Operand " << OpI << ":\n";
for (const Value *V : Operands[OpI])
dbgs().indent(2) << *V << "\n";
}
dbgs() << "Scalars: \n";
for (Value *V : Scalars) {
dbgs().indent(2) << *V
<< ((S && S.isExpandedBinOp(V)) ? " [[Expanded]]\n"
: "\n");
}
dbgs() << "State: ";
if (S && hasCopyableElements())
dbgs() << "[[Copyable]] ";
switch (State) {
case Vectorize:
if (InterleaveFactor > 0) {
dbgs() << "Vectorize with interleave factor " << InterleaveFactor
<< "\n";
} else {
dbgs() << "Vectorize\n";
}
break;
case ScatterVectorize:
dbgs() << "ScatterVectorize\n";
break;
case StridedVectorize:
dbgs() << "StridedVectorize\n";
break;
case ExpandVectorize:
dbgs() << "ExpandVectorize\n";
break;
case CompressVectorize:
dbgs() << "CompressVectorize\n";
break;
case BlendedLoadVectorize:
dbgs() << "BlendedLoadVectorize\n";
break;
case NeedToGather:
dbgs() << "NeedToGather\n";
break;
case CombinedVectorize:
dbgs() << "CombinedVectorize\n";
break;
case SplitVectorize:
dbgs() << "SplitVectorize\n";
break;
}
if (S) {
dbgs() << "MainOp: " << *S.getMainOp() << "\n";
dbgs() << "AltOp: " << *S.getAltOp() << "\n";
} else {
dbgs() << "MainOp: NULL\n";
dbgs() << "AltOp: NULL\n";
}
dbgs() << "VectorizedValue: ";
if (VectorizedValue)
dbgs() << *VectorizedValue << "\n";
else
dbgs() << "NULL\n";
dbgs() << "ReuseShuffleIndices: ";
if (ReuseShuffleIndices.empty())
dbgs() << "Empty";
else
for (int ReuseIdx : ReuseShuffleIndices)
dbgs() << ReuseIdx << ", ";
dbgs() << "\n";
dbgs() << "ReorderIndices: ";
for (unsigned ReorderIdx : ReorderIndices)
dbgs() << ReorderIdx << ", ";
dbgs() << "\n";
dbgs() << "UserTreeIndex: ";
if (UserTreeIndex)
dbgs() << UserTreeIndex;
else
dbgs() << "<invalid>";
dbgs() << "\n";
if (!StructEVIndices.empty()) {
dbgs() << "StructEVIndices: ";
interleaveComma(StructEVIndices, dbgs());
dbgs() << "\n";
}
if (!CombinedEntriesWithIndices.empty()) {
dbgs() << "Combined entries: ";
interleaveComma(CombinedEntriesWithIndices, dbgs(), [&](const auto &P) {
dbgs() << "Entry index " << P.first << " with offset " << P.second;
});
dbgs() << "\n";
}
}
#endif
};
#ifndef NDEBUG
void dumpTreeCosts(const TreeEntry *E, InstructionCost ReuseShuffleCost,
InstructionCost VecCost, InstructionCost ScalarCost,
StringRef Banner) const {
dbgs() << "SLP: " << Banner << ":\n";
E->dump();
dbgs() << "SLP: Costs:\n";
dbgs() << "SLP: ReuseShuffleCost = " << ReuseShuffleCost << "\n";
dbgs() << "SLP: VectorCost = " << VecCost << "\n";
dbgs() << "SLP: ScalarCost = " << ScalarCost << "\n";
dbgs() << "SLP: ReuseShuffleCost + VecCost - ScalarCost = "
<< ReuseShuffleCost + VecCost - ScalarCost << "\n";
}
#endif
/// Create a new gather TreeEntry
TreeEntry *newGatherTreeEntry(ArrayRef<Value *> VL,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {}) {
auto Invalid = ScheduleBundle::invalid();
return newTreeEntry(VL, Invalid, S, UserTreeIdx, ReuseShuffleIndices);
}
/// Create a new VectorizableTree entry.
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {},
unsigned InterleaveFactor = 0) {
TreeEntry::EntryState EntryState =
Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
TreeEntry *E = newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
ReuseShuffleIndices, ReorderIndices);
if (E && InterleaveFactor > 0)
E->setInterleave(InterleaveFactor);
return E;
}
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
TreeEntry::EntryState EntryState,
ScheduleBundle &Bundle, const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
ArrayRef<unsigned> ReorderIndices = {}) {
assert(((!Bundle && (EntryState == TreeEntry::NeedToGather ||
EntryState == TreeEntry::SplitVectorize)) ||
(Bundle && EntryState != TreeEntry::NeedToGather &&
EntryState != TreeEntry::SplitVectorize)) &&
"Need to vectorize gather entry?");
// Gathered loads still gathered? Do not create entry, use the original one.
if (GatheredLoadsEntriesFirst.has_value() &&
EntryState == TreeEntry::NeedToGather && S &&
S.getOpcode() == Instruction::Load && UserTreeIdx.EdgeIdx == UINT_MAX &&
!UserTreeIdx.UserTE)
return nullptr;
VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree));
TreeEntry *Last = VectorizableTree.back().get();
Last->Idx = VectorizableTree.size() - 1;
Last->State = EntryState;
if (UserTreeIdx.UserTE)
OperandsToTreeEntry.try_emplace(
std::make_pair(UserTreeIdx.UserTE, UserTreeIdx.EdgeIdx), Last);
Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
ReuseShuffleIndices.end());
if (ReorderIndices.empty()) {
Last->Scalars.assign(VL.begin(), VL.end());
if (S)
Last->setOperations(S);
} else {
// Reorder scalars and build final mask.
Last->Scalars.assign(VL.size(), nullptr);
transform(ReorderIndices, Last->Scalars.begin(),
[VL](unsigned Idx) -> Value * {
if (Idx >= VL.size())
return UndefValue::get(VL.front()->getType());
return VL[Idx];
});
InstructionsState S = getSameOpcode(Last->Scalars, *TLI);
if (S)
Last->setOperations(S);
Last->ReorderIndices.append(ReorderIndices.begin(), ReorderIndices.end());
}
if (EntryState == TreeEntry::SplitVectorize) {
assert(S && "Split nodes must have operations.");
Last->setOperations(S);
SmallPtrSet<Value *, 4> Processed;
for (Instruction *I : make_isa_range<Instruction>(VL)) {
auto It = ScalarsInSplitNodes.find(I);
if (It == ScalarsInSplitNodes.end()) {
ScalarsInSplitNodes.try_emplace(I).first->getSecond().push_back(Last);
(void)Processed.insert(I);
} else if (Processed.insert(I).second) {
assert(!is_contained(It->getSecond(), Last) &&
"Value already associated with the node.");
It->getSecond().push_back(Last);
}
}
} else if (!Last->isGather()) {
if (isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()) ||
(!S.areInstructionsWithCopyableElements() &&
doesNotNeedToSchedule(VL)) ||
all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); }))
Last->setDoesNotNeedToSchedule();
SmallPtrSet<Value *, 4> Processed;
for (Value *V : make_filter_range(
VL, [](Value *V) { return !isa<PoisonValue>(V); })) {
if (S.isCopyableElement(V)) {
Last->addCopyableElement(V);
continue;
}
auto It = ScalarToTreeEntries.find(V);
if (It == ScalarToTreeEntries.end()) {
ScalarToTreeEntries.try_emplace(V).first->getSecond().push_back(Last);
(void)Processed.insert(V);
} else if (Processed.insert(V).second) {
assert(!is_contained(It->getSecond(), Last) &&
"Value already associated with the node.");
It->getSecond().push_back(Last);
}
}
// Update the scheduler bundle to point to this TreeEntry.
assert((!Bundle.getBundle().empty() || Last->doesNotNeedToSchedule()) &&
"Bundle and VL out of sync");
if (!Bundle.getBundle().empty()) {
#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
auto *BundleMember = Bundle.getBundle().begin();
SmallPtrSet<Value *, 4> Processed;
for (Value *V : VL) {
if (S.isNonSchedulable(V) || !Processed.insert(V).second)
continue;
++BundleMember;
}
assert(BundleMember == Bundle.getBundle().end() &&
"Bundle and VL out of sync");
#endif
Bundle.setTreeEntry(Last);
}
} else {
// Build a map for gathered scalars to the nodes where they are used.
bool AllConstsOrCasts = true;
for (Value *V : VL) {
if (S && S.areInstructionsWithCopyableElements() &&
S.isCopyableElement(V))
Last->addCopyableElement(V);
if (!isConstant(V)) {
auto *I = dyn_cast<CastInst>(V);
AllConstsOrCasts &= I && I->getType()->isIntegerTy();
if (UserTreeIdx.EdgeIdx != UINT_MAX || !UserTreeIdx.UserTE ||
!UserTreeIdx.UserTE->isGather())
ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last);
}
}
if (AllConstsOrCasts)
CastMaxMinBWSizes =
std::make_pair(std::numeric_limits<unsigned>::max(), 1);
MustGather.insert_range(VL);
}
if (UserTreeIdx.UserTE)
Last->UserTreeIndex = UserTreeIdx;
return Last;
}
/// -- Vectorization State --
/// Holds all of the tree entries.
TreeEntry::VecTreeTy VectorizableTree;
#ifndef NDEBUG
/// Debug printer.
LLVM_DUMP_METHOD void dumpVectorizableTree() const {
for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
VectorizableTree[Id]->dump();
if (TransformedToGatherNodes.contains(VectorizableTree[Id].get()))
dbgs() << "[[TRANSFORMED TO GATHER]]";
else if (DeletedNodes.contains(VectorizableTree[Id].get()))
dbgs() << "[[DELETED NODE]]";
dbgs() << "\n";
}
}
#endif
/// Get list of vector entries, associated with the value \p V.
ArrayRef<TreeEntry *> getTreeEntries(const Value *V) const {
assert(V && "V cannot be nullptr.");
auto It = ScalarToTreeEntries.find(V);
if (It == ScalarToTreeEntries.end())
return {};
return It->getSecond();
}
/// Get list of split vector entries, associated with the value \p V.
ArrayRef<TreeEntry *> getSplitTreeEntries(Value *V) const {
assert(V && "V cannot be nullptr.");
auto It = ScalarsInSplitNodes.find(V);
if (It == ScalarsInSplitNodes.end())
return {};
return It->getSecond();
}
/// Returns first vector node for value \p V, matching values \p VL.
TreeEntry *getSameValuesTreeEntry(Value *V, ArrayRef<Value *> VL,
bool SameVF = false) const {
assert(V && "V cannot be nullptr.");
for (TreeEntry *TE : ScalarToTreeEntries.lookup(V))
if ((!SameVF || TE->getVectorFactor() == VL.size()) && TE->isSame(VL))
return TE;
return nullptr;
}
/// Contains all the outputs of legality analysis for a list of values to
/// vectorize.
class ScalarsVectorizationLegality {
InstructionsState S;
bool IsLegal;
bool TryToFindDuplicates;
bool TrySplitVectorize;
public:
ScalarsVectorizationLegality(InstructionsState S, bool IsLegal,
bool TryToFindDuplicates = true,
bool TrySplitVectorize = false)
: S(S), IsLegal(IsLegal), TryToFindDuplicates(TryToFindDuplicates),
TrySplitVectorize(TrySplitVectorize) {
assert((!IsLegal || (S.valid() && TryToFindDuplicates)) &&
"Inconsistent state");
}
const InstructionsState &getInstructionsState() const { return S; };
bool isLegal() const { return IsLegal; }
bool tryToFindDuplicates() const { return TryToFindDuplicates; }
bool trySplitVectorize() const { return TrySplitVectorize; }
};
/// Checks if the specified list of the instructions/values can be vectorized
/// in general.
ScalarsVectorizationLegality
getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx) const;
/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
TreeEntry::EntryState getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo,
SmallVectorImpl<int> &ReuseShuffleIndices);
/// Maps a specific scalar to its tree entry(ies).
SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarToTreeEntries;
/// List of deleted non-profitable nodes.
SmallPtrSet<const TreeEntry *, 8> DeletedNodes;
/// List of nodes, transformed to gathered, with their conservative
/// gather/buildvector cost estimation.
SmallDenseMap<const TreeEntry *, InstructionCost> TransformedToGatherNodes;
/// Maps the operand index and entry to the corresponding tree entry.
SmallDenseMap<std::pair<const TreeEntry *, unsigned>, TreeEntry *>
OperandsToTreeEntry;
/// Scalars, used in split vectorize nodes.
SmallDenseMap<Value *, SmallVector<TreeEntry *>> ScalarsInSplitNodes;
/// Maps a value to the proposed vectorizable size.
SmallDenseMap<Value *, unsigned> InstrElementSize;
/// A list of scalars that we found that we need to keep as scalars.
ValueSet MustGather;
/// Maps each peeled reassociated scalar to owning entries. Keeps them
/// treated as vectorized while an owner is live.
SmallDenseMap<const Value *, SmallVector<const TreeEntry *>>
ReassocScalarToTreeEntries;
/// Peeled reassociated scalars that must survive erasure: claimed by a
/// gather node, listed in some tree entry's scalars, or feeding another
/// kept scalar.
SmallPtrSet<const Value *, 8> KeptReassocScalars;
/// A set of first non-schedulable values.
ValueSet NonScheduledFirst;
/// A map between the vectorized entries and the last instructions in the
/// bundles. The bundles are built in use order, not in the def order of the
/// instructions. So, we cannot rely directly on the last instruction in the
/// bundle being the last instruction in the program order during
/// vectorization process since the basic blocks are affected, need to
/// pre-gather them before.
SmallDenseMap<const TreeEntry *, WeakTrackingVH> EntryToLastInstruction;
/// Keeps the mapping between the last instructions and their insertion
/// points, which is an instruction-after-the-last-instruction.
SmallDenseMap<const Instruction *, Instruction *> LastInstructionToPos;
/// List of gather nodes, depending on other gather/vector nodes, which should
/// be emitted after the vector instruction emission process to correctly
/// handle order of the vector instructions and shuffles.
SetVector<const TreeEntry *> PostponedGathers;
using ValueToGatherNodesMap =
DenseMap<Value *, SmallSetVector<const TreeEntry *, 4>>;
ValueToGatherNodesMap ValueToGatherNodes;
SmallDenseMap<TreeEntry *, StridedPtrInfo> TreeEntryToStridedPtrInfoMap;
/// A list of the load entries (node indices), which can be vectorized using
/// strided or masked gather approach, but attempted to be represented as
/// contiguous loads.
SetVector<unsigned> LoadEntriesToVectorize;
/// true if graph nodes transforming mode is on.
bool IsGraphTransformMode = false;
/// The index of the first gathered load entry in the VectorizeTree.
std::optional<unsigned> GatheredLoadsEntriesFirst;
/// Root entries of the subtrees built for the splat gather nodes' unique
/// scalars. They have no users in the tree and must be emitted explicitly
/// before the root node.
SmallVector<TreeEntry *> SplatGatheredScalarsRoots;
/// Checks if any of \p Entries is an internal gather of a splat gather
/// subtree.
bool hasSplatSubtreeGather(
ArrayRef<SmallVector<const TreeEntry *>> Entries) const {
if (SplatGatheredScalarsRoots.empty())
return false;
return any_of(Entries, [&](ArrayRef<const TreeEntry *> TEs) {
return any_of(TEs, [&](const TreeEntry *TE) {
if (!TE->isGather())
return false;
while (TE->UserTreeIndex)
TE = TE->UserTreeIndex.UserTE;
return is_contained(SplatGatheredScalarsRoots, TE);
});
});
}
/// Number of tree entries added while building the splat gather subtrees.
/// The subtrees are auxiliary and must not inflate the tree size recorded
/// for failed store chain attempts.
unsigned NumCanonicalSplatSubtreeEntries = 0;
/// Maps compress entries to their mask data for the final codegen.
SmallDenseMap<const TreeEntry *,
std::tuple<SmallVector<int>, VectorType *, unsigned, bool>>
CompressEntryToData;
/// The loop nest, used to check if only a single loop nest is vectorized, not
/// multiple, to avoid side-effects from the loop-aware cost model.
SmallVector<const Loop *> CurrentLoopNest;
/// Per-depth SCEVs trip counts at every loop level where the tree builder has
/// joined diverging sibling loops.
SmallVector<const SCEV *> MergedLoopBTCs;
/// Maps the loops to their loop nests.
SmallDenseMap<const Loop *, SmallVector<const Loop *>> LoopToLoopNest;
/// Per-loop cache of nest scale factors: the product of trip counts of the
/// loop and all of its ancestors. Shared by getLoopNestScale() and (via it)
/// by getScaleToLoopIterations() and getGatherNodeEffectiveScale().
SmallDenseMap<const Loop *, uint64_t> LoopNestScaleCache;
/// This POD struct describes one external user in the vectorized tree.
struct ExternalUser {
ExternalUser(Value *S, llvm::User *U, const TreeEntry &E, unsigned L)
: Scalar(S), User(U), E(E), Lane(L) {}
/// Which scalar in our function.
Value *Scalar = nullptr;
/// Which user that uses the scalar.
llvm::User *User = nullptr;
/// Vector node, the value is part of.
const TreeEntry &E;
/// Which lane does the scalar belong to.
unsigned Lane;
};
using UserList = SmallVector<ExternalUser, 16>;
/// Checks if two instructions may access the same memory.
///
/// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
/// is invariant in the calling loop.
bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
Instruction *Inst2) {
assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instruction.");
// First check if the result is already in the cache.
AliasCacheKey Key = std::make_pair(Inst1, Inst2);
auto Res = AliasCache.try_emplace(Key);
if (!Res.second)
return Res.first->second;
bool Aliased = isModOrRefSet(BatchAA->getModRefInfo(Inst2, Loc1));
// Store the result in the cache.
Res.first->getSecond() = Aliased;
return Aliased;
}
/// Returns true if the may-alias dependency between simple load/store
/// instructions \p Inst1 and \p Inst2 could be disambiguated by a runtime
/// alias check.
bool isRuntimeCheckableAliasPair(Instruction *Inst1, Instruction *Inst2);
/// Records the (distinct base object) pair behind the may-alias dependency
/// of \p Inst1 and \p Inst2 as a runtime alias check guarding the region in
/// block \p BB. Returns true if the pair was recorded.
bool recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
Instruction *Inst2);
/// Emits the collected runtime alias checks and versions the affected block,
/// duplicating its body into a scalar fallback guarded by the checks.
void versionBlocksForRuntimeChecks();
/// Builds the i1 value that is true when any pair of checked base objects
/// overlaps at runtime. The base address bounds are materialized from their
/// SCEVs with \p Exp.
Value *emitRuntimeAliasCheck(IRBuilderBase &Builder, SCEVExpander &Exp);
/// Data to model and emit the runtime alias checks.
struct RuntimeAliasCheckInfo {
/// The block whose body is guarded by the checks. Exactly one block is
/// supported per attempt.
BasicBlock *BB = nullptr;
/// Pairs of base objects that must be proven disjoint.
SmallSetVector<std::pair<const Value *, const Value *>, 4> BasePairs;
/// Accessed address range [Low, High) for each involved base object.
SmallMapVector<const Value *, std::pair<const SCEV *, const SCEV *>, 4>
Bounds;
void clear() {
BB = nullptr;
BasePairs.clear();
Bounds.clear();
}
};
/// When true, scheduling drops may-alias memory dependencies between
/// distinct, range-checkable base objects and records them as runtime alias
/// checks instead.
bool TryRuntimeAliasChecks = false;
/// When true, the stored values of the root store node are split into
/// halves, which are vectorized as separate subtrees.
bool SplitStoredValues = false;
/// Runtime alias checks collected during the last optimistic buildTree().
RuntimeAliasCheckInfo RTChecks;
/// Base-object pairs already proven disjoint by the block's runtime alias
/// check.
SmallDenseMap<BasicBlock *,
SmallDenseSet<std::pair<const Value *, const Value *>, 4>, 2>
VersionedBlockCheckedPairs;
/// Scalar fallback blocks.
SmallPtrSet<BasicBlock *, 4> ScalarFallbackBlocks;
/// Blocks for which a runtime-checks versioning attempt was made
/// and did not produce a profitable versioning.
SmallPtrSet<BasicBlock *, 8> FailedRuntimeChecksBlocks;
/// Returns true if a may-alias dependency between the simple load/store
/// instructions \p Inst1 and \p Inst2 in block \p BB is already covered by a
/// runtime alias check emitted for \p BB by a previous versioning.
bool isCoveredByExistingVersionCheck(BasicBlock *BB, Instruction *Inst1,
Instruction *Inst2) const;
/// True, if a may-alias dependency between distinct, range-checkable base
/// objects is observed (whether or not it was dropped).
bool HasRuntimeCheckableBlockers = false;
/// True, if a kept may-alias dependency is not runtime-checkable (call or a
/// non-simple memaccess).
bool HasNonCheckableMemBlocker = false;
/// Runtime checks are validated and bounded the collected checks.
bool RTChecksFinalized = false;
/// Set when a block was versioned with runtime alias checks, which changes
/// the CFG. Used to drop CFG-analysis preservation for the run.
bool CFGChanged = false;
/// Guarded block body (non-PHI, non-alloca, non-terminator) in original
/// source order.
SmallVector<Instruction *> RTOrigBodyOrder;
using AliasCacheKey = std::pair<Instruction *, Instruction *>;
/// Cache for alias results.
/// TODO: consider moving this to the AliasAnalysis itself.
SmallDenseMap<AliasCacheKey, bool> AliasCache;
AAResults *AA;
// Cache for pointerMayBeCaptured calls inside AA. Vectorizing a tree can
// create new captures (e.g. a pointer inserted into a vector of pointers
// feeding a masked gather), so this is reset after each vectorized tree.
std::optional<BatchAAResults> BatchAA;
/// Temporary store for deleted instructions. Instructions will be deleted
/// eventually when the BoUpSLP is destructed. The deferral is required to
/// ensure that there are no incorrect collisions in the AliasCache, which
/// can happen if a new instruction is allocated at the same address as a
/// previously deleted instruction.
DenseSet<Instruction *> DeletedInstructions;
/// The cached analyses results, cleared after each successful
/// vectorizeTree().
mutable SLPAnalysisCache Cache;
/// Set of the instruction, being analyzed already for reductions.
SmallPtrSet<Instruction *, 16> AnalyzedReductionsRoots;
/// Set of hashes for the list of reduction values already being analyzed.
DenseSet<size_t> AnalyzedReductionVals;
/// Set of hashes for the bundles, rejected as non-vectorizable.
SmallDenseSet<size_t, 8> AnalyzedBundles;
/// Set of the values, which were a part of the analyzed vector nodes.
SmallPtrSet<const Value *, 32> AnalyzedScalars;
/// Cache of the number of parts for the types and the parts limit.
mutable SmallDenseMap<std::tuple<Type *, Type *, unsigned>, unsigned>
NumberOfPartsCache;
/// Cache of the number of regs for the types and the parts limit.
mutable SmallDenseMap<std::tuple<Type *, Type *, unsigned>, unsigned>
NumberOfRegsCache;
/// Values already analyzed for minimal bitwidth and found to be
/// non-profitable, mapped to the size of the tree used for the analysis.
SmallDenseMap<Value *, unsigned> AnalyzedMinBWVals;
/// A list of values that need to extracted out of the tree.
/// This list holds pairs of (Internal Scalar : External User). External User
/// can be nullptr, it means that this Internal Scalar will be used later,
/// after vectorization.
UserList ExternalUses;
/// A list of GEPs which can be reaplced by scalar GEPs instead of
/// extractelement instructions.
SmallPtrSet<Value *, 4> ExternalUsesAsOriginalScalar;
/// A list of scalar to be extracted without specific user necause of too many
/// uses.
SmallPtrSet<Value *, 4> ExternalUsesWithNonUsers;
/// Externally used values and replacements emitted for the external uses
/// without users, consumed after the tree vectorization; must not be
/// collected as dead operands of the erased scalars.
SmallPtrSet<Value *, 4> ExternalUseReplacements;
/// Values used only by @llvm.assume calls.
SmallPtrSet<const Value *, 32> EphValues;
/// Holds all of the instructions that we gathered, shuffle instructions and
/// extractelements.
SetVector<Instruction *> GatherShuffleExtractSeq;
/// A list of blocks that we are going to CSE.
DenseSet<BasicBlock *> CSEBlocks;
/// List of hashes of vector of loads, which are known to be non vectorizable.
DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
/// Represents a scheduling entity, either ScheduleData, ScheduleCopyableData
/// or ScheduleBundle. ScheduleData used to gather dependecies for a single
/// instructions, while ScheduleBundle represents a batch of instructions,
/// going to be groupped together. ScheduleCopyableData models extra user for
/// "copyable" instructions.
class ScheduleEntity {
friend class ScheduleBundle;
friend class ScheduleData;
friend class ScheduleCopyableData;
protected:
enum class Kind { ScheduleData, ScheduleBundle, ScheduleCopyableData };
Kind getKind() const { return K; }
ScheduleEntity(Kind K) : K(K) {}
private:
/// Used for getting a "good" final ordering of instructions.
int SchedulingPriority = 0;
/// True if this instruction (or bundle) is scheduled (or considered as
/// scheduled in the dry-run).
bool IsScheduled = false;
/// The kind of the ScheduleEntity.
const Kind K = Kind::ScheduleData;
public:
ScheduleEntity() = delete;
/// Gets/sets the scheduling priority.
void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
int getSchedulingPriority() const { return SchedulingPriority; }
bool isReady() const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->isReady();
if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
return CD->isReady();
return cast<ScheduleBundle>(this)->isReady();
}
/// Returns true if the dependency information has been calculated.
/// Note that depenendency validity can vary between instructions within
/// a single bundle.
bool hasValidDependencies() const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->hasValidDependencies();
if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
return CD->hasValidDependencies();
return cast<ScheduleBundle>(this)->hasValidDependencies();
}
/// Gets the number of unscheduled dependencies.
int getUnscheduledDeps() const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->getUnscheduledDeps();
if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
return CD->getUnscheduledDeps();
return cast<ScheduleBundle>(this)->unscheduledDepsInBundle();
}
/// Increments the number of unscheduled dependencies.
int incrementUnscheduledDeps(int Incr) {
if (auto *SD = dyn_cast<ScheduleData>(this))
return SD->incrementUnscheduledDeps(Incr);
return cast<ScheduleCopyableData>(this)->incrementUnscheduledDeps(Incr);
}
/// Gets the number of dependencies.
int getDependencies() const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->getDependencies();
return cast<ScheduleCopyableData>(this)->getDependencies();
}
/// Gets the instruction.
Instruction *getInst() const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->getInst();
return cast<ScheduleCopyableData>(this)->getInst();
}
/// Gets/sets if the bundle is scheduled.
bool isScheduled() const { return IsScheduled; }
void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
static bool classof(const ScheduleEntity *) { return true; }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void dump(raw_ostream &OS) const {
if (const auto *SD = dyn_cast<ScheduleData>(this))
return SD->dump(OS);
if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
return CD->dump(OS);
return cast<ScheduleBundle>(this)->dump(OS);
}
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
};
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
friend inline raw_ostream &operator<<(raw_ostream &OS,
const BoUpSLP::ScheduleEntity &SE) {
SE.dump(OS);
return OS;
}
#endif
/// Contains all scheduling relevant data for an instruction.
/// A ScheduleData either represents a single instruction or a member of an
/// instruction bundle (= a group of instructions which is combined into a
/// vector instruction).
class ScheduleData final : public ScheduleEntity {
public:
// The initial value for the dependency counters. It means that the
// dependencies are not calculated yet.
enum { InvalidDeps = -1 };
ScheduleData() : ScheduleEntity(Kind::ScheduleData) {}
static bool classof(const ScheduleEntity *Entity) {
return Entity->getKind() == Kind::ScheduleData;
}
void init(int BlockSchedulingRegionID, Instruction *I) {
NextLoadStore = nullptr;
IsScheduled = false;
SchedulingRegionID = BlockSchedulingRegionID;
clearDependencies();
Inst = I;
}
/// Verify basic self consistency properties
void verify() {
if (hasValidDependencies()) {
assert(UnscheduledDeps <= Dependencies && "invariant");
} else {
assert(UnscheduledDeps == Dependencies && "invariant");
}
if (IsScheduled) {
assert(hasValidDependencies() && UnscheduledDeps == 0 &&
"unexpected scheduled state");
}
}
/// Returns true if the dependency information has been calculated.
/// Note that depenendency validity can vary between instructions within
/// a single bundle.
bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
/// bundle.
int incrementUnscheduledDeps(int Incr) {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
assert(UnscheduledDeps >= 0 &&
"Expected valid number of unscheduled deps");
return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Clears all dependency information.
void clearDependencies() {
clearDirectDependencies();
MemoryDependencies.clear();
ControlDependencies.clear();
}
/// Clears all direct dependencies only, except for control and memory
/// dependencies.
/// Required for copyable elements to correctly handle control/memory deps
/// and avoid extra reclaculation of such deps.
void clearDirectDependencies() {
Dependencies = InvalidDeps;
resetUnscheduledDeps();
IsScheduled = false;
}
/// Gets the number of unscheduled dependencies.
int getUnscheduledDeps() const { return UnscheduledDeps; }
/// Gets the number of dependencies.
int getDependencies() const { return Dependencies; }
/// Initializes the number of dependencies.
void initDependencies() { Dependencies = 0; }
/// Increments the number of dependencies.
void incDependencies() { Dependencies++; }
/// Gets scheduling region ID.
int getSchedulingRegionID() const { return SchedulingRegionID; }
/// Gets the instruction.
Instruction *getInst() const { return Inst; }
/// Gets the list of memory dependencies.
ArrayRef<ScheduleData *> getMemoryDependencies() const {
return MemoryDependencies;
}
/// Adds a memory dependency.
void addMemoryDependency(ScheduleData *Dep) {
MemoryDependencies.push_back(Dep);
}
/// Gets the list of control dependencies.
ArrayRef<ScheduleData *> getControlDependencies() const {
return ControlDependencies;
}
/// Adds a control dependency.
void addControlDependency(ScheduleData *Dep) {
ControlDependencies.push_back(Dep);
}
/// Gets/sets the next load/store instruction in the block.
ScheduleData *getNextLoadStore() const { return NextLoadStore; }
void setNextLoadStore(ScheduleData *Next) { NextLoadStore = Next; }
void dump(raw_ostream &OS) const { OS << *Inst; }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
private:
Instruction *Inst = nullptr;
/// Single linked list of all memory instructions (e.g. load, store, call)
/// in the block - until the end of the scheduling region.
ScheduleData *NextLoadStore = nullptr;
/// The dependent memory instructions.
/// This list is derived on demand in calculateDependencies().
SmallVector<ScheduleData *> MemoryDependencies;
/// List of instructions which this instruction could be control dependent
/// on. Allowing such nodes to be scheduled below this one could introduce
/// a runtime fault which didn't exist in the original program.
/// ex: this is a load or udiv following a readonly call which inf loops
SmallVector<ScheduleData *> ControlDependencies;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
/// The number of dependencies. Constitutes of the number of users of the
/// instruction plus the number of dependent memory instructions (if any).
/// This value is calculated on demand.
/// If InvalidDeps, the number of dependencies is not calculated yet.
int Dependencies = InvalidDeps;
/// The number of dependencies minus the number of dependencies of scheduled
/// instructions. As soon as this is zero, the instruction/bundle gets ready
/// for scheduling.
/// Note that this is negative as long as Dependencies is not calculated.
int UnscheduledDeps = InvalidDeps;
};
#ifndef NDEBUG
friend inline raw_ostream &operator<<(raw_ostream &OS,
const BoUpSLP::ScheduleData &SD) {
SD.dump(OS);
return OS;
}
#endif
class ScheduleBundle final : public ScheduleEntity {
/// The schedule data for the instructions in the bundle.
SmallVector<ScheduleEntity *> Bundle;
/// True if this bundle is valid.
bool IsValid = true;
/// The TreeEntry that this instruction corresponds to.
TreeEntry *TE = nullptr;
ScheduleBundle(bool IsValid)
: ScheduleEntity(Kind::ScheduleBundle), IsValid(IsValid) {}
public:
ScheduleBundle() : ScheduleEntity(Kind::ScheduleBundle) {}
static bool classof(const ScheduleEntity *Entity) {
return Entity->getKind() == Kind::ScheduleBundle;
}
/// Verify basic self consistency properties
void verify() const {
for (const ScheduleEntity *SD : Bundle) {
if (SD->hasValidDependencies()) {
assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
"invariant");
} else {
assert(SD->getUnscheduledDeps() == SD->getDependencies() &&
"invariant");
}
if (isScheduled()) {
assert(SD->hasValidDependencies() && SD->getUnscheduledDeps() == 0 &&
"unexpected scheduled state");
}
}
}
/// Returns the number of unscheduled dependencies in the bundle.
int unscheduledDepsInBundle() const {
assert(*this && "bundle must not be empty");
int Sum = 0;
for (const ScheduleEntity *BundleMember : Bundle) {
if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
return ScheduleData::InvalidDeps;
Sum += BundleMember->getUnscheduledDeps();
}
return Sum;
}
/// Returns true if the dependency information has been calculated.
/// Note that depenendency validity can vary between instructions within
/// a single bundle.
bool hasValidDependencies() const {
return all_of(Bundle, [](const ScheduleEntity *SD) {
return SD->hasValidDependencies();
});
}
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
bool isReady() const {
assert(*this && "bundle must not be empty");
return unscheduledDepsInBundle() == 0 && !isScheduled();
}
/// Returns the bundle of scheduling data, associated with the current
/// instruction.
ArrayRef<ScheduleEntity *> getBundle() { return Bundle; }
ArrayRef<const ScheduleEntity *> getBundle() const { return Bundle; }
/// Adds an instruction to the bundle.
void add(ScheduleEntity *SD) { Bundle.push_back(SD); }
/// Gets/sets the associated tree entry.
void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
TreeEntry *getTreeEntry() const { return TE; }
static ScheduleBundle invalid() { return {false}; }
operator bool() const { return IsValid; }
#ifndef NDEBUG
void dump(raw_ostream &OS) const {
if (!*this) {
OS << "[]";
return;
}
OS << '[';
interleaveComma(Bundle, OS, [&](const ScheduleEntity *SD) {
if (isa<ScheduleCopyableData>(SD))
OS << "<Copyable>";
OS << *SD->getInst();
});
OS << ']';
}
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
#endif // NDEBUG
};
#ifndef NDEBUG
friend inline raw_ostream &operator<<(raw_ostream &OS,
const BoUpSLP::ScheduleBundle &Bundle) {
Bundle.dump(OS);
return OS;
}
#endif
/// Contains all scheduling relevant data for the copyable instruction.
/// It models the virtual instructions, supposed to replace the original
/// instructions. E.g., if instruction %0 = load is a part of the bundle [%0,
/// %1], where %1 = add, then the ScheduleCopyableData models virtual
/// instruction %virt = add %0, 0.
class ScheduleCopyableData final : public ScheduleEntity {
/// The source schedule data for the instruction.
Instruction *Inst = nullptr;
/// The edge information for the instruction.
const EdgeInfo EI;
/// This ScheduleData is in the current scheduling region if this matches
/// the current SchedulingRegionID of BlockScheduling.
int SchedulingRegionID = 0;
/// Bundle, this data is part of.
ScheduleBundle &Bundle;
public:
ScheduleCopyableData(int BlockSchedulingRegionID, Instruction *I,
const EdgeInfo &EI, ScheduleBundle &Bundle)
: ScheduleEntity(Kind::ScheduleCopyableData), Inst(I), EI(EI),
SchedulingRegionID(BlockSchedulingRegionID), Bundle(Bundle) {}
static bool classof(const ScheduleEntity *Entity) {
return Entity->getKind() == Kind::ScheduleCopyableData;
}
/// Returns true if the dependency information has been calculated.
/// Note that depenendency validity can vary between instructions within
/// a single bundle.
bool hasValidDependencies() const {
return Dependencies != ScheduleData::InvalidDeps;
}
/// Returns true if it is ready for scheduling, i.e. it has no more
/// unscheduled depending instructions/bundles.
bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
/// Modifies the number of unscheduled dependencies for this instruction,
/// and returns the number of remaining dependencies for the containing
/// bundle.
int incrementUnscheduledDeps(int Incr) {
assert(hasValidDependencies() &&
"increment of unscheduled deps would be meaningless");
UnscheduledDeps += Incr;
assert(UnscheduledDeps >= 0 && "invariant");
return UnscheduledDeps;
}
/// Sets the number of unscheduled dependencies to the number of
/// dependencies.
void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
/// Gets the number of unscheduled dependencies.
int getUnscheduledDeps() const { return UnscheduledDeps; }
/// Gets the number of dependencies.
int getDependencies() const { return Dependencies; }
/// Initializes the number of dependencies.
void initDependencies() { Dependencies = 0; }
/// Increments the number of dependencies.
void incDependencies() { Dependencies++; }
/// Gets scheduling region ID.
int getSchedulingRegionID() const { return SchedulingRegionID; }
/// Gets the instruction.
Instruction *getInst() const { return Inst; }
/// Clears all dependency information.
void clearDependencies() {
Dependencies = ScheduleData::InvalidDeps;
UnscheduledDeps = ScheduleData::InvalidDeps;
IsScheduled = false;
}
/// Gets the edge information.
const EdgeInfo &getEdgeInfo() const { return EI; }
/// Gets the bundle.
ScheduleBundle &getBundle() { return Bundle; }
const ScheduleBundle &getBundle() const { return Bundle; }
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void dump(raw_ostream &OS) const { OS << "[Copyable]" << *getInst(); }
LLVM_DUMP_METHOD void dump() const {
dump(dbgs());
dbgs() << '\n';
}
#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
private:
/// true, if it has valid dependency information. These nodes always have
/// only single dependency.
int Dependencies = ScheduleData::InvalidDeps;
/// The number of dependencies minus the number of dependencies of scheduled
/// instructions. As soon as this is zero, the instruction/bundle gets ready
/// for scheduling.
/// Note that this is negative as long as Dependencies is not calculated.
int UnscheduledDeps = ScheduleData::InvalidDeps;
};
#ifndef NDEBUG
friend inline raw_ostream &
operator<<(raw_ostream &OS, const BoUpSLP::ScheduleCopyableData &SD) {
SD.dump(OS);
return OS;
}
#endif
friend struct GraphTraits<BoUpSLP *>;
friend struct DOTGraphTraits<BoUpSLP *>;
friend class ::HorizontalReduction;
/// Contains all scheduling data for a basic block.
/// It does not schedules instructions, which are not memory read/write
/// instructions and their operands are either constants, or arguments, or
/// phis, or instructions from others blocks, or their users are phis or from
/// the other blocks. The resulting vector instructions can be placed at the
/// beginning of the basic block without scheduling (if operands does not need
/// to be scheduled) or at the end of the block (if users are outside of the
/// block). It allows to save some compile time and memory used by the
/// compiler.
/// ScheduleData is assigned for each instruction in between the boundaries of
/// the tree entry, even for those, which are not part of the graph. It is
/// required to correctly follow the dependencies between the instructions and
/// their correct scheduling. The ScheduleData is not allocated for the
/// instructions, which do not require scheduling, like phis, nodes with
/// extractelements/insertelements only or nodes with instructions, with
/// uses/operands outside of the block.
struct BlockScheduling {
BlockScheduling(BasicBlock *BB, int ScheduleRegionSizeLimit)
: BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize),
ScheduleRegionSizeLimit(ScheduleRegionSizeLimit) {}
void clear() {
ScheduledBundles.clear();
ScheduledBundlesList.clear();
ScheduleCopyableDataMap.clear();
ScheduleCopyableDataMapByInst.clear();
ScheduleCopyableDataMapByInstUser.clear();
ScheduleCopyableDataMapByUsers.clear();
ReadyInsts.clear();
RecalcCopyableOperandDeps.clear();
IgnoredMemDeps.clear();
ScheduleStart = nullptr;
ScheduleEnd = nullptr;
FirstLoadStoreInRegion = nullptr;
LastLoadStoreInRegion = nullptr;
RegionHasStackSave = false;
RegionHasMayThrow = false;
// Reduce the maximum schedule region size by the size of the
// previous scheduling run.
ScheduleRegionSizeLimit -= ScheduleRegionSize;
if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
ScheduleRegionSizeLimit = MinScheduleRegionSize;
ScheduleRegionSize = 0;
// Make a new scheduling region, i.e. all existing ScheduleData is not
// in the new region yet.
++SchedulingRegionID;
}
ScheduleData *getScheduleData(Instruction *I) {
if (!I)
return nullptr;
if (BB != I->getParent())
// Avoid lookup if can't possibly be in map.
return nullptr;
ScheduleData *SD = ScheduleDataMap.lookup(I);
if (SD && isInSchedulingRegion(*SD))
return SD;
return nullptr;
}
ScheduleData *getScheduleData(Value *V) {
return getScheduleData(dyn_cast<Instruction>(V));
}
/// Returns the ScheduleCopyableData for the given edge (user tree entry and
/// operand number) and value.
ScheduleCopyableData *getScheduleCopyableData(const EdgeInfo &EI,
const Value *V) const {
if (ScheduleCopyableDataMap.empty())
return nullptr;
auto It = ScheduleCopyableDataMap.find(std::make_pair(EI, V));
if (It == ScheduleCopyableDataMap.end())
return nullptr;
ScheduleCopyableData *SD = It->getSecond().get();
if (!isInSchedulingRegion(*SD))
return nullptr;
return SD;
}
/// Returns the ScheduleCopyableData for the given user \p User, operand
/// number and operand \p V.
SmallVector<ScheduleCopyableData *>
getScheduleCopyableData(const Value *User, unsigned OperandIdx,
const Value *V) {
if (ScheduleCopyableDataMapByInstUser.empty())
return {};
const auto It = ScheduleCopyableDataMapByInstUser.find(
std::make_pair(std::make_pair(User, OperandIdx), V));
if (It == ScheduleCopyableDataMapByInstUser.end())
return {};
SmallVector<ScheduleCopyableData *> Res;
for (ScheduleCopyableData *SD : It->getSecond()) {
if (isInSchedulingRegion(*SD))
Res.push_back(SD);
}
return Res;
}
/// Returns true if all operands of the given instruction \p User are
/// replaced by copyable data.
/// \param User The user instruction.
/// \param Op The operand, which might be replaced by the copyable data.
/// \param SLP The SLP tree.
/// \param NumOps The number of operands used. If the instruction uses the
/// same operand several times, check for the first use, then the second,
/// etc.
bool areAllOperandsReplacedByCopyableData(Instruction *User,
Instruction *Op, BoUpSLP &SLP,
unsigned NumOps) const {
assert(NumOps > 0 && "No operands");
if (ScheduleCopyableDataMap.empty())
return false;
SmallDenseMap<TreeEntry *, unsigned> PotentiallyReorderedEntriesCount;
ArrayRef<TreeEntry *> Entries = SLP.getTreeEntries(User);
if (Entries.empty())
return false;
unsigned CurNumOps = 0;
for (const Use &U : make_filter_range(
User->operands(), [&](const Use &U) { return U.get() == Op; })) {
++CurNumOps;
// Check all tree entries, if they have operands replaced by copyable
// data.
for (TreeEntry *TE : Entries) {
unsigned Inc = 0;
bool IsNonSchedulableWithParentPhiNode =
TE->doesNotNeedToSchedule() && TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->hasState() &&
TE->UserTreeIndex.UserTE->State != TreeEntry::SplitVectorize &&
TE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI;
// Count the number of unique phi nodes, which are the parent for
// parent entry, and exit, if all the unique phis are processed.
if (IsNonSchedulableWithParentPhiNode) {
SmallPtrSet<Value *, 4> ParentsUniqueUsers;
const TreeEntry *ParentTE = TE->UserTreeIndex.UserTE;
for (PHINode *PHI : make_isa_range<PHINode>(ParentTE->Scalars))
if (ParentsUniqueUsers.insert(PHI).second &&
is_contained(PHI->incoming_values(), User))
++Inc;
} else {
Inc = count(TE->Scalars, User);
}
// Check if the user is commutative.
// The commutatives are handled later, as their operands can be
// reordered.
// Same applies even for non-commutative cmps, because we can invert
// their predicate potentially and, thus, reorder the operands.
bool IsCommutativeUser =
isCommutative(User) &&
isCommutableOperand(User, User, U.getOperandNo());
if (!IsCommutativeUser) {
Instruction *MainOp = TE->getMatchingMainOpOrAltOp(User);
IsCommutativeUser =
isCommutative(MainOp, User) &&
isCommutableOperand(MainOp, User, U.getOperandNo());
}
// The commutative user with the same operands can be safely
// considered as non-commutative, operands reordering does not change
// the semantics. Same for cmps with the same operands: inverting
// the predicate does not change the operand columns in this case.
assert(
(!IsCommutativeUser ||
(((isCommutative(User) && isCommutableOperand(User, User, 0) &&
isCommutableOperand(User, User, 1)) ||
(isCommutative(TE->getMatchingMainOpOrAltOp(User), User) &&
isCommutableOperand(TE->getMatchingMainOpOrAltOp(User), User,
0) &&
isCommutableOperand(TE->getMatchingMainOpOrAltOp(User), User,
1))))) &&
"Expected commutative user with 2 first commutable operands");
bool IsCommutativeWithSameOps =
IsCommutativeUser && User->getOperand(0) == User->getOperand(1);
if ((!IsCommutativeUser || IsCommutativeWithSameOps) &&
(!isa<CmpInst>(User) ||
User->getOperand(0) == User->getOperand(1))) {
if (CurNumOps != NumOps)
continue;
// A reassociated node flattens the operand chain, so the operand
// may be placed in any operand column rather than at the
// instruction's operand number.
if (TE->hasReassocScalars()) {
bool ReplacedByCopyable = false;
for (auto It = find(TE->Scalars, User); It != TE->Scalars.end();
It = find(make_range(std::next(It), TE->Scalars.end()),
User)) {
int Lane = std::distance(TE->Scalars.begin(), It);
for (unsigned OpIdx : seq<unsigned>(TE->getNumOperands()))
ReplacedByCopyable |=
TE->getOperand(OpIdx)[Lane] == Op &&
getScheduleCopyableData(EdgeInfo(TE, OpIdx), Op);
}
if (ReplacedByCopyable)
continue;
return false;
}
EdgeInfo EI(TE, U.getOperandNo());
if (getScheduleCopyableData(EI, Op))
continue;
return false;
}
// Only count the occurrence matching this call's NumOps.
if (CurNumOps != NumOps)
continue;
PotentiallyReorderedEntriesCount.try_emplace(TE, 0)
.first->getSecond() += Inc;
}
}
if (PotentiallyReorderedEntriesCount.empty())
return true;
// Check the commutative/cmp entries.
for (auto &P : PotentiallyReorderedEntriesCount) {
SmallPtrSet<Value *, 4> ParentsUniqueUsers;
bool IsNonSchedulableWithParentPhiNode =
P.first->doesNotNeedToSchedule() && P.first->UserTreeIndex &&
P.first->UserTreeIndex.UserTE->hasState() &&
P.first->UserTreeIndex.UserTE->State != TreeEntry::SplitVectorize &&
P.first->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI;
auto *It = find(P.first->Scalars, User);
do {
assert(It != P.first->Scalars.end() &&
"User is not in the tree entry");
int Lane = std::distance(P.first->Scalars.begin(), It);
assert(Lane >= 0 && "Lane is not found");
if (isa<StoreInst, InsertValueInst>(User) &&
!P.first->ReorderIndices.empty())
Lane = P.first->ReorderIndices[Lane];
assert(Lane < static_cast<int>(P.first->Scalars.size()) &&
"Couldn't find extract lane");
// Count the number of unique phi nodes, which are the parent for
// parent entry, and exit, if all the unique phis are processed.
if (IsNonSchedulableWithParentPhiNode) {
const TreeEntry *ParentTE = P.first->UserTreeIndex.UserTE;
Value *User = ParentTE->Scalars[Lane];
if (!ParentsUniqueUsers.insert(User).second) {
It =
find(make_range(std::next(It), P.first->Scalars.end()), User);
continue;
}
}
// Flattened nodes may place an operand in any column; scan all of
// them so copyable scheduling does not double-count.
for (unsigned OpIdx :
seq<unsigned>(P.first->hasReassocScalars()
? P.first->getNumOperands()
: getNumberOfPotentiallyCommutativeOps(
P.first->getMainOp()))) {
if (P.first->getOperand(OpIdx)[Lane] == Op &&
getScheduleCopyableData(EdgeInfo(P.first, OpIdx), Op))
--P.getSecond();
}
// If parent node is schedulable, it will be handled correctly.
It = find(make_range(std::next(It), P.first->Scalars.end()), User);
} while (It != P.first->Scalars.end());
}
return all_of(PotentiallyReorderedEntriesCount,
[&](const auto &P) { return P.second == NumOps - 1; });
}
SmallVector<ScheduleCopyableData *>
getScheduleCopyableData(const Instruction *I) const {
if (ScheduleCopyableDataMapByInst.empty())
return {};
const auto It = ScheduleCopyableDataMapByInst.find(I);
if (It == ScheduleCopyableDataMapByInst.end())
return {};
SmallVector<ScheduleCopyableData *> Res;
for (ScheduleCopyableData *SD : It->getSecond()) {
if (isInSchedulingRegion(*SD))
Res.push_back(SD);
}
return Res;
}
SmallVector<ScheduleCopyableData *>
getScheduleCopyableDataUsers(const Instruction *User) const {
if (ScheduleCopyableDataMapByUsers.empty())
return {};
const auto It = ScheduleCopyableDataMapByUsers.find(User);
if (It == ScheduleCopyableDataMapByUsers.end())
return {};
SmallVector<ScheduleCopyableData *> Res;
for (ScheduleCopyableData *SD : It->getSecond()) {
if (isInSchedulingRegion(*SD))
Res.push_back(SD);
}
return Res;
}
/// Reordering \p TE permutes its operand columns and may move an operand
/// between the edges covered and not covered by copyable scheduling
/// data, making the computed dependency counts stale. Mark the schedule
/// data of \p TE's copyable-modeled operands for recalculation at the
/// next bundle scheduling.
void markCopyableDepsForRecalc(const TreeEntry &TE) {
for (unsigned OpIdx : seq<unsigned>(TE.getNumOperands()))
for (Instruction *I : make_isa_range<Instruction>(TE.getOperand(OpIdx)))
if (ScheduleData *SD = getScheduleData(I);
SD && !getScheduleCopyableData(I).empty())
RecalcCopyableOperandDeps.insert(SD);
}
ScheduleCopyableData &addScheduleCopyableData(const EdgeInfo &EI,
Instruction *I,
int SchedulingRegionID,
ScheduleBundle &Bundle) {
assert(!getScheduleCopyableData(EI, I) && "already in the map");
ScheduleCopyableData *CD =
ScheduleCopyableDataMap
.try_emplace(std::make_pair(EI, I),
std::make_unique<ScheduleCopyableData>(
SchedulingRegionID, I, EI, Bundle))
.first->getSecond()
.get();
ScheduleCopyableDataMapByInst[I].push_back(CD);
if (EI.UserTE) {
ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
const auto *It = find(Op, I);
assert(It != Op.end() && "Lane not set");
SmallPtrSet<Instruction *, 4> Visited;
do {
int Lane = std::distance(Op.begin(), It);
assert(Lane >= 0 && "Lane not set");
if (isa<StoreInst, InsertValueInst>(EI.UserTE->Scalars[Lane]) &&
!EI.UserTE->ReorderIndices.empty())
Lane = EI.UserTE->ReorderIndices[Lane];
assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
"Couldn't find extract lane");
auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
if (!Visited.insert(In).second) {
It = find(make_range(std::next(It), Op.end()), I);
continue;
}
ScheduleCopyableDataMapByInstUser
.try_emplace(std::make_pair(std::make_pair(In, EI.EdgeIdx), I))
.first->getSecond()
.push_back(CD);
ScheduleCopyableDataMapByUsers.try_emplace(I)
.first->getSecond()
.insert(CD);
// Remove extra deps for users, becoming non-immediate users of the
// instruction. It may happen, if the chain of same copyable elements
// appears in the tree. With the self-operand modeling (op(V, V)) the
// user node has the value on more than one operand column, so its
// copyable data releases the instruction's schedule data directly
// and must stay a counted user.
if (In == I) {
unsigned NumCols = 0;
for (unsigned OpIdx : seq<unsigned>(EI.UserTE->getNumOperands()))
NumCols += EI.UserTE->getOperand(OpIdx)[Lane] == I;
if (NumCols <= 1) {
EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
if (ScheduleCopyableData *UserCD =
getScheduleCopyableData(UserEI, In))
ScheduleCopyableDataMapByUsers[I].remove(UserCD);
}
}
It = find(make_range(std::next(It), Op.end()), I);
} while (It != Op.end());
} else {
ScheduleCopyableDataMapByUsers.try_emplace(I).first->getSecond().insert(
CD);
}
return *CD;
}
/// Unregisters \p Bundle from its instructions and removes its copyable
/// data. The caller must destroy the bundle and recalculate the
/// dependencies.
void cancelScheduling(ScheduleBundle &Bundle) {
for (ScheduleEntity *SE : Bundle.getBundle()) {
Instruction *I = SE->getInst();
auto *CD = dyn_cast<ScheduleCopyableData>(SE);
if (!CD) {
llvm::erase(ScheduledBundles.find(I)->getSecond(), &Bundle);
continue;
}
const EdgeInfo EI = CD->getEdgeInfo();
llvm::erase(ScheduleCopyableDataMapByInst.find(I)->getSecond(), CD);
ScheduleCopyableDataMapByUsers[I].remove(CD);
if (EI.UserTE) {
ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
const auto *It = find(Op, I);
assert(It != Op.end() && "Lane not set");
SmallPtrSet<Instruction *, 4> Visited;
bool HadSelfUse = false;
do {
int Lane = std::distance(Op.begin(), It);
assert(Lane >= 0 && "Lane not set");
if (isa<StoreInst, InsertValueInst>(EI.UserTE->Scalars[Lane]) &&
!EI.UserTE->ReorderIndices.empty())
Lane = EI.UserTE->ReorderIndices[Lane];
assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
"Couldn't find extract lane");
auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
if (!Visited.insert(In).second) {
It = find(make_range(std::next(It), Op.end()), I);
continue;
}
HadSelfUse |= In == I;
auto ByUserIt = ScheduleCopyableDataMapByInstUser.find(
std::make_pair(std::make_pair(In, EI.EdgeIdx), I));
assert(ByUserIt != ScheduleCopyableDataMapByInstUser.end() &&
is_contained(ByUserIt->getSecond(), CD) &&
"copyable data is not registered for the user");
llvm::erase(ByUserIt->getSecond(), CD);
It = find(make_range(std::next(It), Op.end()), I);
} while (It != Op.end());
// Restore the parent-edge copyable data as a user of I only if the
// cancelled data displaced it at creation (chained copyable
// self-use); otherwise the extra user dependency is never released.
if (HadSelfUse) {
EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
if (ScheduleCopyableData *UserCD =
getScheduleCopyableData(UserEI, I))
ScheduleCopyableDataMapByUsers[I].insert(UserCD);
}
}
if (ScheduleCopyableDataMapByUsers[I].empty())
ScheduleCopyableDataMapByUsers.erase(I);
ScheduleCopyableDataMap.erase(std::make_pair(EI, I));
}
}
/// Clears the dependencies of all schedule data in the scheduling region.
void clearDependencies() {
for_each(ScheduleDataMap, [&](auto &P) {
if (BB != P.first->getParent())
return;
ScheduleData *SD = P.second;
if (isInSchedulingRegion(*SD))
SD->clearDependencies();
});
for_each(ScheduleCopyableDataMapByInst, [&](auto &P) {
for_each(P.second, [&](ScheduleCopyableData *SD) {
if (isInSchedulingRegion(*SD))
SD->clearDependencies();
});
});
}
ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return {};
auto It = ScheduledBundles.find(I);
if (It == ScheduledBundles.end())
return {};
return It->getSecond();
}
/// Returns true if the entity is in the scheduling region.
bool isInSchedulingRegion(const ScheduleEntity &SD) const {
if (const auto *Data = dyn_cast<ScheduleData>(&SD))
return Data->getSchedulingRegionID() == SchedulingRegionID;
if (const auto *CD = dyn_cast<ScheduleCopyableData>(&SD))
return CD->getSchedulingRegionID() == SchedulingRegionID;
return all_of(cast<ScheduleBundle>(SD).getBundle(),
[&](const ScheduleEntity *BundleMember) {
return isInSchedulingRegion(*BundleMember);
});
}
/// Marks an instruction as scheduled and puts all dependent ready
/// instructions into the ready-list.
template <typename ReadyListType>
void schedule(const BoUpSLP &R, const InstructionsState &S,
const EdgeInfo &EI, ScheduleEntity *Data,
ReadyListType &ReadyList) {
auto ProcessBundleMember = [&](ScheduleEntity *BundleMember,
ArrayRef<ScheduleBundle *> Bundles) {
// Handle the def-use chain dependencies.
// Decrement the unscheduled counter and insert to ready list if ready.
auto DecrUnsched = [&](auto *Data, bool IsControl = false) {
if ((IsControl || Data->hasValidDependencies()) &&
Data->incrementUnscheduledDeps(-1) == 0) {
// There are no more unscheduled dependencies after
// decrementing, so we can put the dependent instruction
// into the ready list.
SmallVector<ScheduleBundle *, 1> CopyableBundle;
ArrayRef<ScheduleBundle *> Bundles;
if (auto *CD = dyn_cast<ScheduleCopyableData>(Data)) {
CopyableBundle.push_back(&CD->getBundle());
Bundles = CopyableBundle;
} else {
Bundles = getScheduleBundles(Data->getInst());
}
if (!Bundles.empty()) {
for (ScheduleBundle *Bundle : Bundles) {
if (Bundle->unscheduledDepsInBundle() == 0) {
assert(!Bundle->isScheduled() &&
"already scheduled bundle gets ready");
ReadyList.insert(Bundle);
LLVM_DEBUG(dbgs()
<< "SLP: gets ready: " << *Bundle << "\n");
}
}
return;
}
assert(!Data->isScheduled() &&
"already scheduled bundle gets ready");
assert(!isa<ScheduleCopyableData>(Data) &&
"Expected non-copyable data");
ReadyList.insert(Data);
LLVM_DEBUG(dbgs() << "SLP: gets ready: " << *Data << "\n");
}
};
auto DecrUnschedForInst = [&](Instruction *User, unsigned OpIdx,
Instruction *I) {
if (!ScheduleCopyableDataMap.empty()) {
SmallVector<ScheduleCopyableData *> CopyableData =
getScheduleCopyableData(User, OpIdx, I);
bool ReleasedAsCopyable = false;
// Copyable elements modeled on a copyable user lane depend on
// the user's copyable scheduling data, not on the user itself,
// and are released when that copyable data is scheduled. The
// user's own schedule data still carries the def-use dependency
// in this case, so it must be released below.
auto IsNotCopyable = [&](ScheduleCopyableData *CD) {
return !CD->getEdgeInfo().UserTE->isCopyableElement(User);
};
for (ScheduleCopyableData *CD :
make_filter_range(CopyableData, IsNotCopyable)) {
DecrUnsched(CD, /*IsControl=*/false);
ReleasedAsCopyable = true;
}
if (ReleasedAsCopyable)
return;
}
if (ScheduleData *OpSD = getScheduleData(I))
DecrUnsched(OpSD, /*IsControl=*/false);
};
// If BundleMember is a vector bundle, its operands may have been
// reordered during buildTree(). We therefore need to get its operands
// through the TreeEntry.
if (!Bundles.empty()) {
auto *In = BundleMember->getInst();
// Count uses of each instruction operand.
SmallDenseMap<const Instruction *, unsigned> OperandsUses;
unsigned TotalOpCount = 0;
if (isa<ScheduleCopyableData>(BundleMember)) {
// Copyable data is used only once (uses itself).
TotalOpCount = OperandsUses[In] = 1;
} else {
for (const Use &U : In->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
auto Res = OperandsUses.try_emplace(I, 0);
unsigned ExtraDeps = 1;
// Count all expanded operands in the binops.
for (ScheduleBundle *Bundle : Bundles) {
if (const TreeEntry *TE = Bundle->getTreeEntry()) {
if (TE->isExpandedBinOp(In))
++ExtraDeps;
} else if (S.isExpandedBinOp(In)) {
++ExtraDeps;
}
}
Res.first->getSecond() += ExtraDeps;
TotalOpCount += ExtraDeps;
}
}
}
// Tracks whether the bundle member instruction itself shows up in
// some operand column of its node (only copyable elements modeled
// through their own operands, like absorbed fmuls, do not).
bool FoundInOpColumns = false;
// Decrement the unscheduled counter and insert to ready list if
// ready.
auto DecrUnschedForInst =
[&](Instruction *I, TreeEntry *UserTE, unsigned OpIdx,
SmallDenseSet<std::pair<const ScheduleEntity *, unsigned>>
&Checked,
bool IsExpandedOperand = false,
bool CopyableDepsOnly = false) {
if (!ScheduleCopyableDataMap.empty()) {
const EdgeInfo EI = {UserTE, OpIdx};
if (ScheduleCopyableData *CD =
getScheduleCopyableData(EI, I)) {
if (!Checked.insert(std::make_pair(CD, OpIdx)).second)
return;
DecrUnsched(CD, /*IsControl=*/false);
return;
}
}
if (CopyableDepsOnly)
return;
auto It = OperandsUses.find(I);
if (It == OperandsUses.end()) {
// Column value may be a peeled intermediate, not a direct
// operand of In; its deps are released when it is scheduled.
LLVM_DEBUG(dbgs() << "SLP: operand " << *I
<< " not modeled as a direct operand of "
<< *In << ", skipping.\n");
return;
}
if (It->second > 0) {
if (ScheduleData *OpSD = getScheduleData(I)) {
if (!IsExpandedOperand &&
!Checked.insert(std::make_pair(OpSD, OpIdx)).second)
return;
--It->getSecond();
assert(TotalOpCount > 0 && "No more operands to decrement");
--TotalOpCount;
DecrUnsched(OpSD, /*IsControl=*/false);
} else {
--It->getSecond();
assert(TotalOpCount > 0 && "No more operands to decrement");
--TotalOpCount;
}
}
};
SmallDenseSet<std::pair<const ScheduleEntity *, unsigned>> Checked;
for (ScheduleBundle *Bundle : Bundles) {
if (ScheduleCopyableDataMap.empty() && TotalOpCount == 0)
break;
SmallPtrSet<Value *, 4> ParentsUniqueUsers;
// Need to search for the lane since the tree entry can be
// reordered.
auto *It = find(Bundle->getTreeEntry()->Scalars, In);
bool IsNonSchedulableWithParentPhiNode =
Bundle->getTreeEntry()->doesNotNeedToSchedule() &&
Bundle->getTreeEntry()->UserTreeIndex &&
Bundle->getTreeEntry()->UserTreeIndex.UserTE->hasState() &&
Bundle->getTreeEntry()->UserTreeIndex.UserTE->State !=
TreeEntry::SplitVectorize &&
Bundle->getTreeEntry()->UserTreeIndex.UserTE->getOpcode() ==
Instruction::PHI;
do {
int Lane =
std::distance(Bundle->getTreeEntry()->Scalars.begin(), It);
assert(Lane >= 0 && "Lane not set");
if (isa<StoreInst, InsertValueInst>(In) &&
!Bundle->getTreeEntry()->ReorderIndices.empty())
Lane = Bundle->getTreeEntry()->ReorderIndices[Lane];
assert(Lane < static_cast<int>(
Bundle->getTreeEntry()->Scalars.size()) &&
"Couldn't find extract lane");
// Since vectorization tree is being built recursively this
// assertion ensures that the tree entry has all operands set
// before reaching this code. Couple of exceptions known at the
// moment are extracts where their second (immediate) operand is
// not added. Since immediates do not affect scheduler behavior
// this is considered okay.
assert(
In &&
(isa<ExtractValueInst, ExtractElementInst, CallBase>(In) ||
In->getNumOperands() ==
Bundle->getTreeEntry()->getNumOperands() ||
(isa<ZExtInst>(In) && Bundle->getTreeEntry()->getOpcode() ==
Instruction::Select) ||
Bundle->getTreeEntry()->isCopyableElement(In) ||
Bundle->getTreeEntry()->hasReassocScalars()) &&
"Missed TreeEntry operands?");
// Count the number of unique phi nodes, which are the parent
// entry, and handle the non-copyable deps only on the first lane
// for each such phi. Copyable deps are counted per operand column
// lane and are released on every lane.
bool CopyableDepsOnly =
IsNonSchedulableWithParentPhiNode &&
!ParentsUniqueUsers
.insert(Bundle->getTreeEntry()
->UserTreeIndex.UserTE->Scalars[Lane])
.second;
// A blended-load operand node is the synthetic blend mask, not an
// IR operand of the load. Use the real pointer operand for
// scheduling so the def-use counters stay balanced; the mask is
// available earlier through the pointer's select.
bool IsBlended = Bundle->getTreeEntry()->State ==
TreeEntry::BlendedLoadVectorize;
for (unsigned OpIdx :
seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
if (auto *I = dyn_cast<Instruction>(
IsBlended ? In->getOperand(OpIdx)
: Bundle->getTreeEntry()->getOperand(
OpIdx)[Lane])) {
FoundInOpColumns |= (I == In) && !CopyableDepsOnly;
LLVM_DEBUG(dbgs() << "SLP: check for readiness (def): "
<< *I << "\n");
DecrUnschedForInst(
I, Bundle->getTreeEntry(), OpIdx, Checked,
Bundle->getTreeEntry()->isExpandedOperand(In, OpIdx),
/*CopyableDepsOnly=*/CopyableDepsOnly);
}
// If parent node is schedulable, it will be handled correctly.
if (Bundle->getTreeEntry()->isCopyableElement(In))
break;
It = std::find(std::next(It),
Bundle->getTreeEntry()->Scalars.end(), In);
} while (It != Bundle->getTreeEntry()->Scalars.end());
}
// A copyable element absorbed into its user modeling (e.g. a
// copyable fmul turned into fmuladd(a, b, -0.0)) does not appear in
// the operand columns of its own node, so the scan above never
// releases the schedule data of the copyable instruction itself.
// Release it here to keep the unscheduled-deps counters balanced,
// consuming its self-use count so the reassociated-operand release
// below cannot release the same schedule data twice.
if (isa<ScheduleCopyableData>(BundleMember) && !FoundInOpColumns) {
auto UseIt = OperandsUses.find(In);
if (UseIt != OperandsUses.end() && UseIt->second > 0) {
--UseIt->getSecond();
--TotalOpCount;
}
if (ScheduleData *OpSD = getScheduleData(In))
DecrUnsched(OpSD, /*IsControl=*/false);
}
// Vector intrinsics may keep some arguments scalar (e.g. the
// exponent of llvm.powi). Such scalar arguments are not modeled as
// tree-entry operands, so the per-lane loop above never releases the
// dependency that calculateDependencies() registered for the
// definition feeding such an argument. Release it here to keep the
// unscheduled-deps counters balanced; otherwise the operand's bundle
// may never become ready and scheduling would assert.
if (TotalOpCount > 0) {
if (auto *CI = dyn_cast<CallInst>(In)) {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, R.TLI);
for (unsigned ArgIdx : seq<unsigned>(CI->arg_size())) {
if (!isVectorIntrinsicWithScalarOpAtArg(ID, ArgIdx, R.TTI))
continue;
auto *OpI = dyn_cast<Instruction>(CI->getArgOperand(ArgIdx));
if (!OpI)
continue;
auto UseIt = OperandsUses.find(OpI);
if (UseIt == OperandsUses.end() || UseIt->second == 0)
continue;
--UseIt->getSecond();
--TotalOpCount;
if (ScheduleData *OpSD = getScheduleData(OpI)) {
LLVM_DEBUG(dbgs()
<< "SLP: check for readiness (scalar arg): "
<< *OpI << "\n");
DecrUnsched(OpSD, /*IsControl=*/false);
}
}
}
// Peeled intermediates stay as direct operands but drop out of
// operand columns; release their scheduling deps here.
for (const ScheduleBundle *Bundle : Bundles) {
if (TotalOpCount == 0)
break;
TreeEntry *TE = Bundle->getTreeEntry();
if (!TE->hasReassocScalars())
continue;
for (Instruction *OpI :
make_isa_range<Instruction>(TE->getReassocScalars())) {
auto UseIt = OperandsUses.find(OpI);
if (UseIt == OperandsUses.end() || UseIt->second == 0)
continue;
LLVM_DEBUG(dbgs() << "SLP: check for readiness "
"(reassociated operand): "
<< *OpI << "\n");
// Copyable deps may live on per-edge ScheduleCopyableData.
bool ReleasedAsCopyable = false;
if (!ScheduleCopyableDataMap.empty()) {
auto MatchesOp = [&](const Use &U) { return U.get() == OpI; };
for (const Use &U :
make_filter_range(In->operands(), MatchesOp)) {
for (ScheduleCopyableData *CD :
getScheduleCopyableData(In, U.getOperandNo(), OpI)) {
// Deps of reassoc scalars modeled as copyable tree
// operands are released by the operand scan above;
// release each remaining dep only once.
if (Checked.insert(std::make_pair(CD, U.getOperandNo()))
.second)
DecrUnsched(CD, /*IsControl=*/false);
}
}
// The dep is released through copyable data only if this
// very entry models the scalar as a copyable operand on one
// of its edges, mirroring the dependency calculation;
// copyable data on some other entry's edge does not cover
// the dep registered for this entry.
for (auto It = find(TE->Scalars, In);
It != TE->Scalars.end() && !ReleasedAsCopyable;
It = find(make_range(std::next(It), TE->Scalars.end()),
In)) {
int Lane = std::distance(TE->Scalars.begin(), It);
for (unsigned OpIdx : seq<unsigned>(TE->getNumOperands()))
ReleasedAsCopyable |=
TE->getOperand(OpIdx)[Lane] == OpI &&
getScheduleCopyableData(EdgeInfo(TE, OpIdx), OpI);
}
}
if (!ReleasedAsCopyable) {
if (ScheduleData *OpSD = getScheduleData(OpI))
for (unsigned I = 0, E = UseIt->second; I != E; ++I)
DecrUnsched(OpSD, /*IsControl=*/false);
}
TotalOpCount -= UseIt->second;
UseIt->second = 0;
}
}
}
} else {
// If BundleMember is a stand-alone instruction, no operand reordering
// has taken place, so we directly access its operands.
for (Use &U : BundleMember->getInst()->operands()) {
if (auto *I = dyn_cast<Instruction>(U.get())) {
LLVM_DEBUG(dbgs()
<< "SLP: check for readiness (def): " << *I << "\n");
DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(), I);
}
}
}
// Handle the memory dependencies.
auto *SD = dyn_cast<ScheduleData>(BundleMember);
if (!SD)
return;
SmallPtrSet<const ScheduleData *, 4> VisitedMemory;
for (ScheduleData *MemoryDep : SD->getMemoryDependencies()) {
if (!VisitedMemory.insert(MemoryDep).second)
continue;
// There are no more unscheduled dependencies after decrementing,
// so we can put the dependent instruction into the ready list.
LLVM_DEBUG(dbgs() << "SLP: check for readiness (mem): "
<< *MemoryDep << "\n");
DecrUnsched(MemoryDep);
}
// Handle the control dependencies.
SmallPtrSet<const ScheduleData *, 4> VisitedControl;
for (ScheduleData *Dep : SD->getControlDependencies()) {
if (!VisitedControl.insert(Dep).second)
continue;
// There are no more unscheduled dependencies after decrementing,
// so we can put the dependent instruction into the ready list.
LLVM_DEBUG(dbgs()
<< "SLP: check for readiness (ctrl): " << *Dep << "\n");
DecrUnsched(Dep, /*IsControl=*/true);
}
};
if (auto *SD = dyn_cast<ScheduleData>(Data)) {
SD->setScheduled(/*Scheduled=*/true);
LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
SmallVector<std::unique_ptr<ScheduleBundle>> PseudoBundles;
SmallVector<ScheduleBundle *> Bundles;
Instruction *In = SD->getInst();
ArrayRef<TreeEntry *> Entries = R.getTreeEntries(In);
if (!Entries.empty()) {
for (TreeEntry *TE : make_filter_range(Entries, [&](TreeEntry *TE) {
return isa<ExtractValueInst, ExtractElementInst, CallBase>(
In) ||
In->getNumOperands() == TE->getNumOperands() ||
TE->hasReassocScalars();
})) {
auto &BundlePtr =
PseudoBundles.emplace_back(std::make_unique<ScheduleBundle>());
BundlePtr->setTreeEntry(TE);
BundlePtr->add(SD);
Bundles.push_back(BundlePtr.get());
}
}
ProcessBundleMember(SD, Bundles);
} else {
ScheduleBundle &Bundle = *cast<ScheduleBundle>(Data);
Bundle.setScheduled(/*Scheduled=*/true);
LLVM_DEBUG(dbgs() << "SLP: schedule " << Bundle << "\n");
auto AreAllBundlesScheduled =
[&](const ScheduleEntity *SD,
ArrayRef<ScheduleBundle *> SDBundles) {
if (isa<ScheduleCopyableData>(SD))
return true;
return !SDBundles.empty() &&
all_of(SDBundles, [&](const ScheduleBundle *SDBundle) {
return SDBundle->isScheduled();
});
};
for (ScheduleEntity *SD : Bundle.getBundle()) {
ArrayRef<ScheduleBundle *> SDBundles;
if (!isa<ScheduleCopyableData>(SD))
SDBundles = getScheduleBundles(SD->getInst());
if (!AreAllBundlesScheduled(SD, SDBundles))
continue;
SD->setScheduled(/*Scheduled=*/true);
Instruction *In = SD->getInst();
// The instruction may also belong to tree entries that do not need
// scheduling (e.g. all their values are used outside the block), so
// no schedule bundle is registered for them. Such an entry can still
// model one of this instruction's operands as a copyable element, or
// model the instruction itself as an expanded binop, registered on
// that non-scheduled parent edge. That dependency would never be
// decremented when the instruction is scheduled through a different
// bundle, leaving the operand's bundle permanently unscheduled and
// tripping the unscheduled-deps assertion. Add pseudo-bundles for
// these missing tree entries, so their operand dependencies are
// decremented here as well. Real operand dependencies are protected
// against double counting by the per-operand use counter.
if (isa<ScheduleCopyableData>(SD) ||
(ScheduleCopyableDataMap.empty() &&
none_of(R.getTreeEntries(In), [&](const TreeEntry *TE) {
return TE->isExpandedBinOp(In);
}))) {
ProcessBundleMember(SD, isa<ScheduleCopyableData>(SD) ? &Bundle
: SDBundles);
continue;
}
SmallVector<std::unique_ptr<ScheduleBundle>> PseudoBundles;
SmallVector<ScheduleBundle *> AllBundles(SDBundles.begin(),
SDBundles.end());
for (TreeEntry *TE :
make_filter_range(R.getTreeEntries(In), [&](TreeEntry *TE) {
return !TE->isCopyableElement(In);
})) {
if (!isa<ExtractValueInst, ExtractElementInst, CallBase>(In) &&
In->getNumOperands() != TE->getNumOperands() &&
!TE->hasReassocScalars())
continue;
if (any_of(SDBundles, [&](const ScheduleBundle *SDBundle) {
return SDBundle->getTreeEntry() == TE;
}))
continue;
ScheduleBundle &PseudoBundle =
*PseudoBundles.emplace_back(std::make_unique<ScheduleBundle>());
PseudoBundle.setTreeEntry(TE);
PseudoBundle.add(SD);
AllBundles.push_back(&PseudoBundle);
}
ProcessBundleMember(SD, AllBundles);
}
}
}
/// Verify basic self consistency properties of the data structure.
void verify() {
if (!ScheduleStart)
return;
assert(ScheduleStart->getParent() == ScheduleEnd->getParent() &&
ScheduleStart->comesBefore(ScheduleEnd) &&
"Not a valid scheduling region?");
for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(I);
if (!Bundles.empty()) {
for (ScheduleBundle *Bundle : Bundles) {
assert(isInSchedulingRegion(*Bundle) &&
"primary schedule data not in window?");
Bundle->verify();
}
continue;
}
auto *SD = getScheduleData(I);
if (!SD)
continue;
assert(isInSchedulingRegion(*SD) &&
"primary schedule data not in window?");
SD->verify();
}
assert(all_of(ReadyInsts,
[](const ScheduleEntity *Bundle) {
return Bundle->isReady();
}) &&
"item in ready list not ready?");
}
/// Put all instructions into the ReadyList which are ready for scheduling.
template <typename ReadyListType>
void initialFillReadyList(ReadyListType &ReadyList) {
SmallPtrSet<ScheduleBundle *, 16> Visited;
for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
ScheduleData *SD = getScheduleData(I);
if (SD && SD->hasValidDependencies() && SD->isReady()) {
if (ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(I);
!Bundles.empty()) {
for (ScheduleBundle *Bundle :
make_filter_range(Bundles, [&](ScheduleBundle *Bundle) {
return Visited.insert(Bundle).second;
})) {
if (Bundle->hasValidDependencies() && Bundle->isReady()) {
ReadyList.insert(Bundle);
LLVM_DEBUG(dbgs() << "SLP: initially in ready list: "
<< *Bundle << "\n");
}
}
continue;
}
ReadyList.insert(SD);
LLVM_DEBUG(dbgs()
<< "SLP: initially in ready list: " << *SD << "\n");
}
}
}
/// Build a bundle from the ScheduleData nodes corresponding to the
/// scalar instruction for each lane.
/// \param VL The list of scalar instructions.
/// \param S The state of the instructions.
/// \param EI The edge in the SLP graph or the user node/operand number.
ScheduleBundle &buildBundle(ArrayRef<Value *> VL,
const InstructionsState &S, const EdgeInfo &EI);
/// Checks if a bundle of instructions can be scheduled, i.e. has no
/// cyclic dependencies. This is only a dry-run, no instructions are
/// actually moved at this stage.
/// \returns the scheduling bundle. The returned Optional value is not
/// std::nullopt if \p VL is allowed to be scheduled.
std::optional<ScheduleBundle *>
tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S, const EdgeInfo &EI);
/// Allocates schedule data chunk.
ScheduleData *allocateScheduleDataChunks();
/// Extends the scheduling region so that V is inside the region.
/// \returns true if the region size is within the limit.
bool extendSchedulingRegion(Value *V, const InstructionsState &S);
/// Initialize the ScheduleData structures for new instructions in the
/// scheduling region.
void initScheduleData(Instruction *FromI, Instruction *ToI,
ScheduleData *PrevLoadStore,
ScheduleData *NextLoadStore);
/// Updates the dependency information of a bundle and of all instructions/
/// bundles which depend on the original bundle.
void calculateDependencies(ScheduleBundle &Bundle, bool InsertInReadyList,
BoUpSLP *SLP,
const SmallPtrSetImpl<Value *> &ExpandedOps,
ArrayRef<ScheduleData *> ControlDeps = {});
/// Sets all instruction in the scheduling region to un-scheduled.
void resetSchedule();
BasicBlock *BB;
/// Simple memory allocation for ScheduleData.
SmallVector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
/// The size of a ScheduleData array in ScheduleDataChunks.
int ChunkSize;
/// The allocator position in the current chunk, which is the last entry
/// of ScheduleDataChunks.
int ChunkPos;
/// Attaches ScheduleData to Instruction.
/// Note that the mapping survives during all vectorization iterations, i.e.
/// ScheduleData structures are recycled.
SmallDenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
/// Attaches ScheduleCopyableData to EdgeInfo (UserTreeEntry + operand
/// number) and the operand instruction, represented as copyable element.
SmallDenseMap<std::pair<EdgeInfo, const Value *>,
std::unique_ptr<ScheduleCopyableData>>
ScheduleCopyableDataMap;
/// Represents mapping between instruction and all related
/// ScheduleCopyableData (for all uses in the tree, represenedt as copyable
/// element). The SLP tree may contain several representations of the same
/// instruction.
SmallDenseMap<const Instruction *, SmallVector<ScheduleCopyableData *>>
ScheduleCopyableDataMapByInst;
/// Represents mapping between user value and operand number, the operand
/// value and all related ScheduleCopyableData. The relation is 1:n, because
/// the same user may refernce the same operand in different tree entries
/// and the operand may be modelled by the different copyable data element.
SmallDenseMap<std::pair<std::pair<const Value *, unsigned>, const Value *>,
SmallVector<ScheduleCopyableData *>>
ScheduleCopyableDataMapByInstUser;
/// Represents mapping between instruction and all related
/// ScheduleCopyableData. It represents the mapping between the actual
/// instruction and the last copyable data element in the chain. E.g., if
/// the graph models the following instructions:
/// %0 = non-add instruction ...
/// ...
/// %4 = add %3, 1
/// %5 = add %4, 1
/// %6 = insertelement poison, %0, 0
/// %7 = insertelement %6, %5, 1
/// And the graph is modeled as:
/// [%5, %0] -> [%4, copyable %0 <0> ] -> [%3, copyable %0 <1> ]
/// -> [1, 0] -> [%1, 0]
///
/// this map will map %0 only to the copyable element <1>, which is the last
/// user (direct user of the actual instruction). <0> uses <1>, so <1> will
/// keep the map to <0>, not the %0.
SmallDenseMap<const Instruction *,
SmallSetVector<ScheduleCopyableData *, 4>>
ScheduleCopyableDataMapByUsers;
/// Attaches ScheduleBundle to Instruction.
SmallDenseMap<Instruction *, SmallVector<ScheduleBundle *>>
ScheduledBundles;
/// The list of ScheduleBundles.
SmallVector<std::unique_ptr<ScheduleBundle>> ScheduledBundlesList;
/// The ready-list for scheduling (only used for the dry-run).
SetVector<ScheduleEntity *> ReadyInsts;
/// The first instruction of the scheduling region.
Instruction *ScheduleStart = nullptr;
/// The first instruction _after_ the scheduling region.
Instruction *ScheduleEnd = nullptr;
/// The first memory accessing instruction in the scheduling region
/// (can be null).
ScheduleData *FirstLoadStoreInRegion = nullptr;
/// The last memory accessing instruction in the scheduling region
/// (can be null).
ScheduleData *LastLoadStoreInRegion = nullptr;
/// Is there an llvm.stacksave or llvm.stackrestore in the scheduling
/// region? Used to optimize the dependence calculation for the
/// common case where there isn't.
bool RegionHasStackSave = false;
/// Used to skip the dependencies calculation for writes that are visible
/// after unwinding.
bool RegionHasMayThrow = false;
/// Returns true if \p I writes memory that the caller can still see
/// after an unwind.
bool isWriteVisibleOnUnwind(Instruction *I) const {
if (!RegionHasMayThrow || !I->mayWriteToMemory())
return false;
MemoryLocation Loc = getLocation(I);
if (!Loc.Ptr)
return true;
bool RequiresNoCaptureBeforeUnwind = false;
return !isNotVisibleOnUnwind(getUnderlyingObject(Loc.Ptr),
RequiresNoCaptureBeforeUnwind) ||
RequiresNoCaptureBeforeUnwind;
}
/// The current size of the scheduling region.
int ScheduleRegionSize = 0;
/// The maximum size allowed for the scheduling region.
int ScheduleRegionSizeLimit;
/// Operands that are modeled as copyable elements in a previously built
/// vectorized node and that are used directly by another,
/// not-yet-registered node sharing a schedulable instruction with it. Their
/// direct dependencies must be recomputed at the next bundle scheduling,
/// when the new node is already registered in the tree, so that the direct
/// use is accounted for. If the new node is the last scheduled bundle and
/// no further scheduling consumes this list, the leftover entries are
/// dropped on the next region reset and the dependencies are recomputed
/// against the full tree in scheduleBlock instead. A set is used to avoid
/// recomputing the same operand more than once.
SmallSetVector<ScheduleData *, 8> RecalcCopyableOperandDeps;
/// Ordered pairs (Src, Dst) of memory instructions whose may-alias
/// dependency has been dropped in favor of a runtime alias check.
SmallDenseSet<std::pair<Instruction *, Instruction *>, 8> IgnoredMemDeps;
/// The ID of the scheduling region. For a new vectorization iteration this
/// is incremented which "removes" all ScheduleData from the region.
/// Make sure that the initial SchedulingRegionID is greater than the
/// initial SchedulingRegionID in ScheduleData (which is 0).
int SchedulingRegionID = 1;
};
/// Attaches the BlockScheduling structures to basic blocks.
MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
/// Performs the "real" scheduling. Done before vectorization is actually
/// performed in a basic block.
void scheduleBlock(const BoUpSLP &R, BlockScheduling *BS);
/// List of users to ignore during scheduling and that don't need extracting.
const SmallDenseSet<Value *> *UserIgnoreList = nullptr;
/// Narrowed reduction chain instructions, dropped together with the
/// reduction. Subset of UserIgnoreList.
SmallPtrSet<Value *, 4> NarrowedChainInsts;
/// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
/// sorted SmallVectors of unsigned.
struct OrdersTypeDenseMapInfo {
static unsigned getHashValue(const OrdersType &V) {
return static_cast<unsigned>(hash_combine_range(V));
}
static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
return LHS == RHS;
}
};
// Analysis and block reference.
Function *F;
ScalarEvolution *SE;
TargetTransformInfo *TTI;
TargetLibraryInfo *TLI;
LoopInfo *LI;
DominatorTree *DT;
AssumptionCache *AC;
DemandedBits *DB;
const DataLayout *DL;
OptimizationRemarkEmitter *ORE;
/// Cached cost-model mode for this function.
/// If -Os/-Oz, use CodeSize. Otherwise use RecipThroughput.
const TargetTransformInfo::TargetCostKind CostKind;
unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
unsigned MinVecRegSize; // Set by cl::opt (default: 128).
/// Instruction builder to construct the vectorized tree.
IRBuilder<TargetFolder> Builder;
/// A map of scalar integer values to the smallest bit width with which they
/// can legally be represented. The values map to (width, signed) pairs,
/// where "width" indicates the minimum bit width and "signed" is True if the
/// value must be signed-extended, rather than zero-extended, back to its
/// original width.
DenseMap<const TreeEntry *, std::pair<uint64_t, bool>> MinBWs;
/// A booleanized node is an i1 node registered in MinBWs to be emitted in
/// the wide leaf type. Genuine demoted nodes never have i1 scalars (i1
/// cannot be demoted), so an i1 node in MinBWs is always a booleanized one.
bool isBooleanizedNode(const TreeEntry *E) const {
auto It = MinBWs.find(E);
return It != MinBWs.end() && E->Scalars.front()->getType()->isIntegerTy(1);
}
/// Final size of the reduced vector, if the current graph represents the
/// input for the reduction and it was possible to narrow the size of the
/// reduction.
unsigned ReductionBitWidth = 0;
/// Canonical graph size before the transformations.
unsigned BaseGraphSize = 1;
/// If the tree contains any zext/sext/trunc nodes, contains max-min pair of
/// type sizes, used in the tree.
std::optional<std::pair<unsigned, unsigned>> CastMaxMinBWSizes;
/// Indices of the vectorized nodes, which supposed to be the roots of the new
/// bitwidth analysis attempt, like trunc, IToFP or ICmp.
DenseSet<unsigned> ExtraBitWidthNodes;
};
template <> struct llvm::DenseMapInfo<BoUpSLP::EdgeInfo> {
using FirstInfo = DenseMapInfo<BoUpSLP::TreeEntry *>;
using SecondInfo = DenseMapInfo<unsigned>;
static unsigned getHashValue(const BoUpSLP::EdgeInfo &Val) {
return detail::combineHashValue(FirstInfo::getHashValue(Val.UserTE),
SecondInfo::getHashValue(Val.EdgeIdx));
}
static bool isEqual(const BoUpSLP::EdgeInfo &LHS,
const BoUpSLP::EdgeInfo &RHS) {
return LHS == RHS;
}
};
template <> struct llvm::GraphTraits<BoUpSLP *> {
using TreeEntry = BoUpSLP::TreeEntry;
/// NodeRef has to be a pointer per the GraphWriter.
using NodeRef = TreeEntry *;
using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy;
/// Add the VectorizableTree to the index iterator to be able to return
/// TreeEntry pointers.
struct ChildIteratorType
: public iterator_adaptor_base<
ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> {
ContainerTy &VectorizableTree;
ChildIteratorType(SmallVector<BoUpSLP::EdgeInfo, 1>::iterator W,
ContainerTy &VT)
: ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
NodeRef operator*() { return I->UserTE; }
};
static NodeRef getEntryNode(BoUpSLP &R) { return &R.getRootNode(); }
static ChildIteratorType child_begin(NodeRef N) {
return {&N->UserTreeIndex, N->Container};
}
static ChildIteratorType child_end(NodeRef N) {
return {&N->UserTreeIndex + 1, N->Container};
}
/// For the node iterator we just need to turn the TreeEntry iterator into a
/// TreeEntry* iterator so that it dereferences to NodeRef.
class nodes_iterator {
using ItTy = ContainerTy::iterator;
ItTy It;
public:
nodes_iterator(const ItTy &It2) : It(It2) {}
NodeRef operator*() { return It->get(); }
nodes_iterator operator++() {
++It;
return *this;
}
bool operator!=(const nodes_iterator &N2) const { return N2.It != It; }
};
static nodes_iterator nodes_begin(BoUpSLP *R) {
return nodes_iterator(R->VectorizableTree.begin());
}
static nodes_iterator nodes_end(BoUpSLP *R) {
return nodes_iterator(R->VectorizableTree.end());
}
static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
};
template <>
struct llvm::DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
using TreeEntry = BoUpSLP::TreeEntry;
DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {}
std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
std::string Str;
raw_string_ostream OS(Str);
OS << Entry->Idx << ".\n";
if (isSplat(Entry->Scalars))
OS << "<splat> ";
for (auto *V : Entry->Scalars) {
OS << *V;
if (llvm::any_of(R->ExternalUses, [&](const BoUpSLP::ExternalUser &EU) {
return EU.Scalar == V;
}))
OS << " <extract>";
OS << "\n";
}
return Str;
}
static std::string getNodeAttributes(const TreeEntry *Entry,
const BoUpSLP *) {
if (Entry->isGather())
return "color=red";
if (Entry->State == TreeEntry::ScatterVectorize ||
Entry->State == TreeEntry::StridedVectorize ||
Entry->State == TreeEntry::ExpandVectorize ||
Entry->State == TreeEntry::CompressVectorize ||
Entry->State == TreeEntry::BlendedLoadVectorize)
return "color=blue";
return "";
}
};
BoUpSLP::~BoUpSLP() {
SmallVector<WeakTrackingVH> DeadInsts;
for (auto *I : DeletedInstructions) {
if (!I->getParent()) {
// Temporarily insert instruction back to erase them from parent and
// memory later.
if (isa<PHINode>(I))
// Phi nodes must be the very first instructions in the block.
I->insertBefore(F->getEntryBlock(),
F->getEntryBlock().getFirstNonPHIIt());
else
I->insertBefore(F->getEntryBlock().getTerminator()->getIterator());
continue;
}
for (Instruction *Op : make_isa_range<Instruction>(I->operands()))
if (!DeletedInstructions.count(Op) && Op->hasOneUser() &&
wouldInstructionBeTriviallyDead(Op, TLI))
DeadInsts.emplace_back(Op);
I->dropAllReferences();
}
for (auto *I : DeletedInstructions) {
assert(I->use_empty() &&
"trying to erase instruction with users.");
I->eraseFromParent();
}
// Cleanup any dead scalar code feeding the vectorized instructions
RecursivelyDeleteTriviallyDeadInstructions(DeadInsts, TLI);
#ifdef EXPENSIVE_CHECKS
// If we could guarantee that this call is not extremely slow, we could
// remove the ifdef limitation (see PR47712).
assert(!verifyFunction(*F, &dbgs()));
#endif
}
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
bool TopToBottom, bool IgnoreReorder) {
assert(TE.isGather() && "Expected gather node only.");
// Try to find subvector extract/insert patterns and reorder only such
// patterns.
SmallVector<Value *> GatheredScalars(TE.Scalars.begin(), TE.Scalars.end());
Type *ScalarTy = GatheredScalars.front()->getType();
size_t NumScalars = GatheredScalars.size();
if (!isValidElementType(ScalarTy, Opts.slp_revec))
return std::nullopt;
auto *VecTy = getWidenedType(ScalarTy, NumScalars);
unsigned NumParts = getNumberOfParts(VecTy, ScalarTy, NumScalars);
SmallVector<int> ExtractMask;
SmallVector<int> Mask;
SmallVector<SmallVector<const TreeEntry *>> Entries;
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>> ExtractShuffles =
tryToGatherExtractElements(GatheredScalars, ExtractMask, NumParts);
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>> GatherShuffles =
isGatherShuffledEntry(&TE, GatheredScalars, Mask, Entries, NumParts,
ScalarTy, /*ForOrder=*/true);
// The internal gathers of the splat gather subtrees are not used as shuffle
// sources, do not reorder the tree to match them.
if (hasSplatSubtreeGather(Entries)) {
GatherShuffles.clear();
Entries.clear();
}
// No shuffled operands - ignore.
if (GatherShuffles.empty() && ExtractShuffles.empty())
return std::nullopt;
OrdersType CurrentOrder(NumScalars, NumScalars);
if (GatherShuffles.size() == 1 &&
*GatherShuffles.front() == TTI::SK_PermuteSingleSrc &&
Entries.front().front()->isSame(TE.Scalars)) {
// If the full matched node in whole tree rotation - no need to consider the
// matching order, rotating the whole tree.
if (TopToBottom)
return std::nullopt;
// No need to keep the order for the same user node.
if (Entries.front().front()->UserTreeIndex.UserTE ==
TE.UserTreeIndex.UserTE)
return std::nullopt;
// No need to keep the order for the matched root node, if it can be freely
// reordered.
if (!IgnoreReorder && Entries.front().front()->Idx == 0)
return std::nullopt;
// If shuffling 2 elements only and the matching node has reverse reuses -
// no need to count order, both work fine.
if (!Entries.front().front()->ReuseShuffleIndices.empty() &&
TE.getVectorFactor() == 2 && Mask.size() == 2 &&
any_of(enumerate(Entries.front().front()->ReuseShuffleIndices),
[](const auto &P) {
return P.value() % 2 != static_cast<int>(P.index()) % 2;
}))
return std::nullopt;
// Perfect match in the graph, will reuse the previously vectorized
// node. Cost is 0.
std::iota(CurrentOrder.begin(), CurrentOrder.end(), 0);
return CurrentOrder;
}
auto IsSplatMask = [](ArrayRef<int> Mask) {
int SingleElt = PoisonMaskElem;
return all_of(Mask, [&](int I) {
if (SingleElt == PoisonMaskElem && I != PoisonMaskElem)
SingleElt = I;
return I == PoisonMaskElem || I == SingleElt;
});
};
// Exclusive broadcast mask - ignore.
if ((ExtractShuffles.empty() && IsSplatMask(Mask) &&
(Entries.size() != 1 ||
Entries.front().front()->ReorderIndices.empty())) ||
(GatherShuffles.empty() && IsSplatMask(ExtractMask)))
return std::nullopt;
SmallBitVector ShuffledSubMasks(NumParts);
auto TransformMaskToOrder = [&](MutableArrayRef<unsigned> CurrentOrder,
ArrayRef<int> Mask, int PartSz, int NumParts,
function_ref<unsigned(unsigned)> GetVF) {
for (int I : seq<int>(NumParts)) {
if (ShuffledSubMasks.test(I))
continue;
const int VF = GetVF(I);
if (VF == 0)
continue;
unsigned Limit = getNumElems(CurrentOrder.size(), PartSz, I);
MutableArrayRef<unsigned> Slice = CurrentOrder.slice(I * PartSz, Limit);
// Shuffle of at least 2 vectors - ignore.
if (any_of(Slice, not_equal_to(NumScalars))) {
llvm::fill(Slice, NumScalars);
ShuffledSubMasks.set(I);
continue;
}
// Try to include as much elements from the mask as possible.
int FirstMin = INT_MAX;
int SecondVecFound = false;
for (int K : seq<int>(Limit)) {
int Idx = Mask[I * PartSz + K];
if (Idx == PoisonMaskElem) {
Value *V = GatheredScalars[I * PartSz + K];
if (isConstant(V) && !isa<PoisonValue>(V)) {
SecondVecFound = true;
break;
}
continue;
}
if (Idx < VF) {
if (FirstMin > Idx)
FirstMin = Idx;
} else {
SecondVecFound = true;
break;
}
}
FirstMin = (FirstMin / PartSz) * PartSz;
// Shuffle of at least 2 vectors - ignore.
if (SecondVecFound) {
llvm::fill(Slice, NumScalars);
ShuffledSubMasks.set(I);
continue;
}
for (int K : seq<int>(Limit)) {
int Idx = Mask[I * PartSz + K];
if (Idx == PoisonMaskElem)
continue;
Idx -= FirstMin;
if (Idx >= PartSz) {
// Cross-part / second-vector reference: this slice cannot be
// ordered as a single first-vector permutation, give up.
SecondVecFound = true;
break;
}
// For the last partial slice, Limit < PartSz and Idx in [Limit,
// PartSz) addresses the unused padded tail (no scalar at that
// position). Skip the write but keep ordering the remaining K's.
if (static_cast<unsigned>(I * PartSz + Idx) >= CurrentOrder.size())
continue;
if (CurrentOrder[I * PartSz + Idx] >
static_cast<unsigned>(I * PartSz + K) &&
CurrentOrder[I * PartSz + Idx] !=
static_cast<unsigned>(I * PartSz + Idx))
CurrentOrder[I * PartSz + Idx] = I * PartSz + K;
}
// Shuffle of at least 2 vectors - ignore.
if (SecondVecFound) {
llvm::fill(Slice, NumScalars);
ShuffledSubMasks.set(I);
continue;
}
}
};
int PartSz = getPartNumElems(NumScalars, NumParts);
if (!ExtractShuffles.empty())
TransformMaskToOrder(
CurrentOrder, ExtractMask, PartSz, NumParts, [&](unsigned I) {
if (I >= ExtractShuffles.size() || !ExtractShuffles[I])
return 0U;
unsigned VF = 0;
unsigned Sz = getNumElems(TE.getVectorFactor(), PartSz, I);
for (unsigned Idx : seq<unsigned>(Sz)) {
int K = I * PartSz + Idx;
if (static_cast<unsigned>(K) >= ExtractMask.size())
break;
if (ExtractMask[K] == PoisonMaskElem)
continue;
if (!TE.ReuseShuffleIndices.empty())
K = TE.ReuseShuffleIndices[K];
if (K == PoisonMaskElem)
continue;
if (!TE.ReorderIndices.empty())
K = std::distance(TE.ReorderIndices.begin(),
find(TE.ReorderIndices, K));
auto *EI = dyn_cast<ExtractElementInst>(TE.Scalars[K]);
if (!EI)
continue;
VF = std::max(VF, EI->getVectorOperandType()
->getElementCount()
.getKnownMinValue());
}
return VF;
});
// Check special corner case - single shuffle of the same entry.
if (GatherShuffles.size() == 1 && NumParts != 1) {
if (ShuffledSubMasks.any())
return std::nullopt;
PartSz = NumScalars;
NumParts = 1;
}
if (!Entries.empty())
TransformMaskToOrder(CurrentOrder, Mask, PartSz, NumParts, [&](unsigned I) {
if (I >= GatherShuffles.size() || !GatherShuffles[I])
return 0U;
return std::max(Entries[I].front()->getVectorFactor(),
Entries[I].back()->getVectorFactor());
});
unsigned NumUndefs = count(CurrentOrder, NumScalars);
if (ShuffledSubMasks.all() || (NumScalars > 2 && NumUndefs >= NumScalars / 2))
return std::nullopt;
return std::move(CurrentOrder);
}
/// Checks if strided loads can be generated out of \p VL loads with pointers \p
/// PointerOps:
/// 1. Target with strided load support is detected.
/// 2. The number of loads is greater than MinProfitableStridedLoads, or the
/// potential stride <= MaxProfitableStride and the potential stride is
/// power-of-2 (to avoid perf regressions for the very small number of loads)
/// and max distance > number of loads, or potential stride is -1.
/// 3. The loads are ordered, or number of unordered loads <=
/// MaxProfitableUnorderedLoads, or loads are in reversed order. (this check is
/// to avoid extra costs for very expensive shuffles).
/// 4. Any pointer operand is an instruction with the users outside of the
/// current graph (for masked gathers extra extractelement instructions
/// might be required).
bool BoUpSLP::isStridedLoad(ArrayRef<Value *> PointerOps, Type *ScalarTy,
Align Alignment, const int64_t Diff,
const size_t Sz) const {
if (Diff % (Sz - 1) != 0)
return false;
// Try to generate strided load node.
auto IsAnyPointerUsedOutGraph =
any_of(make_isa_range<Instruction>(PointerOps), [&](Instruction *I) {
return any_of(I->users(), [&](User *U) {
return !isVectorized(U) && !MustGather.contains(U);
});
});
const uint64_t AbsoluteDiff = std::abs(Diff);
auto *VecTy = getWidenedType(ScalarTy, Sz);
if (IsAnyPointerUsedOutGraph ||
(AbsoluteDiff > Sz &&
(Sz > Opts.slp_min_strided_loads ||
(AbsoluteDiff <= Opts.slp_max_stride * Sz && AbsoluteDiff % Sz == 0 &&
has_single_bit(AbsoluteDiff / Sz)))) ||
Diff == -(static_cast<int64_t>(Sz) - 1)) {
int64_t Stride = Diff / static_cast<int64_t>(Sz - 1);
if (Diff != Stride * static_cast<int64_t>(Sz - 1))
return false;
if (!TTI->isLegalStridedLoadStore(VecTy, Alignment))
return false;
return true;
}
return false;
}
bool BoUpSLP::isWidenedStridedLoad(const TreeEntry *E) const {
if (E->State != TreeEntry::StridedVectorize ||
E->getOpcode() != Instruction::Load)
return false;
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
return SPtrInfo.Ty->getNumElements() != E->Scalars.size();
}
bool BoUpSLP::analyzeConstantStrideCandidate(
const ArrayRef<Value *> PointerOps, Type *ScalarTy, Align Alignment,
const SmallVectorImpl<unsigned> &SortedIndices, const int64_t Diff,
Value *Ptr0, StridedPtrInfo &SPtrInfo) const {
const size_t Sz = PointerOps.size();
SmallVector<int64_t> SortedOffsetsFromBase(Sz);
// Go through `PointerOps` in sorted order and record offsets from
// PointerOps[0]. We use PointerOps[0] rather than Ptr0 because
// sortPtrAccesses only validates getPointersDiff for pairs relative to
// PointerOps[0]. This is safe since only offset differences are used below.
for (unsigned I : seq<unsigned>(Sz)) {
Value *Ptr =
SortedIndices.empty() ? PointerOps[I] : PointerOps[SortedIndices[I]];
std::optional<int64_t> Offset =
getPointersDiff(ScalarTy, PointerOps[0], ScalarTy, Ptr, *DL, *SE);
assert(Offset && "sortPtrAccesses should have validated this pointer");
SortedOffsetsFromBase[I] = *Offset;
}
// The code below checks that `SortedOffsetsFromBase` looks as follows:
// ```
// [
// (e_{0, 0}, e_{0, 1}, ..., e_{0, GroupSize - 1}), // first group
// (e_{1, 0}, e_{1, 1}, ..., e_{1, GroupSize - 1}), // secon group
// ...
// (e_{NumGroups - 1, 0}, e_{NumGroups - 1, 1}, ..., e_{NumGroups - 1,
// GroupSize - 1}), // last group
// ]
// ```
// The distance between consecutive elements within each group should all be
// the same `StrideWithinGroup`. The distance between the first elements of
// consecutive groups should all be the same `StrideBetweenGroups`.
int64_t StrideWithinGroup =
SortedOffsetsFromBase[1] - SortedOffsetsFromBase[0];
// Determine size of the first group. Later we will check that all other
// groups have the same size.
auto IsEndOfGroupIndex = [=, &SortedOffsetsFromBase](unsigned Idx) {
return SortedOffsetsFromBase[Idx] - SortedOffsetsFromBase[Idx - 1] !=
StrideWithinGroup;
};
auto Indices = seq<unsigned>(1, Sz);
auto FoundIt = llvm::find_if(Indices, IsEndOfGroupIndex);
unsigned GroupSize = FoundIt != Indices.end() ? *FoundIt : Sz;
unsigned VecSz = Sz;
Type *NewScalarTy = ScalarTy;
// Quick detour: at this point we can say what the type of strided load would
// be if all the checks pass. Check if this type is legal for the target.
bool NeedsWidening = Sz != GroupSize;
const uint64_t UnitBitWidth = DL->getTypeSizeInBits(ScalarTy).getFixedValue();
if (NeedsWidening) {
if (Sz % GroupSize != 0)
return false;
if (StrideWithinGroup != 1)
return false;
VecSz = Sz / GroupSize;
NewScalarTy = Type::getIntNTy(SE->getContext(), UnitBitWidth * GroupSize);
} else if (ScalarTy->isVectorTy()) {
NewScalarTy = Type::getIntNTy(SE->getContext(), UnitBitWidth);
}
if (!isStridedLoad(PointerOps, NewScalarTy, Alignment, Diff, VecSz))
return false;
int64_t StrideIntVal = StrideWithinGroup;
if (NeedsWidening) {
// Continue with checking the "shape" of `SortedOffsetsFromBase`.
// Check that the strides between groups are all the same.
unsigned CurrentGroupStartIdx = GroupSize;
int64_t StrideBetweenGroups =
SortedOffsetsFromBase[GroupSize] - SortedOffsetsFromBase[0];
StrideIntVal = StrideBetweenGroups;
for (; CurrentGroupStartIdx < Sz; CurrentGroupStartIdx += GroupSize) {
if (SortedOffsetsFromBase[CurrentGroupStartIdx] -
SortedOffsetsFromBase[CurrentGroupStartIdx - GroupSize] !=
StrideBetweenGroups)
return false;
}
auto CheckGroup = [=](const unsigned StartIdx) -> bool {
auto Indices = seq<unsigned>(StartIdx + 1, Sz);
auto FoundIt = llvm::find_if(Indices, IsEndOfGroupIndex);
unsigned GroupEndIdx = FoundIt != Indices.end() ? *FoundIt : Sz;
return GroupEndIdx - StartIdx == GroupSize;
};
for (unsigned I = 0; I < Sz; I += GroupSize) {
if (!CheckGroup(I))
return false;
}
}
Type *StrideTy = DL->getIndexType(Ptr0->getType());
SPtrInfo.StrideVal = ConstantInt::getSigned(StrideTy, StrideIntVal);
SPtrInfo.Ty = cast<FixedVectorType>(getWidenedType(NewScalarTy, VecSz));
return true;
}
bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef<Value *> PointerOps,
Type *BaseTy, Align CommonAlignment,
SmallVectorImpl<unsigned> &SortedIndices,
StridedPtrInfo &SPtrInfo,
bool IsLoad) const {
const unsigned Sz = PointerOps.size();
const unsigned MinProfitableStridedOps =
IsLoad ? Opts.slp_min_strided_loads : Opts.slp_min_strided_stores;
if (Sz * getNumElements(BaseTy) < MinProfitableStridedOps)
return false;
// The element type of the strided access is fixed by the number of distinct
// constant offsets the pointers fall into, which has to divide their number.
auto GetStridedTy = [&](unsigned NumOffsets) -> FixedVectorType * {
Type *NewScalarTy = BaseTy;
if (NumOffsets > 1 || BaseTy->isVectorTy())
NewScalarTy = Type::getIntNTy(
SE->getContext(),
DL->getTypeSizeInBits(BaseTy).getFixedValue() * NumOffsets);
return dyn_cast<FixedVectorType>(
getWidenedType(NewScalarTy, Sz / NumOffsets));
};
auto IsLegalStridedTy = [&](FixedVectorType *StridedTy) {
return StridedTy && TTI->isTypeLegal(StridedTy) &&
TTI->isLegalStridedLoadStore(StridedTy, CommonAlignment);
};
// Determining the offsets below builds SCEVs for every pointer, which is
// expensive. Bail out if the target does not support any strided accesses at
// all.
if (none_of(seq<unsigned>(1, Sz), [&](unsigned NumOffsets) {
return Sz % NumOffsets == 0 &&
IsLegalStridedTy(GetStridedTy(NumOffsets));
}))
return false;
// If each value in `PointerOps` is of the form `%x + Offset` where `Offset`
// is constant, we partition `PointerOps` sequence into subsequences of
// pointers with the same offset. For each offset we record values from
// `PointerOps` and their indicies in `PointerOps`.
SmallDenseMap<int64_t, std::pair<SmallVector<Value *>, SmallVector<unsigned>>>
OffsetToPointerOpIdxMap;
// Track to make sure that only VecSz different stride multiples are consumed
// Prevents cases such as:
// 1, x + 0, x + 1, 2x + 0 from being recognized as legal RT strided as there
// are 2 "0" and 2 "1" offsets and a stride of "x" between both offsets
SmallDenseSet<const SCEV *> StrideMultiples;
for (auto [Idx, Ptr] : enumerate(PointerOps)) {
const SCEV *PtrSCEV = SE->getSCEV(Ptr);
if (!PtrSCEV)
return false;
const auto *Add = dyn_cast<SCEVAddExpr>(PtrSCEV);
int64_t Offset = 0;
const SCEV *StrideMultiple = PtrSCEV;
if (Add) {
// `Offset` is non-zero.
for (int I : seq<int>(Add->getNumOperands())) {
const auto *SC = dyn_cast<SCEVConstant>(Add->getOperand(I));
if (!SC)
continue;
Offset = SC->getAPInt().getSExtValue();
if (Offset >= std::numeric_limits<int64_t>::max() - 1) {
Offset = 0;
continue;
}
StrideMultiple = SE->getMinusSCEV(StrideMultiple, SC);
break;
}
}
OffsetToPointerOpIdxMap[Offset].first.push_back(Ptr);
OffsetToPointerOpIdxMap[Offset].second.push_back(Idx);
StrideMultiples.insert(StrideMultiple);
}
unsigned NumOffsets = OffsetToPointerOpIdxMap.size();
// Quick detour: at this point we can say what the type of strided load would
// be if all the checks pass. Check if this type is legal for the target.
unsigned VecSz = Sz;
if (NumOffsets > 1) {
if (Sz % NumOffsets != 0)
return false;
VecSz = Sz / NumOffsets;
}
if (StrideMultiples.size() != VecSz)
return false;
auto *StridedLoadTy = GetStridedTy(NumOffsets);
if (!IsLegalStridedTy(StridedLoadTy))
return false;
// Check if the offsets are contiguous and that each group has the required
// size.
SmallVector<int64_t> SortedOffsetsV(NumOffsets);
for (auto [Idx, MapPair] : enumerate(OffsetToPointerOpIdxMap)) {
if (MapPair.second.first.size() != VecSz)
return false;
SortedOffsetsV[Idx] = MapPair.first;
}
sort(SortedOffsetsV);
if (NumOffsets > 1) {
int64_t BaseBytes = DL->getTypeStoreSize(BaseTy);
for (int I : seq<int>(1, SortedOffsetsV.size())) {
if (SortedOffsetsV[I] - SortedOffsetsV[I - 1] != BaseBytes)
return false;
}
}
// Introduce some notation for the explanations below. Let `PointerOps_j`
// denote the subsequence of `PointerOps` with offsets equal to
// `SortedOffsetsV[j]`. Let `SortedIndices_j` be a such that the sequence
// ```
// PointerOps_j[SortedIndices_j[0]],
// PointerOps_j[SortedIndices_j[1]],
// PointerOps_j[SortedIndices_j[2]],
// ...
// ```
// is sorted. Also, let `IndicesInAllPointerOps_j` be the vector
// of indices of the subsequence `PointerOps_j` in all of `PointerOps`,
// i.e `PointerOps_j[i] = PointerOps[IndicesInAllPointerOps_j[i]]`.
// The entire sorted `PointerOps` looks like this:
// ```
// PointerOps_0[SortedIndices_0[0]] = PointerOps[IndicesInAllPointerOps_0[0]],
// PointerOps_1[SortedIndices_1[0]] = PointerOps[IndicesInAllPointerOps_1[0]],
// PointerOps_2[SortedIndices_2[0]] = PointerOps[IndicesInAllPointerOps_2[0]],
// ...
// PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[0]] =
// PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[0]],
//
// PointerOps_0[SortedIndices_0[1]] = PointerOps[IndicesInAllPointerOps_0[1]],
// PointerOps_1[SortedIndices_1[1]] = PointerOps[IndicesInAllPointerOps_1[1]],
// PointerOps_2[SortedIndices_2[1]] = PointerOps[IndicesInAllPointerOps_2[1]],
// ...
// PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[1]] =
// PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[1]],
//
// PointerOps_0[SortedIndices_0[2]] = PointerOps[IndicesInAllPointerOps_0[2]],
// PointerOps_1[SortedIndices_1[2]] = PointerOps[IndicesInAllPointerOps_1[2]],
// PointerOps_2[SortedIndices_2[2]] = PointerOps[IndicesInAllPointerOps_2[2]],
// ...
// PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[2]] =
// PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[2]],
// ...
// ...
// ...
// PointerOps_0[SortedIndices_0[VecSz - 1]] =
// PointerOps[IndicesInAllPointerOps_0[VecSz - 1]],
// PointerOps_1[SortedIndices_1[VecSz - 1]] =
// PointerOps[IndicesInAllPointerOps_1[VecSz - 1]],
// PointerOps_2[SortedIndices_2[VecSz - 1]] =
// PointerOps[IndicesInAllPointerOps_2[VecSz - 1]],
// ...
// PointerOps_(NumOffsets - 1)[SortedIndices_(NumOffsets - 1)[VecSz - 1]] =
// PointerOps[IndicesInAllPointerOps_(NumOffsets - 1)[VecSz - 1]],
// ```
// In order to be able to generate a strided load, for each `PointerOps_j`
// check that the distance between adjacent pointers are all equal to the same
// value (stride).
//
// As we do that, also calculate SortedIndices. Since we should not modify
// `SortedIndices` unless we know that all the checks succeed, record the
// indicies into `SortedIndicesDraft`.
SmallVector<unsigned> SortedIndicesDraft(Sz);
// Given sorted indices for a particular offset (as calculated by
// calculateRtStride), update the `SortedIndicesDraft` for all of PointerOps.
// Let `Offset` be `SortedOffsetsV[OffsetNum]`.
// \param `OffsetNum` the index of `Offset` in `SortedOffsetsV`.
// \param `IndicesInAllPointerOps` vector of indices of the
// subsequence `PointerOps_OffsetNum` in `PointerOps`, i.e. using the above
// notation `IndicesInAllPointerOps = IndicesInAllPointerOps_OffsetNum`.
// \param `SortedIndicesForOffset = SortedIndices_OffsetNum`
auto UpdateSortedIndices =
[&](SmallVectorImpl<unsigned> &SortedIndicesForOffset,
ArrayRef<unsigned> IndicesInAllPointerOps, const int64_t OffsetNum) {
if (SortedIndicesForOffset.empty()) {
SortedIndicesForOffset.resize(IndicesInAllPointerOps.size());
std::iota(SortedIndicesForOffset.begin(),
SortedIndicesForOffset.end(), 0);
}
for (const auto [Num, Idx] : enumerate(SortedIndicesForOffset)) {
SortedIndicesDraft[Num * NumOffsets + OffsetNum] =
IndicesInAllPointerOps[Idx];
}
};
int64_t LowestOffset = SortedOffsetsV[0];
ArrayRef<Value *> PointerOps0 = OffsetToPointerOpIdxMap[LowestOffset].first;
SmallVector<unsigned> SortedIndicesForOffset0;
const SCEV *Stride0 =
calculateRtStride(PointerOps0, BaseTy, *DL, *SE, SortedIndicesForOffset0);
if (!Stride0)
return false;
ArrayRef<unsigned> IndicesInAllPointerOps0 =
OffsetToPointerOpIdxMap[LowestOffset].second;
UpdateSortedIndices(SortedIndicesForOffset0, IndicesInAllPointerOps0, 0);
// Now that we know what the common stride and coefficients has to be check
// the remaining `PointerOps_j`.
SmallVector<unsigned> SortedIndicesForOffset;
for (int J : seq<int>(1, NumOffsets)) {
SortedIndicesForOffset.clear();
int64_t Offset = SortedOffsetsV[J];
ArrayRef<Value *> PointerOpsForOffset =
OffsetToPointerOpIdxMap[Offset].first;
ArrayRef<unsigned> IndicesInAllPointerOps =
OffsetToPointerOpIdxMap[Offset].second;
const SCEV *StrideWithinGroup = calculateRtStride(
PointerOpsForOffset, BaseTy, *DL, *SE, SortedIndicesForOffset);
if (!StrideWithinGroup || StrideWithinGroup != Stride0)
return false;
UpdateSortedIndices(SortedIndicesForOffset, IndicesInAllPointerOps, J);
}
SortedIndices.clear();
SortedIndices = std::move(SortedIndicesDraft);
SPtrInfo.StrideSCEV = Stride0;
SPtrInfo.Ty = StridedLoadTy;
return true;
}
BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo,
unsigned *BestVF, bool TryRecursiveCheck) const {
// Check that a vectorized load would load the same memory as a scalar
// load. For example, we don't want to vectorize loads that are smaller
// than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
// treats loading/storing it as an i8 struct. If we vectorize loads/stores
// from such a struct, we read/write packed bits disagreeing with the
// unvectorized version.
if (BestVF)
*BestVF = 0;
if (areKnownNonVectorizableLoads(VL))
return LoadsState::Gather;
Type *ScalarTy = VL0->getType();
if (DL->getTypeSizeInBits(ScalarTy) != DL->getTypeAllocSizeInBits(ScalarTy))
return LoadsState::Gather;
// The undef and poison lanes are covered by a single wide load of the
// consecutive loads.
if (any_of(VL, IsaPred<UndefValue>))
return canWidenLoadsOverUndefLanes(VL, *DL, *SE, *AC, *DT, *TLI)
? LoadsState::Vectorize
: LoadsState::Gather;
// Make sure all loads in the bundle are simple - we can't vectorize
// atomic or volatile loads.
PointerOps.clear();
const size_t Sz = VL.size();
PointerOps.resize(Sz);
auto *POIter = PointerOps.begin();
for (Value *V : VL) {
auto *L = dyn_cast<LoadInst>(V);
if (!L || !L->isSimple())
return LoadsState::Gather;
*POIter = L->getPointerOperand();
++POIter;
}
Order.clear();
// Check the order of pointer operands or that all pointers are the same.
bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, Order);
auto *VecTy = dyn_cast<VectorType>(getWidenedType(ScalarTy, Sz));
if (!VecTy)
return LoadsState::Gather;
Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
// Cache masked gather legality - both the !IsSorted path below and the
// post-branch check use the same VecTy/CommonAlignment, and the underlying
// TTI calls are virtual.
std::optional<bool> MaskedGatherLegal;
auto IsMaskedGatherLegal = [&, TTI = TTI] {
if (!MaskedGatherLegal)
MaskedGatherLegal =
TTI->isLegalMaskedGather(VecTy, CommonAlignment) &&
!TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment);
return *MaskedGatherLegal;
};
if (!IsSorted) {
// Check for a group of loads, each selecting its address (directly, or
// via a constant-offset GEP) between the same two candidate base
// pointers - the shape if-converted, fully-unrolled loop bodies of the
// form `x = cond ? A[i] : B[i]` take. If found, model it as two masked
// loads (one per candidate) blended by the (vectorized) condition,
// rather than falling back to a gather of the individual scalar loads.
Value *TrueBase = nullptr;
Value *FalseBase = nullptr;
SmallVector<Value *> Conditions;
if (isSelectedBaseLoad(ScalarTy, PointerOps, *DL, TrueBase, FalseBase,
Conditions) &&
TTI->isLegalMaskedLoad(VecTy, computeBlendedLoadBaseAlignment(VL, *DL),
cast<LoadInst>(VL0)->getPointerAddressSpace()))
return LoadsState::BlendedLoadVectorize;
if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
SPtrInfo, /*isLoad=*/true))
return LoadsState::StridedVectorize;
if (!IsMaskedGatherLegal())
return LoadsState::Gather;
// The gather address vector requires pointers of the same type.
Value *FrontPtr = PointerOps.front();
Type *PtrTy = FrontPtr->getType();
if (!all_of(PointerOps,
[&](Value *P) {
return P->getType() == PtrTy &&
arePointersCompatible(P, FrontPtr, *TLI,
Opts.slp_recursion_max_depth);
}) &&
!isCopyableGEPAddressVector(PointerOps))
return LoadsState::Gather;
} else {
Value *Ptr0;
Value *PtrN;
if (Order.empty()) {
Ptr0 = PointerOps.front();
PtrN = PointerOps.back();
} else {
Ptr0 = PointerOps[Order.front()];
PtrN = PointerOps[Order.back()];
}
// sortPtrAccesses validates getPointersDiff for all pointers relative to
// PointerOps[0], so compute the span using PointerOps[0] as intermediate:
// Diff = offset(PtrN) - offset(Ptr0) relative to PointerOps[0]
std::optional<int64_t> Diff0 =
getPointersDiff(ScalarTy, PointerOps[0], ScalarTy, Ptr0, *DL, *SE);
std::optional<int64_t> DiffN =
getPointersDiff(ScalarTy, PointerOps[0], ScalarTy, PtrN, *DL, *SE);
assert(Diff0 && DiffN &&
"sortPtrAccesses should have validated these pointers");
int64_t Diff = *DiffN - *Diff0;
// Check that the sorted loads are consecutive.
if (static_cast<uint64_t>(Diff) == Sz - 1)
return LoadsState::Vectorize;
if (isMaskedLoadCompress(
VL, PointerOps, Order, *TTI, *DL, *SE, *AC, *DT, *TLI, CostKind,
[&](Value *V) {
return areAllUsersVectorized(cast<Instruction>(V),
UserIgnoreList);
},
Opts.slp_revec, Cache))
return LoadsState::CompressVectorize;
// Widened strided loads must be legal for every group, not just the first
// pointer, which may have a stronger alignment than the remaining loads.
if (analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
Order, Diff, Ptr0, SPtrInfo))
return LoadsState::StridedVectorize;
}
if (!IsMaskedGatherLegal())
return LoadsState::Gather;
// Correctly identify compare the cost of loads + shuffles rather than
// strided/masked gather loads. Returns true if vectorized + shuffles
// representation is better than just gather.
auto CheckForShuffledLoads = [&, &TTI = *TTI](Align CommonAlignment,
unsigned *BestVF,
bool ProfitableGatherPointers) {
if (BestVF)
*BestVF = 0;
// Compare masked gather cost and loads + insert subvector costs.
auto [ScalarGEPCost, VectorGEPCost] =
getGEPCosts(TTI, PointerOps, PointerOps.front(), Instruction::Load,
CostKind, ScalarTy, VecTy);
// Estimate the cost of masked gather GEP. If not a splat, roughly
// estimate as a buildvector, otherwise estimate as splat.
APInt DemandedElts = APInt::getAllOnes(Sz);
Type *PtrScalarTy = PointerOps.front()->getType()->getScalarType();
auto *PtrVecTy = cast<VectorType>(getWidenedType(PtrScalarTy, Sz));
// Cache the underlying object of PointerOps.front() - it is invariant
// across the per-V comparisons below and getUnderlyingObject walks
// GEP/cast chains.
const Value *FrontUO = getUnderlyingObject(PointerOps.front());
if (static_cast<unsigned>(count_if(
PointerOps, IsaPred<GetElementPtrInst>)) < PointerOps.size() - 1 ||
any_of(PointerOps,
[&](Value *V) { return getUnderlyingObject(V) != FrontUO; }))
VectorGEPCost +=
getScalarizationOverhead(TTI, Opts.slp_revec, PtrScalarTy, PtrVecTy,
DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
else
VectorGEPCost +=
getScalarizationOverhead(TTI, Opts.slp_revec, PtrScalarTy, PtrVecTy,
APInt::getOneBitSet(Sz, 0),
/*Insert=*/true, /*Extract=*/false,
CostKind) +
getShuffleCost(TTI, TTI::SK_Broadcast, PtrVecTy, CostKind);
// The cost of scalar loads.
InstructionCost ScalarLoadsCost =
accumulate(VL, InstructionCost(),
[&](InstructionCost C, Value *V) {
return C + TTI.getInstructionCost(cast<Instruction>(V),
CostKind);
}) +
ScalarGEPCost;
// The cost of masked gather.
InstructionCost MaskedGatherCost =
TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_gather, VecTy,
cast<LoadInst>(VL0)->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind) +
(ProfitableGatherPointers ? 0 : VectorGEPCost);
InstructionCost GatherCost =
getScalarizationOverhead(TTI, Opts.slp_revec, ScalarTy, VecTy,
DemandedElts,
/*Insert=*/true,
/*Extract=*/false, CostKind) +
ScalarLoadsCost;
// The list of loads is small or perform partial check already - directly
// compare masked gather cost and gather cost.
constexpr unsigned ListLimit = 4;
if (!TryRecursiveCheck || VL.size() < ListLimit)
return MaskedGatherCost - GatherCost >= -getCostThreshold();
unsigned Sz = DL->getTypeSizeInBits(ScalarTy);
unsigned MinVF = getMinVF(2 * Sz);
DemandedElts.clearAllBits();
// Iterate through possible vectorization factors and check if vectorized +
// shuffles is better than just gather.
for (unsigned VF = getFloorFullVectorNumberOfElements(
TTI, ScalarTy, VL.size() - 1, Opts.slp_revec);
VF >= MinVF; VF = getFloorFullVectorNumberOfElements(
TTI, ScalarTy, VF - 1, Opts.slp_revec)) {
SmallVector<std::pair<unsigned, LoadsState>> States;
for (unsigned Cnt = 0, End = VL.size(); Cnt < End; Cnt += VF) {
const unsigned SliceVF = std::min(VF, End - Cnt);
ArrayRef<Value *> Slice = VL.slice(Cnt, SliceVF);
SmallVector<unsigned> Order;
SmallVector<Value *> PointerOps;
LoadsState LS = canVectorizeLoads(Slice, Slice.front(), Order,
PointerOps, SPtrInfo, BestVF,
/*TryRecursiveCheck=*/false);
// Check that the sorted loads are consecutive.
if (LS == LoadsState::Gather) {
if (BestVF) {
DemandedElts.setAllBits();
break;
}
DemandedElts.setBits(Cnt, Cnt + SliceVF);
continue;
}
// If need the reorder - consider as high-cost masked gather for now.
if ((LS == LoadsState::Vectorize ||
LS == LoadsState::StridedVectorize ||
LS == LoadsState::CompressVectorize) &&
!Order.empty() && !isReverseOrder(Order))
LS = LoadsState::ScatterVectorize;
States.emplace_back(Cnt, LS);
}
if (DemandedElts.isAllOnes())
// All loads gathered - try smaller VF.
continue;
// Can be vectorized later as a serie of loads/insertelements.
InstructionCost VecLdCost = 0;
if (!DemandedElts.isZero()) {
VecLdCost = getScalarizationOverhead(TTI, Opts.slp_revec, ScalarTy,
VecTy, DemandedElts,
/*Insert=*/true,
/*Extract=*/false, CostKind) +
ScalarGEPCost;
for (unsigned Idx : seq<unsigned>(VL.size()))
if (DemandedElts[Idx])
VecLdCost +=
TTI.getInstructionCost(cast<Instruction>(VL[Idx]), CostKind);
}
for (const auto &[SliceStart, LS] : States) {
const unsigned SliceVF = std::min<unsigned>(VF, VL.size() - SliceStart);
auto *SubVecTy = cast<VectorType>(getWidenedType(ScalarTy, SliceVF));
auto *LI0 = cast<LoadInst>(VL[SliceStart]);
InstructionCost VectorGEPCost =
(LS == LoadsState::ScatterVectorize && ProfitableGatherPointers)
? 0
: getGEPCosts(TTI,
ArrayRef(PointerOps).slice(SliceStart, SliceVF),
LI0->getPointerOperand(), Instruction::Load,
CostKind, ScalarTy, SubVecTy)
.second;
if (LS == LoadsState::ScatterVectorize) {
if (static_cast<unsigned>(
count_if(PointerOps, IsaPred<GetElementPtrInst>)) <
PointerOps.size() - 1 ||
any_of(PointerOps, [&](Value *V) {
return getUnderlyingObject(V) != FrontUO;
}))
VectorGEPCost += getScalarizationOverhead(
TTI, Opts.slp_revec, ScalarTy, SubVecTy,
APInt::getAllOnes(SliceVF),
/*Insert=*/true, /*Extract=*/false, CostKind);
else
VectorGEPCost +=
getScalarizationOverhead(
TTI, Opts.slp_revec, ScalarTy, SubVecTy,
APInt::getOneBitSet(SliceVF, 0),
/*Insert=*/true, /*Extract=*/false, CostKind) +
getShuffleCost(TTI, TTI::SK_Broadcast, SubVecTy, CostKind);
}
switch (LS) {
case LoadsState::Vectorize:
VecLdCost += TTI.getMemoryOpCost(
Instruction::Load, SubVecTy, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI0->getPointerOperand())) +
VectorGEPCost;
break;
case LoadsState::StridedVectorize:
VecLdCost += TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::experimental_vp_strided_load,
SubVecTy, LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind) +
VectorGEPCost;
break;
case LoadsState::CompressVectorize:
VecLdCost +=
TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_load, SubVecTy,
CommonAlignment,
LI0->getPointerAddressSpace()),
CostKind) +
getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy, CostKind);
break;
case LoadsState::ScatterVectorize:
VecLdCost += TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::masked_gather, SubVecTy,
LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind) +
VectorGEPCost;
break;
case LoadsState::BlendedLoadVectorize:
// Two masked loads (one per candidate base) plus a select; no address
// vector is materialized, so VectorGEPCost is skipped.
VecLdCost +=
getBlendedLoadCost(TTI, SubVecTy,
computeBlendedLoadBaseAlignment(
VL.slice(SliceStart, SliceVF), *DL),
LI0->getPointerAddressSpace(), CostKind);
break;
case LoadsState::Gather:
llvm_unreachable("Gathers are not added to States");
}
SmallVector<int> ShuffleMask(VL.size());
const unsigned SliceIdx = SliceStart / VF;
for (int Idx : seq<int>(VL.size()))
ShuffleMask[Idx] = Idx / VF == SliceIdx ? VL.size() + Idx % VF : Idx;
if (SliceStart > 0)
VecLdCost +=
getShuffleCost(TTI, TTI::SK_InsertSubvector, VecTy, CostKind,
ShuffleMask, SliceStart, SubVecTy);
}
// If masked gather cost is higher - better to vectorize, so
// consider it as a gather node. It will be better estimated
// later.
if (MaskedGatherCost >= VecLdCost &&
VecLdCost - GatherCost < -getCostThreshold()) {
if (BestVF)
*BestVF = VF;
return true;
}
}
return MaskedGatherCost - GatherCost >= -getCostThreshold();
};
// TODO: need to improve analysis of the pointers, if not all of them are
// GEPs or have > 2 operands, we end up with a gather node, which just
// increases the cost.
Loop *L = LI->getLoopFor(cast<LoadInst>(VL0)->getParent());
bool ProfitableGatherPointers =
L && Sz > 2 && static_cast<unsigned>(count_if(PointerOps, [L](Value *V) {
return L->isLoopInvariant(V);
})) <= Sz / 2;
if (ProfitableGatherPointers || all_of(PointerOps, [](Value *P) {
auto *GEP = dyn_cast<GetElementPtrInst>(P);
return (!GEP && doesNotNeedToBeScheduled(P)) ||
(GEP && GEP->getNumOperands() == 2 &&
isa<Constant, Instruction>(GEP->getOperand(1)));
})) {
// Check if potential masked gather can be represented as series
// of loads + insertsubvectors.
// If masked gather cost is higher - better to vectorize, so
// consider it as a gather node. It will be better estimated
// later.
if (!TryRecursiveCheck || !CheckForShuffledLoads(CommonAlignment, BestVF,
ProfitableGatherPointers))
return LoadsState::ScatterVectorize;
}
return LoadsState::Gather;
}
std::optional<BoUpSLP::OrdersType>
BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
assert(TE.isGather() && "Expected gather node only.");
Type *ScalarTy = TE.Scalars[0]->getType();
SmallVector<Value *> Ptrs;
Ptrs.reserve(TE.Scalars.size());
SmallVector<BasicBlock *> BBs;
BBs.reserve(TE.Scalars.size());
for (Value *V : TE.Scalars) {
auto *L = dyn_cast<LoadInst>(V);
if (!L || !L->isSimple())
return std::nullopt;
Ptrs.push_back(L->getPointerOperand());
BBs.push_back(L->getParent());
}
BoUpSLP::OrdersType Order;
if (!LoadEntriesToVectorize.contains(TE.Idx) &&
clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE,
Opts.slp_recursion_max_depth, Order, Cache))
return std::move(Order);
return std::nullopt;
}
/// Check if two insertelement instructions are from the same buildvector.
static bool areTwoInsertFromSameBuildVector(
InsertElementInst *VU, InsertElementInst *V,
function_ref<Value *(InsertElementInst *)> GetBaseOperand) {
// Instructions must be from the same basic blocks.
if (VU->getParent() != V->getParent())
return false;
// Checks if 2 insertelements are from the same buildvector.
if (VU->getType() != V->getType())
return false;
// Multiple used inserts are separate nodes.
if (!VU->hasOneUse() && !V->hasOneUse())
return false;
auto *IE1 = VU;
auto *IE2 = V;
std::optional<unsigned> Idx1 = getElementIndex(IE1);
std::optional<unsigned> Idx2 = getElementIndex(IE2);
if (Idx1 == std::nullopt || Idx2 == std::nullopt)
return false;
// Go through the vector operand of insertelement instructions trying to find
// either VU as the original vector for IE2 or V as the original vector for
// IE1.
SmallBitVector ReusedIdx(VU->getType()->getElementCount().getKnownMinValue());
bool IsReusedIdx = false;
do {
if (IE2 == VU && !IE1)
return VU->hasOneUse();
if (IE1 == V && !IE2)
return V->hasOneUse();
if (IE1 && IE1 != V) {
unsigned Idx1 = getElementIndex(IE1).value_or(*Idx2);
IsReusedIdx |= ReusedIdx.test(Idx1);
ReusedIdx.set(Idx1);
if ((IE1 != VU && !IE1->hasOneUse()) || IsReusedIdx)
IE1 = nullptr;
else
IE1 = dyn_cast_if_present<InsertElementInst>(GetBaseOperand(IE1));
}
if (IE2 && IE2 != VU) {
unsigned Idx2 = getElementIndex(IE2).value_or(*Idx1);
IsReusedIdx |= ReusedIdx.test(Idx2);
ReusedIdx.set(Idx2);
if ((IE2 != V && !IE2->hasOneUse()) || IsReusedIdx)
IE2 = nullptr;
else
IE2 = dyn_cast_if_present<InsertElementInst>(GetBaseOperand(IE2));
}
} while (!IsReusedIdx && (IE1 || IE2));
return false;
}
std::optional<BoUpSLP::OrdersType>
BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
bool IgnoreReorder) {
// No need to reorder if need to shuffle reuses, still need to shuffle the
// node.
if (!TE.ReuseShuffleIndices.empty()) {
if (isSplat(TE.Scalars))
return std::nullopt;
// Check if reuse shuffle indices can be improved by reordering.
// For this, check that reuse mask is "clustered", i.e. each scalar values
// is used once in each submask of size <number_of_scalars>.
// Example: 4 scalar values.
// ReuseShuffleIndices mask: 0, 1, 2, 3, 3, 2, 0, 1 - clustered.
// 0, 1, 2, 3, 3, 3, 1, 0 - not clustered, because
// element 3 is used twice in the second submask.
unsigned Sz = TE.Scalars.size();
if (TE.isGather()) {
if (std::optional<OrdersType> CurrentOrder =
findReusedOrderedScalars(TE, TopToBottom, IgnoreReorder)) {
SmallVector<int> Mask;
fixupOrderingIndices(*CurrentOrder);
inversePermutation(*CurrentOrder, Mask);
addMask(Mask, TE.ReuseShuffleIndices);
OrdersType Res(TE.getVectorFactor(), TE.getVectorFactor());
unsigned Sz = TE.Scalars.size();
for (int K = 0, E = TE.getVectorFactor() / Sz; K < E; ++K) {
for (auto [I, Idx] : enumerate(ArrayRef(Mask).slice(K * Sz, Sz)))
if (Idx != PoisonMaskElem)
Res[Idx + K * Sz] = I + K * Sz;
}
return std::move(Res);
}
}
if (Sz == 2 && TE.getVectorFactor() == 4 &&
getNumberOfParts(
getWidenedType(getValueType(TE.Scalars.front(), Opts.slp_revec),
2 * TE.getVectorFactor()),
getValueType(TE.Scalars.front(), Opts.slp_revec)) == 1)
return std::nullopt;
if (TE.ReuseShuffleIndices.size() % Sz != 0)
return std::nullopt;
if (!ShuffleVectorInst::isOneUseSingleSourceMask(TE.ReuseShuffleIndices,
Sz)) {
SmallVector<int> ReorderMask(Sz, PoisonMaskElem);
if (TE.ReorderIndices.empty())
std::iota(ReorderMask.begin(), ReorderMask.end(), 0);
else
inversePermutation(TE.ReorderIndices, ReorderMask);
addMask(ReorderMask, TE.ReuseShuffleIndices);
unsigned VF = ReorderMask.size();
OrdersType ResOrder(VF, VF);
unsigned NumParts = divideCeil(VF, Sz);
SmallBitVector UsedVals(NumParts);
for (unsigned I = 0; I < VF; I += Sz) {
int Val = PoisonMaskElem;
unsigned UndefCnt = 0;
unsigned Limit = std::min(Sz, VF - I);
if (any_of(ArrayRef(ReorderMask).slice(I, Limit),
[&](int Idx) {
if (Val == PoisonMaskElem && Idx != PoisonMaskElem)
Val = Idx;
if (Idx == PoisonMaskElem)
++UndefCnt;
return Idx != PoisonMaskElem && Idx != Val;
}) ||
Val >= static_cast<int>(NumParts) || Val == PoisonMaskElem ||
UsedVals.test(Val) || UndefCnt > Sz / 2)
return std::nullopt;
UsedVals.set(Val);
for (unsigned K = 0; K < NumParts; ++K) {
unsigned Idx = Val + Sz * K;
if (Idx < VF && I + K < VF)
ResOrder[Idx] = I + K;
}
}
return std::move(ResOrder);
}
unsigned VF = TE.getVectorFactor();
// Try build correct order for extractelement instructions.
SmallVector<int> ReusedMask(TE.ReuseShuffleIndices.begin(),
TE.ReuseShuffleIndices.end());
if (TE.hasState() && TE.getOpcode() == Instruction::ExtractElement &&
!TE.hasCopyableElements() &&
all_of(make_isa_range<Instruction>(TE.Scalars), [Sz](Instruction *I) {
std::optional<unsigned> Idx = getExtractIndex(I);
return Idx && *Idx < Sz;
})) {
assert(!TE.isAltShuffle() && "Alternate instructions are only supported "
"by BinaryOperator and CastInst.");
SmallVector<int> ReorderMask(Sz, PoisonMaskElem);
if (TE.ReorderIndices.empty())
std::iota(ReorderMask.begin(), ReorderMask.end(), 0);
else
inversePermutation(TE.ReorderIndices, ReorderMask);
for (unsigned I = 0; I < VF; ++I) {
int &Idx = ReusedMask[I];
if (Idx == PoisonMaskElem)
continue;
Value *V = TE.Scalars[ReorderMask[Idx]];
std::optional<unsigned> EI = getExtractIndex(cast<Instruction>(V));
Idx = std::distance(ReorderMask.begin(), find(ReorderMask, *EI));
}
}
// Build the order of the VF size, need to reorder reuses shuffles, they are
// always of VF size.
OrdersType ResOrder(VF);
std::iota(ResOrder.begin(), ResOrder.end(), 0);
auto *It = ResOrder.begin();
for (unsigned K = 0; K < VF; K += Sz) {
OrdersType CurrentOrder(TE.ReorderIndices);
SmallVector<int> SubMask{ArrayRef(ReusedMask).slice(K, Sz)};
if (SubMask.front() == PoisonMaskElem)
std::iota(SubMask.begin(), SubMask.end(), 0);
reorderOrder(CurrentOrder, SubMask);
transform(CurrentOrder, It, [K](unsigned Pos) { return Pos + K; });
std::advance(It, Sz);
}
if (TE.isGather() && all_of(enumerate(ResOrder), [](const auto &Data) {
return Data.index() == Data.value();
}))
return std::nullopt; // No need to reorder.
return std::move(ResOrder);
}
if (TE.State == TreeEntry::StridedVectorize && !TopToBottom &&
(!TE.UserTreeIndex || !TE.UserTreeIndex.UserTE->hasState() ||
!Instruction::isBinaryOp(TE.UserTreeIndex.UserTE->getOpcode())) &&
(TE.ReorderIndices.empty() || TE.isReverse()))
return std::nullopt;
if (TE.State == TreeEntry::SplitVectorize ||
((TE.State == TreeEntry::Vectorize ||
TE.State == TreeEntry::StridedVectorize ||
TE.State == TreeEntry::ExpandVectorize ||
TE.State == TreeEntry::CompressVectorize ||
TE.State == TreeEntry::BlendedLoadVectorize) &&
(isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
(TopToBottom && isa<StoreInst, InsertElementInst, InsertValueInst>(
TE.getMainOp()))))) {
assert((TE.State == TreeEntry::SplitVectorize || !TE.isAltShuffle()) &&
"Alternate instructions are only supported by "
"BinaryOperator and CastInst.");
return TE.ReorderIndices;
}
if (!TopToBottom && IgnoreReorder && TE.State == TreeEntry::Vectorize &&
TE.isAltShuffle()) {
assert(TE.ReuseShuffleIndices.empty() &&
"ReuseShuffleIndices should be "
"empty for alternate instructions.");
SmallVector<int> Mask;
TE.buildAltOpShuffleMask(
[&](Instruction *I) {
assert(TE.getMatchingMainOpOrAltOp(I) &&
"Unexpected main/alternate opcode");
return isAlternateInstruction(I, TE.getMainOp(), TE.getAltOp(), *TLI);
},
Mask);
const int VF = TE.getVectorFactor();
OrdersType ResOrder(VF, VF);
for (unsigned I : seq<unsigned>(VF)) {
if (Mask[I] == PoisonMaskElem)
continue;
ResOrder[Mask[I] % VF] = I;
}
return std::move(ResOrder);
}
if (!TE.ReorderIndices.empty())
return TE.ReorderIndices;
if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::PHI) {
if (!TE.ReorderIndices.empty())
return TE.ReorderIndices;
SmallVector<Instruction *> UserBVHead(TE.Scalars.size());
for (auto [I, V] :
make_filter_range(zip(UserBVHead, TE.Scalars), [](const auto &P) {
Value *V = std::get<1>(P);
return !isa<Constant>(V) && V->hasNUsesOrMore(1);
})) {
auto *II = dyn_cast<InsertElementInst>(*V->user_begin());
if (!II)
continue;
Instruction *BVHead = nullptr;
BasicBlock *BB = II->getParent();
while (II && II->hasOneUse() && II->getParent() == BB) {
BVHead = II;
II = dyn_cast<InsertElementInst>(II->getOperand(0));
}
I = BVHead;
}
auto CompareByBasicBlocks = [&](BasicBlock *BB1, BasicBlock *BB2) {
assert(BB1 != BB2 && "Expected different basic blocks.");
if (!DT->isReachableFromEntry(BB1))
return false;
if (!DT->isReachableFromEntry(BB2))
return true;
auto *NodeA = DT->getNode(BB1);
auto *NodeB = DT->getNode(BB2);
assert(NodeA && "Should only process reachable instructions");
assert(NodeB && "Should only process reachable instructions");
assert((NodeA == NodeB) ==
(NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
return NodeA->getDFSNumIn() < NodeB->getDFSNumIn();
};
auto PHICompare = [&](unsigned I1, unsigned I2) {
Value *V1 = TE.Scalars[I1];
Value *V2 = TE.Scalars[I2];
if (V1 == V2 || (V1->use_empty() && V2->use_empty()))
return false;
if (isa<PoisonValue>(V1))
return true;
if (isa<PoisonValue>(V2))
return false;
if (V1->getNumUses() < V2->getNumUses())
return true;
if (V1->getNumUses() > V2->getNumUses())
return false;
auto *FirstUserOfPhi1 = cast<Instruction>(*V1->user_begin());
auto *FirstUserOfPhi2 = cast<Instruction>(*V2->user_begin());
if (FirstUserOfPhi1->getParent() != FirstUserOfPhi2->getParent())
return CompareByBasicBlocks(FirstUserOfPhi1->getParent(),
FirstUserOfPhi2->getParent());
auto *IE1 = dyn_cast<InsertElementInst>(FirstUserOfPhi1);
auto *IE2 = dyn_cast<InsertElementInst>(FirstUserOfPhi2);
auto *EE1 = dyn_cast<ExtractElementInst>(FirstUserOfPhi1);
auto *EE2 = dyn_cast<ExtractElementInst>(FirstUserOfPhi2);
if (IE1 && !IE2)
return true;
if (!IE1 && IE2)
return false;
if (IE1 && IE2) {
if (UserBVHead[I1] && !UserBVHead[I2])
return true;
if (!UserBVHead[I1])
return false;
if (UserBVHead[I1] == UserBVHead[I2])
return getElementIndex(IE1) < getElementIndex(IE2);
if (UserBVHead[I1]->getParent() != UserBVHead[I2]->getParent())
return CompareByBasicBlocks(UserBVHead[I1]->getParent(),
UserBVHead[I2]->getParent());
return UserBVHead[I1]->comesBefore(UserBVHead[I2]);
}
if (EE1 && !EE2)
return true;
if (!EE1 && EE2)
return false;
if (EE1 && EE2) {
auto *Inst1 = dyn_cast<Instruction>(EE1->getOperand(0));
auto *Inst2 = dyn_cast<Instruction>(EE2->getOperand(0));
auto *P1 = dyn_cast<Argument>(EE1->getOperand(0));
auto *P2 = dyn_cast<Argument>(EE2->getOperand(0));
if (!Inst2 && !P2)
return Inst1 || P1;
if (EE1->getOperand(0) == EE2->getOperand(0))
return getElementIndex(EE1) < getElementIndex(EE2);
if (!Inst1 && Inst2)
return false;
if (Inst1 && Inst2) {
if (Inst1->getParent() != Inst2->getParent())
return CompareByBasicBlocks(Inst1->getParent(), Inst2->getParent());
return Inst1->comesBefore(Inst2);
}
if (!P1 && P2)
return false;
assert(P1 && P2 &&
"Expected either instructions or arguments vector operands.");
return P1->getArgNo() < P2->getArgNo();
}
return false;
};
OrdersType Phis(TE.Scalars.size());
std::iota(Phis.begin(), Phis.end(), 0);
stable_sort(Phis, PHICompare);
if (isIdentityOrder(Phis))
return std::nullopt; // No need to reorder.
return std::move(Phis);
}
if (TE.isGather() &&
(!TE.hasState() || !TE.isAltShuffle() ||
ScalarsInSplitNodes.contains(TE.getMainOp())) &&
allSameType(TE.Scalars)) {
// TODO: add analysis of other gather nodes with extractelement
// instructions and other values/instructions, not only undefs.
// Nodes with copyable lanes may mix in non-extract lanes, for which the
// extract-index order is not applicable.
if (((TE.hasState() && TE.getOpcode() == Instruction::ExtractElement &&
!TE.hasCopyableElements()) ||
(all_of(TE.Scalars, IsaPred<UndefValue, ExtractElementInst>) &&
any_of(TE.Scalars, IsaPred<ExtractElementInst>))) &&
all_of(make_isa_range<ExtractElementInst>(TE.Scalars),
[](ExtractElementInst *EE) {
return isa<FixedVectorType>(EE->getVectorOperandType());
})) {
// Check that gather of extractelements can be represented as
// just a shuffle of a single vector.
OrdersType CurrentOrder;
bool Reuse =
canReuseExtract(TE.Scalars, CurrentOrder, /*ResizeAllowed=*/true);
if (Reuse || !CurrentOrder.empty())
return std::move(CurrentOrder);
}
// If the gather node is <undef, v, .., poison> and
// insertelement poison, v, 0 [+ permute]
// is cheaper than
// insertelement poison, v, n - try to reorder.
// If rotating the whole graph, exclude the permute cost, the whole graph
// might be transformed.
int Sz = TE.Scalars.size();
if (isSplat(TE.Scalars) && !allConstant(TE.Scalars) &&
count_if(TE.Scalars, IsaPred<UndefValue>) == Sz - 1) {
const auto *It = find_if_not(TE.Scalars, isConstant);
if (It == TE.Scalars.begin())
return OrdersType();
auto *Ty =
cast<VectorType>(getWidenedType(TE.Scalars.front()->getType(), Sz));
if (It != TE.Scalars.end()) {
OrdersType Order(Sz, Sz);
unsigned Idx = std::distance(TE.Scalars.begin(), It);
Order[Idx] = 0;
fixupOrderingIndices(Order);
SmallVector<int> Mask;
inversePermutation(Order, Mask);
InstructionCost PermuteCost =
TopToBottom ? 0
: getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, Ty,
CostKind, Mask);
InstructionCost InsertFirstCost =
TTI->getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, 0,
PoisonValue::get(Ty), *It);
InstructionCost InsertIdxCost =
TTI->getVectorInstrCost(Instruction::InsertElement, Ty, CostKind,
Idx, PoisonValue::get(Ty), *It);
if (InsertFirstCost + PermuteCost < InsertIdxCost) {
OrdersType Order(Sz, Sz);
Order[Idx] = 0;
return std::move(Order);
}
}
}
if (isSplat(TE.Scalars))
return std::nullopt;
if (TE.Scalars.size() >= 3)
if (std::optional<OrdersType> Order = findPartiallyOrderedLoads(TE))
return Order;
// Check if can include the order of vectorized loads. For masked gathers do
// extra analysis later, so include such nodes into a special list.
if (TE.hasState() && TE.getOpcode() == Instruction::Load) {
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
OrdersType CurrentOrder;
LoadsState Res = canVectorizeLoads(TE.Scalars, TE.Scalars.front(),
CurrentOrder, PointerOps, SPtrInfo);
if (Res == LoadsState::Vectorize || Res == LoadsState::StridedVectorize ||
Res == LoadsState::CompressVectorize ||
Res == LoadsState::BlendedLoadVectorize)
return std::move(CurrentOrder);
}
if (std::optional<OrdersType> CurrentOrder =
findReusedOrderedScalars(TE, TopToBottom, IgnoreReorder))
return CurrentOrder;
}
return std::nullopt;
}
void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
// Reorder reuses mask.
reorderReuses(TE.ReuseShuffleIndices, Mask);
const unsigned Sz = TE.Scalars.size();
// For vectorized and non-clustered reused no need to do anything else.
if (!TE.isGather() ||
!ShuffleVectorInst::isOneUseSingleSourceMask(TE.ReuseShuffleIndices,
Sz) ||
!isRepeatedNonIdentityClusteredMask(TE.ReuseShuffleIndices, Sz))
return;
SmallVector<int> NewMask;
inversePermutation(TE.ReorderIndices, NewMask);
addMask(NewMask, TE.ReuseShuffleIndices);
// Clear reorder since it is going to be applied to the new mask.
TE.ReorderIndices.clear();
// Try to improve gathered nodes with clustered reuses, if possible.
ArrayRef<int> Slice = ArrayRef(NewMask).slice(0, Sz);
SmallVector<unsigned> NewOrder(Slice);
inversePermutation(NewOrder, NewMask);
reorderScalars(TE.Scalars, NewMask);
// Fill the reuses mask with the identity submasks.
for (auto *It = TE.ReuseShuffleIndices.begin(),
*End = TE.ReuseShuffleIndices.end();
It != End; std::advance(It, Sz))
std::iota(It, std::next(It, Sz), 0);
}
bool BoUpSLP::isProfitableToReorder() const {
if (Opts.slp_disable_tree_reorder)
return false;
constexpr unsigned TinyVF = 2;
constexpr unsigned TinyTree = 10;
constexpr unsigned PhiOpsLimit = 12;
constexpr unsigned GatherLoadsLimit = 2;
if (VectorizableTree.size() <= TinyTree)
return true;
if (getRootNode().hasState() && !getRootNode().isGather() &&
(getRootNode().getOpcode() == Instruction::Store ||
getRootNode().getOpcode() == Instruction::PHI ||
(getRootNode().getVectorFactor() <= TinyVF &&
(getRootNode().getOpcode() == Instruction::PtrToInt ||
getRootNode().getOpcode() == Instruction::PtrToAddr ||
getRootNode().getOpcode() == Instruction::ICmp))) &&
getRootNode().ReorderIndices.empty()) {
// Check if the tree has only single store and single (unordered) load node,
// other nodes are phis or geps/binops, combined with phis, and/or single
// gather load node
if (getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::PHI &&
getRootNodeScalars().size() == TinyVF &&
getRootNode().getNumOperands() > PhiOpsLimit)
return false;
// Single node, which require reorder - skip.
if (getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::Store &&
getRootNode().ReorderIndices.empty()) {
const unsigned ReorderedSplitsCnt =
count_if(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->State == TreeEntry::SplitVectorize &&
!TE->ReorderIndices.empty() && TE->UserTreeIndex.UserTE &&
TE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
isCommutative(TE->UserTreeIndex.UserTE->getMainOp());
});
if (ReorderedSplitsCnt <= 1 &&
static_cast<unsigned>(count_if(
VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return ((!TE->isGather() &&
(TE->ReorderIndices.empty() ||
(TE->UserTreeIndex.UserTE &&
TE->UserTreeIndex.UserTE->State ==
TreeEntry::Vectorize &&
!TE->UserTreeIndex.UserTE->ReuseShuffleIndices
.empty()))) ||
(TE->isGather() && TE->ReorderIndices.empty() &&
(!TE->hasState() || TE->isAltShuffle() ||
TE->getOpcode() == Instruction::Load ||
TE->getOpcode() == Instruction::ZExt ||
TE->getOpcode() == Instruction::SExt))) &&
(getRootNode().getVectorFactor() > TinyVF ||
!TE->isGather() || none_of(TE->Scalars, [&](Value *V) {
return !isConstant(V) && isVectorized(V);
}));
})) >= VectorizableTree.size() - ReorderedSplitsCnt)
return false;
}
bool HasPhis = false;
bool HasLoad = true;
unsigned GatherLoads = 0;
for (const std::unique_ptr<TreeEntry> &TE :
ArrayRef(VectorizableTree).drop_front()) {
if (TE->State == TreeEntry::SplitVectorize)
continue;
if (!TE->hasState()) {
if (all_of(TE->Scalars, IsaPred<Constant, PHINode>) ||
all_of(TE->Scalars, IsaPred<BinaryOperator, PHINode>))
continue;
if (getRootNodeScalars().size() == TinyVF &&
any_of(TE->Scalars, IsaPred<PHINode, GEPOperator>))
continue;
return true;
}
if (TE->getOpcode() == Instruction::Load && TE->ReorderIndices.empty()) {
if (!TE->isGather()) {
HasLoad = false;
continue;
}
if (HasLoad)
return true;
++GatherLoads;
if (GatherLoads >= GatherLoadsLimit)
return true;
}
if (TE->getOpcode() == Instruction::GetElementPtr ||
Instruction::isBinaryOp(TE->getOpcode()))
continue;
if (TE->getOpcode() != Instruction::PHI &&
(!TE->hasCopyableElements() ||
static_cast<unsigned>(count_if(TE->Scalars, IsaPred<PHINode>)) <
TE->Scalars.size() / 2))
return true;
if (getRootNodeScalars().size() == TinyVF &&
TE->getNumOperands() > PhiOpsLimit)
return false;
HasPhis = true;
}
return !HasPhis;
}
return true;
}
void BoUpSLP::TreeEntry::reorderSplitNode(unsigned Idx, ArrayRef<int> Mask,
ArrayRef<int> MaskOrder) {
assert(State == TreeEntry::SplitVectorize && "Expected split user node.");
SmallVector<int> NewMask(getVectorFactor());
SmallVector<int> NewMaskOrder(getVectorFactor());
std::iota(NewMask.begin(), NewMask.end(), 0);
std::iota(NewMaskOrder.begin(), NewMaskOrder.end(), 0);
if (Idx == 0) {
copy(Mask, NewMask.begin());
copy(MaskOrder, NewMaskOrder.begin());
} else {
assert(Idx == 1 && "Expected either 0 or 1 index.");
unsigned Offset = CombinedEntriesWithIndices.back().second;
for (unsigned I : seq<unsigned>(Mask.size())) {
NewMask[I + Offset] = Mask[I] + Offset;
NewMaskOrder[I + Offset] = MaskOrder[I] + Offset;
}
}
reorderScalars(Scalars, NewMask);
reorderOrder(ReorderIndices, NewMaskOrder, /*BottomOrder=*/true);
if (!ReorderIndices.empty() && BoUpSLP::isIdentityOrder(ReorderIndices))
ReorderIndices.clear();
}
void BoUpSLP::markCopyableDepsForRecalc(TreeEntry &TE) {
if (!TE.hasState())
return;
if (auto It = BlocksSchedules.find(TE.getMainOp()->getParent());
It != BlocksSchedules.end())
It->second->markCopyableDepsForRecalc(TE);
}
void BoUpSLP::reorderTopToBottom() {
// Maps VF to the graph nodes.
DenseMap<unsigned, SetVector<TreeEntry *>> VFToOrderedEntries;
// ExtractElement gather nodes which can be vectorized and need to handle
// their ordering.
DenseMap<const TreeEntry *, OrdersType> GathersToOrders;
// Phi nodes can have preferred ordering based on their result users
DenseMap<const TreeEntry *, OrdersType> PhisToOrders;
// AltShuffles can also have a preferred ordering that leads to fewer
// instructions, e.g., the addsub instruction in x86.
DenseMap<const TreeEntry *, OrdersType> AltShufflesToOrders;
// Maps a TreeEntry to the reorder indices of external users.
DenseMap<const TreeEntry *, SmallVector<OrdersType, 1>>
ExternalUserReorderMap;
// TODO: Reordering of struct types is not supported.
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->State == TreeEntry::Vectorize &&
isa<StructType>(
getValueType(TE->Scalars.front(), Opts.slp_revec));
}))
return;
// Compute IgnoreReorder once - it depends only on UserIgnoreList and
// getRootNode(), which do not change during this loop.
const bool IgnoreReorder =
!UserIgnoreList && getRootNode().hasState() &&
(getRootNode().getOpcode() == Instruction::InsertElement ||
getRootNode().getOpcode() == Instruction::InsertValue ||
getRootNode().getOpcode() == Instruction::Store);
// Find all reorderable nodes with the given VF.
// Currently the are vectorized stores,loads,extracts + some gathering of
// extracts.
for_each(VectorizableTree, [&, &TTIRef = *TTI](
const std::unique_ptr<TreeEntry> &TE) {
// Look for external users that will probably be vectorized.
SmallVector<OrdersType, 1> ExternalUserReorderIndices =
findExternalStoreUsersReorderIndices(TE.get());
if (!ExternalUserReorderIndices.empty()) {
VFToOrderedEntries[TE->getVectorFactor()].insert(TE.get());
ExternalUserReorderMap.try_emplace(TE.get(),
std::move(ExternalUserReorderIndices));
}
// Patterns like [fadd,fsub] can be combined into a single instruction in
// x86. Reordering them into [fsub,fadd] blocks this pattern. So we need
// to take into account their order when looking for the most used order.
if (TE->hasState() && TE->isAltShuffle() &&
TE->State != TreeEntry::SplitVectorize) {
Type *ScalarTy = TE->Scalars[0]->getType();
auto *VecTy =
cast<VectorType>(getWidenedType(ScalarTy, TE->Scalars.size()));
unsigned Opcode0 = TE->getOpcode();
unsigned Opcode1 = TE->getAltOpcode();
SmallBitVector OpcodeMask(
getAltInstrMask(TE->Scalars, ScalarTy, Opcode0, Opcode1));
// If this pattern is supported by the target then we consider the order.
if (TTIRef.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask)) {
VFToOrderedEntries[TE->getVectorFactor()].insert(TE.get());
AltShufflesToOrders.try_emplace(TE.get(), OrdersType());
}
// TODO: Check the reverse order too.
}
if (std::optional<OrdersType> CurrentOrder =
getReorderingData(*TE, /*TopToBottom=*/true, IgnoreReorder)) {
// Do not include ordering for nodes used in the alt opcode vectorization,
// better to reorder them during bottom-to-top stage. If follow the order
// here, it causes reordering of the whole graph though actually it is
// profitable just to reorder the subgraph that starts from the alternate
// opcode vectorization node. Such nodes already end-up with the shuffle
// instruction and it is just enough to change this shuffle rather than
// rotate the scalars for the whole graph.
unsigned Cnt = 0;
const TreeEntry *UserTE = TE.get();
while (UserTE && Cnt < Opts.slp_recursion_max_depth) {
if (!UserTE->UserTreeIndex)
break;
if (UserTE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
UserTE->UserTreeIndex.UserTE->isAltShuffle() &&
UserTE->UserTreeIndex.UserTE->Idx != 0)
return;
UserTE = UserTE->UserTreeIndex.UserTE;
++Cnt;
}
VFToOrderedEntries[TE->getVectorFactor()].insert(TE.get());
if (!(TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::SplitVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::BlendedLoadVectorize) ||
!TE->ReuseShuffleIndices.empty())
GathersToOrders.try_emplace(TE.get(), *CurrentOrder);
if (TE->State == TreeEntry::Vectorize &&
TE->getOpcode() == Instruction::PHI)
PhisToOrders.try_emplace(TE.get(), *CurrentOrder);
}
});
// Reorder the graph nodes according to their vectorization factor.
for (unsigned VF = getRootNode().getVectorFactor();
!VFToOrderedEntries.empty() && VF > 1; --VF) {
auto It = VFToOrderedEntries.find(VF);
if (It == VFToOrderedEntries.end())
continue;
// Try to find the most profitable order. We just are looking for the most
// used order and reorder scalar elements in the nodes according to this
// mostly used order.
ArrayRef<TreeEntry *> OrderedEntries = It->second.getArrayRef();
// Delete VF entry upon exit.
llvm::scope_exit Cleanup([&]() { VFToOrderedEntries.erase(It); });
// All operands are reordered and used only in this node - propagate the
// most used order to the user node.
MapVector<OrdersType, unsigned,
DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo>>
OrdersUses;
// No need to reorder this nodes, still need to extend and to use shuffle,
// just need to merge reordering shuffle and the reuse shuffle.
for (const TreeEntry *OpTE :
make_filter_range(OrderedEntries, [&](const TreeEntry *OpTE) {
return OpTE->ReuseShuffleIndices.empty() ||
GathersToOrders.count(OpTE) ||
OpTE->State == TreeEntry::SplitVectorize;
})) {
// Count number of orders uses.
const auto &Order = [OpTE, &GathersToOrders, &AltShufflesToOrders,
&PhisToOrders]() -> const OrdersType & {
if (OpTE->isGather() || !OpTE->ReuseShuffleIndices.empty()) {
auto It = GathersToOrders.find(OpTE);
if (It != GathersToOrders.end())
return It->second;
}
if (OpTE->hasState() && OpTE->isAltShuffle()) {
auto It = AltShufflesToOrders.find(OpTE);
if (It != AltShufflesToOrders.end())
return It->second;
}
if (OpTE->State == TreeEntry::Vectorize &&
OpTE->getOpcode() == Instruction::PHI) {
auto It = PhisToOrders.find(OpTE);
if (It != PhisToOrders.end())
return It->second;
}
return OpTE->ReorderIndices;
}();
// First consider the order of the external scalar users.
auto It = ExternalUserReorderMap.find(OpTE);
if (It != ExternalUserReorderMap.end()) {
const auto &ExternalUserReorderIndices = It->second;
// If the OpTE vector factor != number of scalars - use natural order,
// it is an attempt to reorder node with reused scalars but with
// external uses.
if (OpTE->getVectorFactor() != OpTE->Scalars.size()) {
OrdersUses.try_emplace(OrdersType(), 0).first->second +=
ExternalUserReorderIndices.size();
} else {
for (const OrdersType &ExtOrder : ExternalUserReorderIndices)
++OrdersUses.try_emplace(ExtOrder, 0).first->second;
}
// No other useful reorder data in this entry.
if (Order.empty())
continue;
}
// Stores actually store the mask, not the order, need to invert.
if (OpTE->State == TreeEntry::Vectorize &&
OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
assert(!OpTE->isAltShuffle() &&
"Alternate instructions are only supported by BinaryOperator "
"and CastInst.");
SmallVector<int> Mask;
inversePermutation(Order, Mask);
unsigned E = Order.size();
OrdersType CurrentOrder(E, E);
transform(Mask, CurrentOrder.begin(), [E](int Idx) {
return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
});
fixupOrderingIndices(CurrentOrder);
++OrdersUses.try_emplace(CurrentOrder, 0).first->second;
} else {
++OrdersUses.try_emplace(Order, 0).first->second;
}
}
if (OrdersUses.empty())
continue;
// Choose the most used order.
unsigned IdentityCnt = 0;
unsigned FilledIdentityCnt = 0;
OrdersType IdentityOrder(VF, VF);
for (auto &Pair : OrdersUses) {
if (Pair.first.empty() || isIdentityOrder(Pair.first)) {
if (!Pair.first.empty())
FilledIdentityCnt += Pair.second;
IdentityCnt += Pair.second;
combineOrders(IdentityOrder, Pair.first);
}
}
MutableArrayRef<unsigned> BestOrder = IdentityOrder;
unsigned Cnt = IdentityCnt;
for (auto &Pair : OrdersUses) {
// Prefer identity order. But, if filled identity found (non-empty order)
// with same number of uses, as the new candidate order, we can choose
// this candidate order.
if (Cnt < Pair.second ||
(Cnt == IdentityCnt && IdentityCnt == FilledIdentityCnt &&
Cnt == Pair.second && !BestOrder.empty() &&
isIdentityOrder(BestOrder))) {
combineOrders(Pair.first, BestOrder);
BestOrder = Pair.first;
Cnt = Pair.second;
} else {
combineOrders(BestOrder, Pair.first);
}
}
// Set order of the user node.
if (isIdentityOrder(BestOrder))
continue;
fixupOrderingIndices(BestOrder);
SmallVector<int> Mask;
inversePermutation(BestOrder, Mask);
SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
unsigned E = BestOrder.size();
transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
return I < E ? static_cast<int>(I) : PoisonMaskElem;
});
// Do an actual reordering, if profitable.
// The splat-gather subtree load roots stay in memory order: the reusing
// gathers fetch lanes by value.
for (std::unique_ptr<TreeEntry> &TE : make_filter_range(
VectorizableTree, [&](std::unique_ptr<TreeEntry> &TE) {
return TE->State != TreeEntry::Vectorize ||
TE->getOpcode() != Instruction::Load ||
!is_contained(SplatGatheredScalarsRoots, TE.get());
})) {
// Just do the reordering for the nodes with the given VF.
if (TE->Scalars.size() != VF) {
if (TE->ReuseShuffleIndices.size() == VF &&
TE->State != TreeEntry::ExpandVectorize) {
assert(TE->State != TreeEntry::SplitVectorize &&
"Split vectorized not expected.");
// Need to reorder the reuses masks of the operands with smaller VF to
// be able to find the match between the graph nodes and scalar
// operands of the given node during vectorization/cost estimation.
assert(
(!TE->UserTreeIndex ||
TE->UserTreeIndex.UserTE->Scalars.size() == VF ||
TE->UserTreeIndex.UserTE->Scalars.size() == TE->Scalars.size() ||
TE->UserTreeIndex.UserTE->State == TreeEntry::SplitVectorize) &&
"All users must be of VF size.");
if (Opts.slp_revec) {
assert(Opts.slp_revec && "Only supported by REVEC.");
// ShuffleVectorInst does not do reorderOperands (and it should not
// because ShuffleVectorInst supports only a limited set of
// patterns). Only do reorderNodeWithReuses if the user is not
// ShuffleVectorInst.
if (TE->UserTreeIndex && TE->UserTreeIndex.UserTE->hasState() &&
isa<ShuffleVectorInst>(TE->UserTreeIndex.UserTE->getMainOp()))
continue;
}
// Update ordering of the operands with the smaller VF than the given
// one.
reorderNodeWithReuses(*TE, Mask);
// Update orders in user split vectorize nodes.
if (TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->State == TreeEntry::SplitVectorize)
TE->UserTreeIndex.UserTE->reorderSplitNode(
TE->UserTreeIndex.EdgeIdx, Mask, MaskOrder);
}
continue;
}
if ((TE->State == TreeEntry::SplitVectorize &&
TE->ReuseShuffleIndices.empty()) ||
((TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::BlendedLoadVectorize) &&
(isa<ExtractElementInst, ExtractValueInst, LoadInst, StoreInst,
InsertElementInst, InsertValueInst>(TE->getMainOp()) ||
(Opts.slp_revec && isa<ShuffleVectorInst>(TE->getMainOp()))))) {
assert(
(!TE->isAltShuffle() || (TE->State == TreeEntry::SplitVectorize &&
TE->ReuseShuffleIndices.empty())) &&
"Alternate instructions are only supported by BinaryOperator "
"and CastInst.");
// Build correct orders for extract{element,value}, loads,
// stores and alternate (split) nodes.
reorderOrder(TE->ReorderIndices, Mask);
if (isa<InsertElementInst, InsertValueInst, StoreInst>(
TE->getMainOp())) {
TE->reorderOperands(Mask);
markCopyableDepsForRecalc(*TE);
}
} else {
// Reorder the node and its operands.
TE->reorderOperands(Mask);
markCopyableDepsForRecalc(*TE);
assert(TE->ReorderIndices.empty() &&
"Expected empty reorder sequence.");
reorderScalars(TE->Scalars, Mask);
}
if (!TE->ReuseShuffleIndices.empty() &&
TE->State != TreeEntry::ExpandVectorize) {
// Apply reversed order to keep the original ordering of the reused
// elements to avoid extra reorder indices shuffling. An ExpandVectorize
// store keeps its expand mask fixed and carries the reorder in
// ReorderIndices, so it is excluded here.
OrdersType CurrentOrder;
reorderOrder(CurrentOrder, MaskOrder);
SmallVector<int> NewReuses;
inversePermutation(CurrentOrder, NewReuses);
addMask(NewReuses, TE->ReuseShuffleIndices);
TE->ReuseShuffleIndices.swap(NewReuses);
} else if (TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->State == TreeEntry::SplitVectorize)
// Update orders in user split vectorize nodes.
TE->UserTreeIndex.UserTE->reorderSplitNode(TE->UserTreeIndex.EdgeIdx,
Mask, MaskOrder);
}
}
}
void BoUpSLP::buildReorderableOperands(
TreeEntry *UserTE, SmallVectorImpl<std::pair<unsigned, TreeEntry *>> &Edges,
const SmallPtrSetImpl<const TreeEntry *> &ReorderableGathers,
SmallVectorImpl<TreeEntry *> &GatherOps) {
for (unsigned I : seq<unsigned>(UserTE->getNumOperands())) {
if (any_of(Edges, [I](const std::pair<unsigned, TreeEntry *> &OpData) {
return OpData.first == I &&
(OpData.second->State == TreeEntry::Vectorize ||
OpData.second->State == TreeEntry::StridedVectorize ||
OpData.second->State == TreeEntry::ExpandVectorize ||
OpData.second->State == TreeEntry::CompressVectorize ||
OpData.second->State == TreeEntry::BlendedLoadVectorize ||
OpData.second->State == TreeEntry::SplitVectorize);
}))
continue;
// Do not request operands, if they do not exist.
if (UserTE->hasState()) {
if (UserTE->getOpcode() == Instruction::ExtractElement ||
UserTE->getOpcode() == Instruction::ExtractValue)
continue;
if ((UserTE->getOpcode() == Instruction::InsertElement ||
UserTE->getOpcode() == Instruction::InsertValue) &&
I == 0)
continue;
if (UserTE->getOpcode() == Instruction::Store && I == 1 &&
(UserTE->State == TreeEntry::Vectorize ||
UserTE->State == TreeEntry::StridedVectorize ||
UserTE->State == TreeEntry::ExpandVectorize))
continue;
if (UserTE->getOpcode() == Instruction::Load &&
(UserTE->State == TreeEntry::Vectorize ||
UserTE->State == TreeEntry::StridedVectorize ||
UserTE->State == TreeEntry::CompressVectorize ||
UserTE->State == TreeEntry::BlendedLoadVectorize))
continue;
}
TreeEntry *TE = getOperandEntry(UserTE, I);
assert(TE && "Expected operand entry.");
if (!TE->isGather()) {
// Add the node to the list of the ordered nodes with the identity
// order.
Edges.emplace_back(I, TE);
// Add ScatterVectorize nodes to the list of operands, where just
// reordering of the scalars is required. Similar to the gathers, so
// simply add to the list of gathered ops.
// If there are reused scalars, process this node as a regular vectorize
// node, just reorder reuses mask.
if (TE->State == TreeEntry::ScatterVectorize &&
TE->ReuseShuffleIndices.empty() && TE->ReorderIndices.empty())
GatherOps.push_back(TE);
continue;
}
if (ReorderableGathers.contains(TE))
GatherOps.push_back(TE);
}
}
void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
struct TreeEntryCompare {
bool operator()(const TreeEntry *LHS, const TreeEntry *RHS) const {
if (LHS->UserTreeIndex && RHS->UserTreeIndex)
return LHS->UserTreeIndex.UserTE->Idx < RHS->UserTreeIndex.UserTE->Idx;
return LHS->Idx < RHS->Idx;
}
};
PriorityQueue<TreeEntry *, SmallVector<TreeEntry *>, TreeEntryCompare> Queue;
DenseSet<const TreeEntry *> GathersToOrders;
// Find all reorderable leaf nodes with the given VF.
// Currently the are vectorized loads,extracts without alternate operands +
// some gathering of extracts.
SmallPtrSet<const TreeEntry *, 4> NonVectorized;
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (TE->State != TreeEntry::Vectorize &&
TE->State != TreeEntry::StridedVectorize &&
TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
TE->State != TreeEntry::BlendedLoadVectorize &&
TE->State != TreeEntry::SplitVectorize)
NonVectorized.insert(TE.get());
if (std::optional<OrdersType> CurrentOrder =
getReorderingData(*TE, /*TopToBottom=*/false, IgnoreReorder)) {
Queue.push(TE.get());
if (!(TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::BlendedLoadVectorize ||
TE->State == TreeEntry::SplitVectorize) ||
!TE->ReuseShuffleIndices.empty())
GathersToOrders.insert(TE.get());
}
}
// 1. Propagate order to the graph nodes, which use only reordered nodes.
// I.e., if the node has operands, that are reordered, try to make at least
// one operand order in the natural order and reorder others + reorder the
// user node itself.
SmallPtrSet<const TreeEntry *, 4> Visited, RevisitedOps;
while (!Queue.empty()) {
// 1. Filter out only reordered nodes.
std::pair<TreeEntry *, SmallVector<std::pair<unsigned, TreeEntry *>>> Users;
TreeEntry *TE = Queue.top();
const TreeEntry *UserTE = TE->UserTreeIndex.UserTE;
Queue.pop();
SmallVector<TreeEntry *> OrderedOps(1, TE);
while (!Queue.empty()) {
TE = Queue.top();
if (!UserTE || UserTE != TE->UserTreeIndex.UserTE)
break;
Queue.pop();
OrderedOps.push_back(TE);
}
for (TreeEntry *TE : make_filter_range(OrderedOps, [&](TreeEntry *TE) {
return (TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
TE->State == TreeEntry::BlendedLoadVectorize ||
TE->State == TreeEntry::SplitVectorize ||
(TE->isGather() && GathersToOrders.contains(TE))) &&
TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->State !=
TreeEntry::BlendedLoadVectorize &&
TE->ReuseShuffleIndices.empty() && Visited.insert(TE).second;
})) {
// Build a map between user nodes and their operands order to speedup
// search. The graph currently does not provide this dependency directly.
Users.first = TE->UserTreeIndex.UserTE;
Users.second.emplace_back(TE->UserTreeIndex.EdgeIdx, TE);
}
if (Users.first) {
auto &Data = Users;
// TODO: Reordering of struct types is not supported.
if (Data.first->State == TreeEntry::Vectorize &&
isa<StructType>(
getValueType(Data.first->Scalars.front(), Opts.slp_revec)))
continue;
if (Data.first->State == TreeEntry::SplitVectorize) {
assert(
Data.second.size() <= 2 &&
"Expected not greater than 2 operands for split vectorize node.");
if (any_of(Data.second,
[](const auto &Op) { return !Op.second->UserTreeIndex; }))
continue;
// Update orders in user split vectorize nodes.
assert(Data.first->CombinedEntriesWithIndices.size() == 2 &&
"Expected exactly 2 entries.");
for (const auto &P : Data.first->CombinedEntriesWithIndices) {
TreeEntry &OpTE = *VectorizableTree[P.first];
// The order of an operand that has both reordered and reused scalars
// cannot be absorbed into the split node cleanly: clearing the
// reorder indices while keeping the reuse mask (or vice versa)
// desyncs the split node scalars from the operand effective order.
// Skip reordering for such operands.
if (OpTE.State != TreeEntry::SplitVectorize &&
!OpTE.ReorderIndices.empty() && !OpTE.ReuseShuffleIndices.empty())
continue;
OrdersType Order = OpTE.ReorderIndices;
if (Order.empty() || !OpTE.ReuseShuffleIndices.empty()) {
if (!OpTE.isGather() && OpTE.ReuseShuffleIndices.empty())
continue;
const auto BestOrder =
getReorderingData(OpTE, /*TopToBottom=*/false, IgnoreReorder);
if (!BestOrder || BestOrder->empty() || isIdentityOrder(*BestOrder))
continue;
Order = *BestOrder;
}
fixupOrderingIndices(Order);
SmallVector<int> Mask;
inversePermutation(Order, Mask);
const unsigned E = Order.size();
SmallVector<int> MaskOrder(E, PoisonMaskElem);
transform(Order, MaskOrder.begin(), [E](unsigned I) {
return I < E ? static_cast<int>(I) : PoisonMaskElem;
});
Data.first->reorderSplitNode(P.second ? 1 : 0, Mask, MaskOrder);
// Clear ordering of the operand.
if (!OpTE.ReorderIndices.empty()) {
OpTE.ReorderIndices.clear();
} else if (!OpTE.ReuseShuffleIndices.empty()) {
reorderReuses(OpTE.ReuseShuffleIndices, Mask);
} else {
assert(OpTE.isGather() && "Expected only gather/buildvector node.");
reorderScalars(OpTE.Scalars, Mask);
}
}
if (Data.first->ReuseShuffleIndices.empty() &&
!Data.first->ReorderIndices.empty()) {
// Insert user node to the list to try to sink reordering deeper in
// the graph.
Queue.push(Data.first);
}
continue;
}
// Do not move the operand order to the root PHI node when the root
// order must be preserved: the root has no user to take over the order
// and it cannot be dropped at the end of the reordering.
if (!IgnoreReorder && Data.first == &getRootNode() &&
!Data.first->UserTreeIndex &&
Data.first->State == TreeEntry::Vectorize &&
Data.first->getOpcode() == Instruction::PHI &&
Data.first->ReuseShuffleIndices.empty())
continue;
// Check that operands are used only in the User node.
SmallVector<TreeEntry *> GatherOps;
buildReorderableOperands(Data.first, Data.second, NonVectorized,
GatherOps);
// All operands are reordered and used only in this node - propagate the
// most used order to the user node.
MapVector<OrdersType, unsigned,
DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo>>
OrdersUses;
// Do the analysis for each tree entry only once, otherwise the order of
// the same node my be considered several times, though might be not
// profitable.
SmallPtrSet<const TreeEntry *, 4> VisitedOps;
SmallPtrSet<const TreeEntry *, 4> VisitedUsers;
for (const auto &Op : Data.second) {
TreeEntry *OpTE = Op.second;
if (!VisitedOps.insert(OpTE).second)
continue;
if (!OpTE->ReuseShuffleIndices.empty() && !GathersToOrders.count(OpTE))
continue;
const auto Order = [&]() -> const OrdersType {
if (OpTE->isGather() || !OpTE->ReuseShuffleIndices.empty())
return getReorderingData(*OpTE, /*TopToBottom=*/false,
IgnoreReorder)
.value_or(OrdersType(1));
return OpTE->ReorderIndices;
}();
// The order is partially ordered, skip it in favor of fully non-ordered
// orders.
if (Order.size() == 1)
continue;
// Check that the reordering does not increase number of shuffles, i.e.
// same-values-nodes has same parents or their parents has same parents.
if (!Order.empty() && !isIdentityOrder(Order)) {
Value *Root = OpTE->hasState()
? OpTE->getMainOp()
: *find_if_not(OpTE->Scalars, isConstant);
auto GetSameNodesUsers = [&](Value *Root) {
SmallSetVector<TreeEntry *, 4> Res;
for (const TreeEntry *TE : ValueToGatherNodes.lookup(Root)) {
if (TE != OpTE && TE->UserTreeIndex &&
TE->getVectorFactor() == OpTE->getVectorFactor() &&
TE->Scalars.size() == OpTE->Scalars.size() &&
((TE->ReorderIndices.empty() && OpTE->isSame(TE->Scalars)) ||
(OpTE->ReorderIndices.empty() && TE->isSame(OpTE->Scalars))))
Res.insert(TE->UserTreeIndex.UserTE);
}
for (const TreeEntry *TE : getTreeEntries(Root)) {
if (TE != OpTE && TE->UserTreeIndex &&
TE->getVectorFactor() == OpTE->getVectorFactor() &&
TE->Scalars.size() == OpTE->Scalars.size() &&
((TE->ReorderIndices.empty() && OpTE->isSame(TE->Scalars)) ||
(OpTE->ReorderIndices.empty() && TE->isSame(OpTE->Scalars))))
Res.insert(TE->UserTreeIndex.UserTE);
}
return Res.takeVector();
};
auto GetNumOperands = [](const TreeEntry *TE) {
if (TE->State == TreeEntry::SplitVectorize)
return TE->getNumOperands();
if (auto *CI = dyn_cast<CallInst>(TE->getMainOp()); CI)
return CI->arg_size();
return TE->getNumOperands();
};
auto NodeShouldBeReorderedWithOperands = [&, TTI = TTI](
const TreeEntry *TE) {
Intrinsic::ID ID = Intrinsic::not_intrinsic;
if (auto *CI = dyn_cast<CallInst>(TE->getMainOp()); CI)
ID = getVectorIntrinsicIDForCall(CI, TLI);
for (unsigned Idx : seq<unsigned>(GetNumOperands(TE))) {
if (ID != Intrinsic::not_intrinsic &&
isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI))
continue;
const TreeEntry *Op = getOperandEntry(TE, Idx);
if (Op->isGather() && Op->hasState()) {
const TreeEntry *VecOp =
getSameValuesTreeEntry(Op->getMainOp(), Op->Scalars);
if (VecOp)
Op = VecOp;
}
if (Op->ReorderIndices.empty() && Op->ReuseShuffleIndices.empty())
return false;
}
return true;
};
SmallVector<TreeEntry *> Users = GetSameNodesUsers(Root);
if (!Users.empty() && !all_of(Users, [&](TreeEntry *UTE) {
if (!RevisitedOps.insert(UTE).second)
return false;
return UTE == Data.first || !UTE->ReorderIndices.empty() ||
!UTE->ReuseShuffleIndices.empty() ||
(UTE->UserTreeIndex &&
UTE->UserTreeIndex.UserTE == Data.first) ||
(Data.first->UserTreeIndex &&
Data.first->UserTreeIndex.UserTE == UTE) ||
(IgnoreReorder && UTE->UserTreeIndex &&
UTE->UserTreeIndex.UserTE->Idx == 0) ||
NodeShouldBeReorderedWithOperands(UTE);
}))
continue;
for (TreeEntry *UTE : Users) {
Intrinsic::ID ID = Intrinsic::not_intrinsic;
if (auto *CI = dyn_cast<CallInst>(UTE->getMainOp()); CI)
ID = getVectorIntrinsicIDForCall(CI, TLI);
for (unsigned Idx : seq<unsigned>(GetNumOperands(UTE))) {
if (ID != Intrinsic::not_intrinsic &&
isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI))
continue;
const TreeEntry *Op = getOperandEntry(UTE, Idx);
Visited.erase(Op);
Queue.push(const_cast<TreeEntry *>(Op));
}
}
}
unsigned NumOps = count_if(
Data.second, [OpTE](const std::pair<unsigned, TreeEntry *> &P) {
return P.second == OpTE;
});
// Stores actually store the mask, not the order, need to invert.
if (OpTE->State == TreeEntry::Vectorize &&
OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
assert(!OpTE->isAltShuffle() &&
"Alternate instructions are only supported by BinaryOperator "
"and CastInst.");
SmallVector<int> Mask;
inversePermutation(Order, Mask);
unsigned E = Order.size();
OrdersType CurrentOrder(E, E);
transform(Mask, CurrentOrder.begin(), [E](int Idx) {
return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
});
fixupOrderingIndices(CurrentOrder);
OrdersUses.try_emplace(CurrentOrder, 0).first->second += NumOps;
} else {
OrdersUses.try_emplace(Order, 0).first->second += NumOps;
}
auto Res = OrdersUses.try_emplace(OrdersType(), 0);
const auto AllowsReordering = [&](const TreeEntry *TE) {
if (!TE->ReorderIndices.empty() || !TE->ReuseShuffleIndices.empty() ||
(TE->State == TreeEntry::Vectorize && TE->isAltShuffle()) ||
(IgnoreReorder && TE->Idx == 0))
return true;
if (TE->isGather()) {
if (GathersToOrders.contains(TE))
return !getReorderingData(*TE, /*TopToBottom=*/false,
IgnoreReorder)
.value_or(OrdersType(1))
.empty();
return true;
}
return false;
};
if (OpTE->UserTreeIndex) {
TreeEntry *UserTE = OpTE->UserTreeIndex.UserTE;
if (!VisitedUsers.insert(UserTE).second)
continue;
// May reorder user node if it requires reordering, has reused
// scalars, is an alternate op vectorize node or its op nodes require
// reordering.
if (AllowsReordering(UserTE))
continue;
// Check if users allow reordering.
// Currently look up just 1 level of operands to avoid increase of
// the compile time.
// Profitable to reorder if definitely more operands allow
// reordering rather than those with natural order.
ArrayRef<std::pair<unsigned, TreeEntry *>> Ops = Users.second;
if (static_cast<unsigned>(count_if(
Ops, [UserTE, &AllowsReordering](
const std::pair<unsigned, TreeEntry *> &Op) {
return AllowsReordering(Op.second) &&
Op.second->UserTreeIndex.UserTE == UserTE;
})) <= Ops.size() / 2)
++Res.first->second;
}
}
if (OrdersUses.empty()) {
Visited.insert_range(llvm::make_second_range(Data.second));
continue;
}
// Choose the most used order.
unsigned IdentityCnt = 0;
unsigned VF = Data.second.front().second->getVectorFactor();
OrdersType IdentityOrder(VF, VF);
for (auto &Pair : OrdersUses) {
if (Pair.first.empty() || isIdentityOrder(Pair.first)) {
IdentityCnt += Pair.second;
combineOrders(IdentityOrder, Pair.first);
}
}
MutableArrayRef<unsigned> BestOrder = IdentityOrder;
unsigned Cnt = IdentityCnt;
for (auto &Pair : OrdersUses) {
// Prefer identity order. But, if filled identity found (non-empty
// order) with same number of uses, as the new candidate order, we can
// choose this candidate order.
if (Cnt < Pair.second) {
combineOrders(Pair.first, BestOrder);
BestOrder = Pair.first;
Cnt = Pair.second;
} else {
combineOrders(BestOrder, Pair.first);
}
}
// Set order of the user node.
if (isIdentityOrder(BestOrder)) {
Visited.insert_range(llvm::make_second_range(Data.second));
continue;
}
fixupOrderingIndices(BestOrder);
// Erase operands from OrderedEntries list and adjust their orders.
VisitedOps.clear();
SmallVector<int> Mask;
inversePermutation(BestOrder, Mask);
SmallVector<int> MaskOrder(BestOrder.size(), PoisonMaskElem);
unsigned E = BestOrder.size();
transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
return I < E ? static_cast<int>(I) : PoisonMaskElem;
});
for (const std::pair<unsigned, TreeEntry *> &Op : Data.second) {
TreeEntry *TE = Op.second;
if (!VisitedOps.insert(TE).second)
continue;
// TODO: Reordering of struct types is not supported.
if (TE->State == TreeEntry::Vectorize &&
isa<StructType>(getValueType(TE->Scalars.front(), Opts.slp_revec)))
continue;
if (TE->ReuseShuffleIndices.size() == BestOrder.size()) {
reorderNodeWithReuses(*TE, Mask);
continue;
}
// Gathers are processed separately.
if (TE->State != TreeEntry::Vectorize &&
TE->State != TreeEntry::StridedVectorize &&
TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
TE->State != TreeEntry::BlendedLoadVectorize &&
TE->State != TreeEntry::SplitVectorize &&
(TE->State != TreeEntry::ScatterVectorize ||
TE->ReorderIndices.empty()))
continue;
assert((BestOrder.size() == TE->ReorderIndices.size() ||
TE->ReorderIndices.empty()) &&
"Non-matching sizes of user/operand entries.");
reorderOrder(TE->ReorderIndices, Mask);
if (IgnoreReorder && TE == &getRootNode())
IgnoreReorder = false;
}
// For gathers just need to reorder its scalars.
for (TreeEntry *Gather : GatherOps) {
assert(Gather->ReorderIndices.empty() &&
"Unexpected reordering of gathers.");
if (!Gather->ReuseShuffleIndices.empty()) {
// Just reorder reuses indices.
reorderReuses(Gather->ReuseShuffleIndices, Mask);
continue;
}
// A ScatterVectorize (masked gather) node is scheduled, and the
// scheduler reads its operand list at the same lane where the scalar
// load sits, so Scalars and the operand list must stay aligned.
// Record the reorder in ReorderIndices (applied by the final shuffle)
// instead of physically permuting the scalars, matching how a scatter
// node with a non-empty order is reordered above.
if (Gather->State == TreeEntry::ScatterVectorize) {
reorderOrder(Gather->ReorderIndices, Mask);
Visited.insert(Gather);
continue;
}
reorderScalars(Gather->Scalars, Mask);
Visited.insert(Gather);
}
// Reorder operands of the user node and set the ordering for the user
// node itself.
auto IsNotProfitableAltCodeNode = [](const TreeEntry &TE) {
return TE.isAltShuffle() &&
(!TE.ReuseShuffleIndices.empty() || TE.getVectorFactor() == 2 ||
TE.ReorderIndices.empty());
};
if (Data.first->State != TreeEntry::Vectorize ||
!isa<ExtractElementInst, ExtractValueInst, LoadInst>(
Data.first->getMainOp()) ||
IsNotProfitableAltCodeNode(*Data.first)) {
Data.first->reorderOperands(Mask);
markCopyableDepsForRecalc(*Data.first);
}
if (!isa<InsertElementInst, InsertValueInst, StoreInst>(
Data.first->getMainOp()) ||
IsNotProfitableAltCodeNode(*Data.first) ||
Data.first->State == TreeEntry::CompressVectorize) {
reorderScalars(Data.first->Scalars, Mask);
reorderOrder(Data.first->ReorderIndices, MaskOrder,
/*BottomOrder=*/true);
if (Data.first->ReuseShuffleIndices.empty() &&
!Data.first->ReorderIndices.empty() &&
!IsNotProfitableAltCodeNode(*Data.first)) {
// Insert user node to the list to try to sink reordering deeper in
// the graph.
Queue.push(Data.first);
}
} else {
reorderOrder(Data.first->ReorderIndices, Mask);
}
}
}
// If the reordering is unnecessary, just remove the reorder.
if (IgnoreReorder && !getRootNode().ReorderIndices.empty() &&
getRootNode().ReuseShuffleIndices.empty())
getRootNode().ReorderIndices.clear();
}
Instruction *BoUpSLP::getRootEntryInstruction(const TreeEntry &Entry) const {
if (Entry.hasState() &&
(Entry.getOpcode() == Instruction::Store ||
Entry.getOpcode() == Instruction::Load) &&
Entry.State == TreeEntry::StridedVectorize && Entry.isReverse())
return dyn_cast<Instruction>(Entry.Scalars[Entry.ReorderIndices.front()]);
return dyn_cast<Instruction>(Entry.Scalars.front());
}
void BoUpSLP::buildExternalUses(
const ExtraValueToDebugLocsMap &ExternallyUsedValues) {
const size_t NumVectScalars = ScalarToTreeEntries.size() + 1;
DenseMap<Value *, unsigned> ScalarToExtUses;
// Peeled scalars still claimed by the tree (gathered, listed in some
// entry's scalars, or modeled as a copyable element, which is emitted as
// a scalar) survive as plain code, along with the peeled scalars
// in their operand chains, which no vector node can rematerialize.
KeptReassocScalars.clear();
SmallVector<const Value *, 8> KeptWorklist;
for (const auto &[V, Owners] : ReassocScalarToTreeEntries)
if ((isGathered(V) || !getTreeEntries(V).empty() ||
any_of(Owners,
[V = V](const TreeEntry *TE) {
return TE->isCopyableElement(const_cast<Value *>(V));
})) &&
KeptReassocScalars.insert(V).second)
KeptWorklist.push_back(V);
while (!KeptWorklist.empty()) {
const Value *V = KeptWorklist.pop_back_val();
for (const Value *Op : cast<Instruction>(V)->operand_values())
if (ReassocScalarToTreeEntries.contains(Op) &&
KeptReassocScalars.insert(Op).second)
KeptWorklist.push_back(Op);
}
// Collect the values that we need to extract from the tree.
for (auto &TEPtr : VectorizableTree) {
TreeEntry *Entry = TEPtr.get();
// No need to handle users of gathered values.
if (Entry->isGather() || Entry->State == TreeEntry::SplitVectorize ||
DeletedNodes.contains(Entry) ||
TransformedToGatherNodes.contains(Entry))
continue;
// For each lane:
for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
Value *Scalar = Entry->Scalars[Lane];
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
bool IsStructScalar = isa<StructType>(Scalar->getType());
// All uses must be replaced already? No need to do it again.
auto It = ScalarToExtUses.find(Scalar);
if (It != ScalarToExtUses.end() && !ExternalUses[It->second].User)
continue;
if (!IsStructScalar && Scalar->hasNUsesOrMore(NumVectScalars)) {
unsigned FoundLane = Entry->findLaneForValue(Scalar);
LLVM_DEBUG(dbgs() << "SLP: Need to extract from lane " << FoundLane
<< " from " << *Scalar << "for many users.\n");
It = ScalarToExtUses.try_emplace(Scalar, ExternalUses.size()).first;
ExternalUses.emplace_back(Scalar, nullptr, *Entry, FoundLane);
ExternalUsesWithNonUsers.insert(Scalar);
continue;
}
// Check if the scalar is externally used as an extra arg.
const auto ExtI = ExternallyUsedValues.find(Scalar);
if (ExtI != ExternallyUsedValues.end()) {
unsigned FoundLane = Entry->findLaneForValue(Scalar);
LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
<< FoundLane << " from " << *Scalar << ".\n");
ScalarToExtUses.try_emplace(Scalar, ExternalUses.size());
ExternalUses.emplace_back(Scalar, nullptr, *Entry, FoundLane);
continue;
}
for (User *U : Scalar->users()) {
LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
Instruction *UserInst = dyn_cast<Instruction>(U);
if (!UserInst || isDeleted(UserInst))
continue;
// Ignore users in the user ignore list.
if (UserIgnoreList && UserIgnoreList->contains(UserInst))
continue;
// Peeled reassociated scalars are subsumed by the flattened node and
// erased during vectorization, not external users. Kept ones survive
// and their uses of erased scalars become extracts like any other.
if (isReassocScalarVectorized(UserInst) &&
!KeptReassocScalars.contains(UserInst)) {
LLVM_DEBUG(dbgs() << "SLP: \tInternal (reassociated) user will be "
"removed:"
<< *U << ".\n");
continue;
}
// Skip in-tree scalars that become vectors
if (ArrayRef<TreeEntry *> UseEntries = getTreeEntries(U);
any_of(UseEntries, [this](const TreeEntry *UseEntry) {
return !DeletedNodes.contains(UseEntry) &&
!TransformedToGatherNodes.contains(UseEntry);
})) {
// Some in-tree scalars will remain as scalar in vectorized
// instructions. If that is the case, the one in FoundLane will
// be used.
if (!((Scalar->getType()->getScalarType()->isPointerTy() &&
isa<LoadInst, StoreInst>(UserInst)) ||
isa<CallInst>(UserInst)) ||
all_of(UseEntries, [&](TreeEntry *UseEntry) {
if (DeletedNodes.contains(UseEntry) ||
TransformedToGatherNodes.contains(UseEntry))
return true;
return UseEntry->State == TreeEntry::ScatterVectorize ||
!doesInTreeUserNeedToExtract(
Scalar, getRootEntryInstruction(*UseEntry), TLI,
TTI);
})) {
LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
<< ".\n");
assert(none_of(UseEntries,
[](TreeEntry *UseEntry) {
return UseEntry->isGather();
}) &&
"Bad state");
continue;
}
if (!IsStructScalar) {
U = nullptr;
if (It != ScalarToExtUses.end()) {
ExternalUses[It->second].User = nullptr;
break;
}
}
}
if (U && !IsStructScalar && Scalar->hasNUsesOrMore(UsesLimit))
U = nullptr;
unsigned FoundLane = Entry->findLaneForValue(Scalar);
LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *UserInst
<< " from lane " << FoundLane << " from " << *Scalar
<< ".\n");
It = ScalarToExtUses.try_emplace(Scalar, ExternalUses.size()).first;
ExternalUses.emplace_back(Scalar, U, *Entry, FoundLane);
ExternalUsesWithNonUsers.insert(Scalar);
if (!U)
break;
}
}
}
// The expansion of the runtime stride may reuse an in-tree instruction with
// the matching SCEV, which gets erased upon vectorization. Register an
// external use for it to replace the stride operand with the extract.
SmallVector<std::pair<const SCEV *, BasicBlock *>> Strides;
for (const auto &[StridedTE, SPtrInfo] : TreeEntryToStridedPtrInfoMap)
if (SPtrInfo.StrideSCEV && !SPtrInfo.StrideVal &&
!DeletedNodes.contains(StridedTE) &&
!TransformedToGatherNodes.contains(StridedTE))
Strides.emplace_back(SPtrInfo.StrideSCEV,
StridedTE->getMainOp()->getParent());
if (Strides.empty())
return;
for (const std::unique_ptr<TreeEntry> &TEPtr : VectorizableTree) {
TreeEntry *Entry = TEPtr.get();
if (Entry->isGather() || Entry->State == TreeEntry::SplitVectorize ||
DeletedNodes.contains(Entry) ||
TransformedToGatherNodes.contains(Entry))
continue;
for (Instruction *Scalar : make_isa_range<Instruction>(Entry->Scalars)) {
if (Entry->isCopyableElement(Scalar) ||
!SE->isSCEVable(Scalar->getType()))
continue;
const SCEV *ScalarSCEV = SE->getSCEV(Scalar);
if (isa<SCEVConstant>(ScalarSCEV) ||
none_of(Strides, [&](const auto &Stride) {
return DT->dominates(Scalar, Stride.second->getTerminator()) &&
SCEVExprContains(Stride.first, [ScalarSCEV](const SCEV *S) {
return S == ScalarSCEV;
});
}))
continue;
auto It = ScalarToExtUses.find(Scalar);
if (It != ScalarToExtUses.end()) {
// Replace all uses: the stride operand is emitted later, during the
// codegen.
ExternalUses[It->second].User = nullptr;
continue;
}
unsigned FoundLane = Entry->findLaneForValue(Scalar);
LLVM_DEBUG(dbgs() << "SLP: Need to extract: strided load stride from "
"lane "
<< FoundLane << " from " << *Scalar << ".\n");
ScalarToExtUses.try_emplace(Scalar, ExternalUses.size());
ExternalUses.emplace_back(Scalar, nullptr, *Entry, FoundLane);
ExternalUsesWithNonUsers.insert(Scalar);
}
}
}
SmallVector<SmallVector<StoreInst *>>
BoUpSLP::collectUserStores(const BoUpSLP::TreeEntry *TE) const {
SmallDenseMap<std::tuple<BasicBlock *, Type *, Value *>,
SmallVector<StoreInst *>, 8>
PtrToStoresMap;
for (unsigned Lane : seq<unsigned>(0, TE->Scalars.size())) {
Value *V = TE->Scalars[Lane];
// Don't iterate over the users of constant data.
if (!isa<Instruction>(V))
continue;
// To save compilation time we don't visit if we have too many users.
if (V->hasNUsesOrMore(UsesLimit))
break;
// Collect stores per pointer object.
for (User *U : V->users()) {
auto *SI = dyn_cast<StoreInst>(U);
// Test whether we can handle the store. V might be a global, which could
// be used in a different function.
if (SI == nullptr || !SI->isSimple() || SI->getFunction() != F ||
!isValidElementType(SI->getValueOperand()->getType(), Opts.slp_revec))
continue;
// Skip entry if already
if (isVectorized(U))
continue;
Value *Ptr = getUnderlyingObject(SI->getPointerOperand(),
Opts.slp_recursion_max_depth);
auto &StoresVec = PtrToStoresMap[{SI->getParent(),
SI->getValueOperand()->getType(), Ptr}];
// For now just keep one store per pointer object per lane.
// TODO: Extend this to support multiple stores per pointer per lane
if (StoresVec.size() > Lane)
continue;
if (!StoresVec.empty()) {
std::optional<int64_t> Diff = getPointersDiff(
SI->getValueOperand()->getType(), SI->getPointerOperand(),
SI->getValueOperand()->getType(),
StoresVec.front()->getPointerOperand(), *DL, *SE,
/*StrictCheck=*/true);
// We failed to compare the pointers so just abandon this store.
if (!Diff)
continue;
}
StoresVec.push_back(SI);
}
}
SmallVector<SmallVector<StoreInst *>> Res(PtrToStoresMap.size());
unsigned I = 0;
for (auto &P : PtrToStoresMap) {
Res[I].swap(P.second);
++I;
}
return Res;
}
bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
OrdersType &ReorderIndices) const {
// We check whether the stores in StoreVec can form a vector by sorting them
// and checking whether they are consecutive.
// To avoid calling getPointersDiff() while sorting we create a vector of
// pairs {store, offset from first} and sort this instead.
SmallVector<std::pair<int64_t, unsigned>> StoreOffsetVec;
StoreInst *S0 = StoresVec[0];
StoreOffsetVec.emplace_back(0, 0);
Type *S0Ty = S0->getValueOperand()->getType();
Value *S0Ptr = S0->getPointerOperand();
for (unsigned Idx : seq<unsigned>(1, StoresVec.size())) {
StoreInst *SI = StoresVec[Idx];
std::optional<int64_t> Diff =
getPointersDiff(S0Ty, S0Ptr, SI->getValueOperand()->getType(),
SI->getPointerOperand(), *DL, *SE,
/*StrictCheck=*/true);
StoreOffsetVec.emplace_back(*Diff, Idx);
}
// Check if the stores are consecutive by checking if their difference is 1.
if (StoreOffsetVec.size() != StoresVec.size())
return false;
sort(StoreOffsetVec, llvm::less_first());
unsigned Idx = 0;
int64_t PrevDist = 0;
for (const auto &P : StoreOffsetVec) {
if (Idx > 0 && P.first != PrevDist + 1)
return false;
PrevDist = P.first;
++Idx;
}
// Calculate the shuffle indices according to their offset against the sorted
// StoreOffsetVec.
ReorderIndices.assign(StoresVec.size(), 0);
bool IsIdentity = true;
for (auto [I, P] : enumerate(StoreOffsetVec)) {
ReorderIndices[P.second] = I;
IsIdentity &= P.second == I;
}
// Identity order (e.g., {0,1,2,3}) is modeled as an empty OrdersType in
// reorderTopToBottom() and reorderBottomToTop(), so we are following the
// same convention here.
if (IsIdentity)
ReorderIndices.clear();
return true;
}
#ifndef NDEBUG
LLVM_DUMP_METHOD static void dumpOrder(const BoUpSLP::OrdersType &Order) {
for (unsigned Idx : Order)
dbgs() << Idx << ", ";
dbgs() << "\n";
}
#endif
SmallVector<BoUpSLP::OrdersType, 1>
BoUpSLP::findExternalStoreUsersReorderIndices(TreeEntry *TE) const {
unsigned NumLanes = TE->Scalars.size();
SmallVector<SmallVector<StoreInst *>> Stores = collectUserStores(TE);
// Holds the reorder indices for each candidate store vector that is a user of
// the current TreeEntry.
SmallVector<OrdersType, 1> ExternalReorderIndices;
// Now inspect the stores collected per pointer and look for vectorization
// candidates. For each candidate calculate the reorder index vector and push
// it into `ExternalReorderIndices`
// If we have fewer than NumLanes stores, then we can't form a vector.
for (ArrayRef<StoreInst *> StoresVec :
make_filter_range(Stores, [&](ArrayRef<StoreInst *> StoresVec) {
return StoresVec.size() == NumLanes;
})) {
// If the stores are not consecutive then abandon this StoresVec.
OrdersType ReorderIndices;
if (!canFormVector(StoresVec, ReorderIndices))
continue;
// We now know that the scalars in StoresVec can form a vector instruction,
// so set the reorder indices.
ExternalReorderIndices.push_back(ReorderIndices);
}
return ExternalReorderIndices;
}
void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
const SmallDenseSet<Value *> &UserIgnoreLst) {
deleteTree();
assert(TreeEntryToStridedPtrInfoMap.empty() &&
"TreeEntryToStridedPtrInfoMap is not cleared");
UserIgnoreList = &UserIgnoreLst;
if (!allSameType(Roots))
return;
buildTreeRec(Roots, 0, EdgeInfo());
// Build splat-gather subtrees here so the reordering passes cover them too.
tryToVectorizeSplatGatheredScalars();
}
void BoUpSLP::buildTree(ArrayRef<Value *> Roots) {
deleteTree();
assert(TreeEntryToStridedPtrInfoMap.empty() &&
"TreeEntryToStridedPtrInfoMap is not cleared");
if (!allSameType(Roots))
return;
buildTreeRec(Roots, 0, EdgeInfo());
// Build splat-gather subtrees here so the reordering passes cover them too.
tryToVectorizeSplatGatheredScalars();
}
/// Tries to find subvector of loads and builds new vector of only loads if can
/// be profitable.
static void gatherPossiblyVectorizableLoads(
const BoUpSLP &R, ArrayRef<Value *> VL, const DataLayout &DL,
ScalarEvolution &SE, const TargetTransformInfo &TTI,
SmallVectorImpl<SmallVector<std::pair<LoadInst *, int64_t>>> &GatheredLoads,
bool AddNew = true) {
if (VL.empty())
return;
Type *ScalarTy = getValueType(VL.front(), R.Opts.slp_revec);
if (!isValidElementType(ScalarTy, R.Opts.slp_revec))
return;
SmallVector<SmallVector<std::pair<LoadInst *, int64_t>>> ClusteredLoads;
SmallVector<DenseMap<int64_t, LoadInst *>> ClusteredDistToLoad;
for (LoadInst *LI : make_isa_range<LoadInst>(VL)) {
if (R.isDeleted(LI) || R.isVectorized(LI) || !LI->isSimple())
continue;
bool IsFound = false;
for (auto [Map, Data] : zip(ClusteredDistToLoad, ClusteredLoads)) {
assert(LI->getParent() == Data.front().first->getParent() &&
LI->getType() == Data.front().first->getType() &&
getUnderlyingObject(LI->getPointerOperand(),
R.Opts.slp_recursion_max_depth) ==
getUnderlyingObject(Data.front().first->getPointerOperand(),
R.Opts.slp_recursion_max_depth) &&
"Expected loads with the same type, same parent and same "
"underlying pointer.");
std::optional<int64_t> Dist = R.getPointersDiff(
LI->getType(), LI->getPointerOperand(), Data.front().first->getType(),
Data.front().first->getPointerOperand(), DL, SE,
/*StrictCheck=*/true);
if (!Dist)
continue;
auto It = Map.find(*Dist);
if (It != Map.end() && It->second != LI)
continue;
if (It == Map.end()) {
Data.emplace_back(LI, *Dist);
Map.try_emplace(*Dist, LI);
}
IsFound = true;
break;
}
if (!IsFound) {
ClusteredLoads.emplace_back().emplace_back(LI, 0);
ClusteredDistToLoad.emplace_back().try_emplace(0, LI);
}
}
auto FindMatchingLoads =
[&](ArrayRef<std::pair<LoadInst *, int64_t>> Loads,
SmallVectorImpl<SmallVector<std::pair<LoadInst *, int64_t>>>
&GatheredLoads,
SetVector<unsigned> &ToAdd, SetVector<unsigned> &Repeated,
int64_t &Offset, unsigned &Start) {
if (Loads.empty())
return GatheredLoads.end();
LoadInst *LI = Loads.front().first;
auto IsFromStart = [&](const auto &P) { return P.index() >= Start; };
for (auto [Idx, Data] :
make_filter_range(enumerate(GatheredLoads), IsFromStart)) {
ToAdd.clear();
if (LI->getParent() != Data.front().first->getParent() ||
LI->getType() != Data.front().first->getType())
continue;
std::optional<int64_t> Dist =
R.getPointersDiff(LI->getType(), LI->getPointerOperand(),
Data.front().first->getType(),
Data.front().first->getPointerOperand(), DL, SE,
/*StrictCheck=*/true);
if (!Dist)
continue;
SmallSet<int64_t, 4> DataDists;
SmallPtrSet<LoadInst *, 4> DataLoads;
for (std::pair<LoadInst *, int64_t> P : Data) {
DataDists.insert(P.second);
DataLoads.insert(P.first);
}
// Found matching gathered loads - check if all loads are unique or
// can be effectively vectorized.
unsigned NumUniques = 0;
for (auto [Cnt, Pair] : enumerate(Loads)) {
bool Used = DataLoads.contains(Pair.first);
if (!Used && !DataDists.contains(*Dist + Pair.second)) {
++NumUniques;
ToAdd.insert(Cnt);
} else if (Used) {
Repeated.insert(Cnt);
}
}
if (NumUniques > 0 &&
(Loads.size() == NumUniques ||
(Loads.size() - NumUniques >= 2 &&
Loads.size() - NumUniques >= Loads.size() / 2 &&
(has_single_bit(Data.size() + NumUniques) ||
bit_ceil(Data.size()) <
bit_ceil(Data.size() + NumUniques))))) {
Offset = *Dist;
Start = Idx + 1;
return std::next(GatheredLoads.begin(), Idx);
}
}
ToAdd.clear();
return GatheredLoads.end();
};
for (ArrayRef<std::pair<LoadInst *, int64_t>> Data : ClusteredLoads) {
unsigned Start = 0;
SetVector<unsigned> ToAdd, LocalToAdd, Repeated;
int64_t Offset = 0;
auto *It = FindMatchingLoads(Data, GatheredLoads, LocalToAdd, Repeated,
Offset, Start);
while (It != GatheredLoads.end()) {
assert(!LocalToAdd.empty() && "Expected some elements to add.");
for (unsigned Idx : LocalToAdd)
It->emplace_back(Data[Idx].first, Data[Idx].second + Offset);
ToAdd.insert_range(LocalToAdd);
It = FindMatchingLoads(Data, GatheredLoads, LocalToAdd, Repeated, Offset,
Start);
}
if (any_of(seq<unsigned>(Data.size()), [&](unsigned Idx) {
return !ToAdd.contains(Idx) && !Repeated.contains(Idx);
})) {
auto AddNewLoads =
[&](SmallVectorImpl<std::pair<LoadInst *, int64_t>> &Loads) {
for (unsigned Idx : seq<unsigned>(Data.size())) {
if (ToAdd.contains(Idx) || Repeated.contains(Idx))
continue;
Loads.push_back(Data[Idx]);
}
};
if (!AddNew) {
LoadInst *LI = Data.front().first;
It = find_if(
GatheredLoads, [&](ArrayRef<std::pair<LoadInst *, int64_t>> PD) {
return PD.front().first->getParent() == LI->getParent() &&
PD.front().first->getType() == LI->getType();
});
while (It != GatheredLoads.end()) {
AddNewLoads(*It);
It = std::find_if(
std::next(It), GatheredLoads.end(),
[&](ArrayRef<std::pair<LoadInst *, int64_t>> PD) {
return PD.front().first->getParent() == LI->getParent() &&
PD.front().first->getType() == LI->getType();
});
}
}
GatheredLoads.emplace_back().append(Data.begin(), Data.end());
AddNewLoads(GatheredLoads.emplace_back());
}
}
}
void BoUpSLP::tryToVectorizeGatheredLoads(
const SmallMapVector<
std::tuple<BasicBlock *, Value *, Type *>,
SmallVector<SmallVector<std::pair<LoadInst *, int64_t>>>, 8>
&GatheredLoads) {
GatheredLoadsEntriesFirst = VectorizableTree.size();
SmallVector<SmallPtrSet<const Value *, 4>> LoadSetsToVectorize(
LoadEntriesToVectorize.size());
for (auto [Idx, Set] : zip(LoadEntriesToVectorize, LoadSetsToVectorize))
Set.insert_range(VectorizableTree[Idx]->Scalars);
// Sort loads by distance.
auto LoadSorter = [](const std::pair<LoadInst *, int64_t> &L1,
const std::pair<LoadInst *, int64_t> &L2) {
return L1.second > L2.second;
};
auto IsMaskedGatherSupported = [&, TTI = TTI](ArrayRef<LoadInst *> Loads) {
ArrayRef<Value *> Values(reinterpret_cast<Value *const *>(Loads.begin()),
Loads.size());
Align Alignment = computeCommonAlignment<LoadInst>(Values);
auto *Ty = cast<VectorType>(
getWidenedType(Loads.front()->getType(), Loads.size()));
return TTI->isLegalMaskedGather(Ty, Alignment) &&
!TTI->forceScalarizeMaskedGather(Ty, Alignment);
};
auto GetVectorizedRanges = [this](ArrayRef<LoadInst *> Loads,
BoUpSLP::ValueSet &VectorizedLoads,
SmallVectorImpl<LoadInst *> &NonVectorized,
bool Final, unsigned MaxVF) {
SmallVector<std::pair<ArrayRef<Value *>, LoadsState>> Results;
unsigned StartIdx = 0;
SmallVector<int> CandidateVFs;
if (isAllowedNonPowerOf2VF(MaxVF, Opts.slp_vectorize_non_power_of_2))
CandidateVFs.push_back(MaxVF);
for (int NumElts = getFloorFullVectorNumberOfElements(
*TTI, Loads.front()->getType(), MaxVF, Opts.slp_revec);
NumElts > 1;
NumElts = getFloorFullVectorNumberOfElements(
*TTI, Loads.front()->getType(), NumElts - 1, Opts.slp_revec)) {
CandidateVFs.push_back(NumElts);
if (Opts.slp_vectorize_non_power_of_2 && NumElts > 2)
CandidateVFs.push_back(NumElts - 1);
}
if (Final && CandidateVFs.empty())
return Results;
unsigned BestVF = Final ? CandidateVFs.back() : 0;
auto IsValidVF = [&](unsigned VF) { return !Final || VF <= BestVF; };
for (unsigned NumElts : make_filter_range(CandidateVFs, IsValidVF)) {
SmallVector<unsigned> MaskedGatherVectorized;
for (unsigned Cnt = StartIdx, E = Loads.size(); Cnt < E;
++Cnt) {
ArrayRef<LoadInst *> Slice =
ArrayRef(Loads).slice(Cnt, std::min(NumElts, E - Cnt));
if (VectorizedLoads.count(Slice.front()) ||
VectorizedLoads.count(Slice.back()) ||
areKnownNonVectorizableLoads(Slice))
continue;
// Check if it is profitable to try vectorizing gathered loads. It is
// profitable if we have more than 3 consecutive loads or if we have
// less but all users are vectorized or deleted.
bool AllowToVectorize = false;
// Check if it is profitable to vectorize 2-elements loads.
if (NumElts == 2) {
bool IsLegalBroadcastLoad = TTI->isLegalBroadcastLoad(
Slice.front()->getType(), ElementCount::getFixed(NumElts));
auto CheckIfAllowed = [=](ArrayRef<LoadInst *> Slice) {
// If single use/user - allow to vectorize.
auto IsNotSingleUse = [](LoadInst *LI) { return !LI->hasOneUse(); };
for (LoadInst *LI : make_filter_range(Slice, IsNotSingleUse)) {
// 1. Check if number of uses equals number of users.
// 2. All users are deleted.
// 3. The load broadcasts are not allowed or the load is not
// broadcasted.
if (static_cast<unsigned int>(std::distance(
LI->user_begin(), LI->user_end())) != LI->getNumUses())
return false;
if (!IsLegalBroadcastLoad)
continue;
if (LI->hasNUsesOrMore(UsesLimit))
return false;
for (User *U : LI->users()) {
if (auto *UI = dyn_cast<Instruction>(U); UI && isDeleted(UI))
continue;
for (const TreeEntry *UTE : getTreeEntries(U)) {
for (int I : seq<int>(UTE->getNumOperands())) {
if (all_of(UTE->getOperand(I), [LI](Value *V) {
return V == LI || isa<PoisonValue>(V);
}))
// Found legal broadcast - do not vectorize.
return false;
}
}
}
}
return true;
};
AllowToVectorize = CheckIfAllowed(Slice);
} else {
AllowToVectorize =
NumElts >= 3 ||
any_of(ValueToGatherNodes.at(Slice.front()),
[=](const TreeEntry *TE) {
return TE->Scalars.size() == 2 &&
((TE->Scalars.front() == Slice.front() &&
TE->Scalars.back() == Slice.back()) ||
(TE->Scalars.front() == Slice.back() &&
TE->Scalars.back() == Slice.front()));
});
}
if (AllowToVectorize) {
SmallVector<Value *> PointerOps;
OrdersType CurrentOrder;
// Try to build vector load.
ArrayRef<Value *> Values(
reinterpret_cast<Value *const *>(Slice.begin()), Slice.size());
StridedPtrInfo SPtrInfo;
LoadsState LS = canVectorizeLoads(Values, Slice.front(), CurrentOrder,
PointerOps, SPtrInfo, &BestVF);
if (LS != LoadsState::Gather ||
(BestVF > 1 && static_cast<unsigned>(NumElts) == 2 * BestVF)) {
if (LS == LoadsState::ScatterVectorize) {
if (MaskedGatherVectorized.empty() ||
Cnt >= MaskedGatherVectorized.back() + NumElts)
MaskedGatherVectorized.push_back(Cnt);
continue;
}
if (LS != LoadsState::Gather) {
Results.emplace_back(Values, LS);
VectorizedLoads.insert_range(Slice);
// If we vectorized initial block, no need to try to vectorize it
// again.
if (Cnt == StartIdx)
StartIdx += NumElts;
}
// Check if the whole array was vectorized already - exit.
if (StartIdx >= Loads.size())
break;
// Erase last masked gather candidate, if another candidate within
// the range is found to be better.
if (!MaskedGatherVectorized.empty() &&
Cnt < MaskedGatherVectorized.back() + NumElts)
MaskedGatherVectorized.pop_back();
Cnt += NumElts - 1;
continue;
}
}
if (!AllowToVectorize || BestVF == 0)
registerNonVectorizableLoads(Slice);
}
// Mark masked gathers candidates as vectorized, if any.
for (unsigned Cnt : MaskedGatherVectorized) {
ArrayRef<LoadInst *> Slice = ArrayRef(Loads).slice(
Cnt, std::min<unsigned>(NumElts, Loads.size() - Cnt));
ArrayRef<Value *> Values(
reinterpret_cast<Value *const *>(Slice.begin()), Slice.size());
Results.emplace_back(Values, LoadsState::ScatterVectorize);
VectorizedLoads.insert_range(Slice);
// If we vectorized initial block, no need to try to vectorize it again.
if (Cnt == StartIdx)
StartIdx += NumElts;
}
}
for (LoadInst *LI : Loads) {
if (!VectorizedLoads.contains(LI))
NonVectorized.push_back(LI);
}
return Results;
};
auto ProcessGatheredLoads =
[&, &TTI = *TTI](
ArrayRef<SmallVector<std::pair<LoadInst *, int64_t>>> GatheredLoads,
bool Final = false) {
SmallVector<LoadInst *> NonVectorized;
for (ArrayRef<std::pair<LoadInst *, int64_t>> LoadsDists :
GatheredLoads) {
if (LoadsDists.size() <= 1) {
NonVectorized.push_back(LoadsDists.back().first);
continue;
}
SmallVector<std::pair<LoadInst *, int64_t>> LocalLoadsDists(
LoadsDists);
SmallVector<LoadInst *> OriginalLoads(make_first_range(LoadsDists));
stable_sort(LocalLoadsDists, LoadSorter);
SmallVector<LoadInst *> Loads;
unsigned MaxConsecutiveDistance = 0;
unsigned CurrentConsecutiveDist = 1;
int64_t LastDist = LocalLoadsDists.front().second;
bool AllowMaskedGather = IsMaskedGatherSupported(OriginalLoads);
auto IsNotVectorized = [&](const std::pair<LoadInst *, int64_t> &L) {
return !isVectorized(L.first);
};
for (const std::pair<LoadInst *, int64_t> &L :
make_filter_range(LocalLoadsDists, IsNotVectorized)) {
assert(LastDist >= L.second &&
"Expected first distance always not less than second");
if (static_cast<uint64_t>(LastDist - L.second) ==
CurrentConsecutiveDist) {
++CurrentConsecutiveDist;
MaxConsecutiveDistance =
std::max(MaxConsecutiveDistance, CurrentConsecutiveDist);
Loads.push_back(L.first);
continue;
}
if (!AllowMaskedGather && CurrentConsecutiveDist == 1 &&
!Loads.empty())
Loads.pop_back();
CurrentConsecutiveDist = 1;
LastDist = L.second;
Loads.push_back(L.first);
}
if (Loads.size() <= 1)
continue;
if (AllowMaskedGather)
MaxConsecutiveDistance = Loads.size();
else if (MaxConsecutiveDistance < 2)
continue;
BoUpSLP::ValueSet VectorizedLoads;
SmallVector<LoadInst *> SortedNonVectorized;
SmallVector<std::pair<ArrayRef<Value *>, LoadsState>> Results =
GetVectorizedRanges(Loads, VectorizedLoads, SortedNonVectorized,
Final, MaxConsecutiveDistance);
if (!Results.empty() && !SortedNonVectorized.empty() &&
OriginalLoads.size() == Loads.size() &&
MaxConsecutiveDistance == Loads.size() &&
all_of(Results,
[](const std::pair<ArrayRef<Value *>, LoadsState> &P) {
return P.second == LoadsState::ScatterVectorize;
})) {
VectorizedLoads.clear();
SmallVector<LoadInst *> UnsortedNonVectorized;
SmallVector<std::pair<ArrayRef<Value *>, LoadsState>>
UnsortedResults =
GetVectorizedRanges(OriginalLoads, VectorizedLoads,
UnsortedNonVectorized, Final,
OriginalLoads.size());
if (SortedNonVectorized.size() >= UnsortedNonVectorized.size()) {
SortedNonVectorized.swap(UnsortedNonVectorized);
Results.swap(UnsortedResults);
}
}
for (ArrayRef<Value *> Slice : make_first_range(Results)) {
LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize gathered loads ("
<< Slice.size() << ")\n");
if (any_of(Slice, [&](Value *V) { return isVectorized(V); })) {
for (Value *L : Slice)
if (!isVectorized(L))
SortedNonVectorized.push_back(cast<LoadInst>(L));
continue;
}
// Select maximum VF as a maximum of user gathered nodes and
// distance between scalar loads in these nodes.
unsigned MaxVF = Slice.size();
unsigned UserMaxVF = 0;
unsigned InterleaveFactor = 0;
if (MaxVF == 2) {
UserMaxVF = MaxVF;
} else {
// Found distance between segments of the interleaved loads.
std::optional<unsigned> InterleavedLoadsDistance = 0;
unsigned Order = 0;
std::optional<unsigned> CommonVF = 0;
DenseMap<const TreeEntry *, unsigned> EntryToPosition;
SmallPtrSet<const TreeEntry *, 8> DeinterleavedNodes;
for (auto [Idx, V] : enumerate(Slice)) {
for (const TreeEntry *E : ValueToGatherNodes.at(V)) {
UserMaxVF = std::max<unsigned>(UserMaxVF, E->Scalars.size());
unsigned Pos =
EntryToPosition.try_emplace(E, Idx).first->second;
UserMaxVF = std::max<unsigned>(UserMaxVF, Idx - Pos + 1);
if (CommonVF) {
if (*CommonVF == 0) {
CommonVF = E->Scalars.size();
continue;
}
if (*CommonVF != E->Scalars.size())
CommonVF.reset();
}
// Check if the load is the part of the interleaved load.
if (Pos != Idx && InterleavedLoadsDistance) {
if (!DeinterleavedNodes.contains(E) &&
any_of(E->Scalars, [&, Slice = Slice](Value *V) {
if (isa<Constant>(V))
return false;
if (isVectorized(V))
return true;
const auto &Nodes = ValueToGatherNodes.at(V);
return (Nodes.size() != 1 || !Nodes.contains(E)) &&
!is_contained(Slice, V);
})) {
InterleavedLoadsDistance.reset();
continue;
}
DeinterleavedNodes.insert(E);
if (*InterleavedLoadsDistance == 0) {
InterleavedLoadsDistance = Idx - Pos;
continue;
}
if ((Idx - Pos) % *InterleavedLoadsDistance != 0 ||
(Idx - Pos) / *InterleavedLoadsDistance < Order)
InterleavedLoadsDistance.reset();
Order = (Idx - Pos) / InterleavedLoadsDistance.value_or(1);
}
}
}
DeinterleavedNodes.clear();
// Check if the large load represents interleaved load operation.
if (InterleavedLoadsDistance.value_or(0) > 1 &&
CommonVF.value_or(0) != 0) {
InterleaveFactor = bit_ceil(*InterleavedLoadsDistance);
unsigned VF = *CommonVF;
OrdersType Order;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
// Segmented load detected - vectorize at maximum vector factor.
if (InterleaveFactor <= Slice.size() &&
TTI.isLegalInterleavedAccessType(
cast<VectorType>(
getWidenedType(Slice.front()->getType(), VF)),
InterleaveFactor,
cast<LoadInst>(Slice.front())->getAlign(),
cast<LoadInst>(Slice.front())
->getPointerAddressSpace()) &&
canVectorizeLoads(Slice, Slice.front(), Order, PointerOps,
SPtrInfo) == LoadsState::Vectorize) {
UserMaxVF = InterleaveFactor * VF;
} else {
InterleaveFactor = 0;
}
}
// Cannot represent the loads as consecutive vectorizable nodes -
// just exit.
unsigned ConsecutiveNodesSize = 0;
if (!LoadEntriesToVectorize.empty() && InterleaveFactor == 0 &&
any_of(zip(LoadEntriesToVectorize, LoadSetsToVectorize),
[&, Slice = Slice](const auto &P) {
const auto *It = find_if(Slice, [&](Value *V) {
return std::get<1>(P).contains(V);
});
if (It == Slice.end())
return false;
const TreeEntry &TE =
*VectorizableTree[std::get<0>(P)];
ArrayRef<Value *> VL = TE.Scalars;
OrdersType Order;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
LoadsState State = canVectorizeLoads(
VL, VL.front(), Order, PointerOps, SPtrInfo);
if (State == LoadsState::ScatterVectorize ||
State == LoadsState::CompressVectorize ||
State == LoadsState::BlendedLoadVectorize)
return false;
ConsecutiveNodesSize += VL.size();
size_t Start = std::distance(Slice.begin(), It);
size_t Sz = Slice.size() - Start;
return Sz < VL.size() ||
Slice.slice(Start, VL.size()) != VL;
}))
continue;
// Try to build long masked gather loads.
UserMaxVF = bit_ceil(UserMaxVF);
if (InterleaveFactor == 0 &&
any_of(seq<unsigned>(Slice.size() / UserMaxVF),
[&, Slice = Slice](unsigned Idx) {
OrdersType Order;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
return canVectorizeLoads(
Slice.slice(Idx * UserMaxVF, UserMaxVF),
Slice[Idx * UserMaxVF], Order, PointerOps,
SPtrInfo) == LoadsState::ScatterVectorize;
}))
UserMaxVF = MaxVF;
if (Slice.size() != ConsecutiveNodesSize)
MaxVF = std::min<unsigned>(MaxVF, UserMaxVF);
}
for (unsigned VF = MaxVF; VF >= 2; VF /= 2) {
bool IsVectorized = true;
for (unsigned I = 0, E = Slice.size(); I < E; I += VF) {
ArrayRef<Value *> SubSlice =
Slice.slice(I, std::min(VF, E - I));
if (isVectorized(SubSlice.front()))
continue;
// Check if the subslice is to be-vectorized entry, which is not
// equal to entry.
if (any_of(zip(LoadEntriesToVectorize, LoadSetsToVectorize),
[&](const auto &P) {
return !SubSlice.equals(
VectorizableTree[std::get<0>(P)]
->Scalars) &&
set_is_subset(SubSlice, std::get<1>(P));
}))
continue;
unsigned Sz = VectorizableTree.size();
// A chunk smaller than InterleaveFactor cannot form an
// interleave group; keep it non-interleaved instead.
buildTreeRec(
SubSlice, 0, EdgeInfo(),
SubSlice.size() >= InterleaveFactor ? InterleaveFactor : 0);
if (Sz == VectorizableTree.size()) {
IsVectorized = false;
// Try non-interleaved vectorization with smaller vector
// factor.
if (InterleaveFactor > 0) {
VF = 2 * (MaxVF / InterleaveFactor);
InterleaveFactor = 0;
}
continue;
}
}
if (IsVectorized)
break;
}
}
NonVectorized.append(SortedNonVectorized);
}
return NonVectorized;
};
for (const auto &GLs : GatheredLoads) {
const auto &Ref = GLs.second;
SmallVector<LoadInst *> NonVectorized = ProcessGatheredLoads(Ref);
if (!Ref.empty() && !NonVectorized.empty() &&
accumulate(
Ref, 0u,
[](unsigned S, ArrayRef<std::pair<LoadInst *, int64_t>> LoadsDists)
-> unsigned { return S + LoadsDists.size(); }) !=
NonVectorized.size() &&
IsMaskedGatherSupported(NonVectorized)) {
SmallVector<SmallVector<std::pair<LoadInst *, int64_t>>>
FinalGatheredLoads;
for (LoadInst *LI : NonVectorized) {
// Reinsert non-vectorized loads to other list of loads with the same
// base pointers.
gatherPossiblyVectorizableLoads(*this, LI, *DL, *SE, *TTI,
FinalGatheredLoads,
/*AddNew=*/false);
}
// Final attempt to vectorize non-vectorized loads.
(void)ProcessGatheredLoads(FinalGatheredLoads, /*Final=*/true);
}
}
// Try to vectorize postponed load entries, previously marked as gathered.
for (unsigned Idx : LoadEntriesToVectorize) {
const TreeEntry &E = *VectorizableTree[Idx];
SmallVector<Value *> GatheredScalars(E.Scalars.begin(), E.Scalars.end());
// Avoid reordering, if possible.
if (!E.ReorderIndices.empty()) {
// Build a mask out of the reorder indices and reorder scalars per this
// mask.
SmallVector<int> ReorderMask;
inversePermutation(E.ReorderIndices, ReorderMask);
reorderScalars(GatheredScalars, ReorderMask);
}
buildTreeRec(GatheredScalars, 0, EdgeInfo());
}
// If no new entries created, consider it as no gathered loads entries must be
// handled.
if (static_cast<unsigned>(*GatheredLoadsEntriesFirst) ==
VectorizableTree.size())
GatheredLoadsEntriesFirst.reset();
}
/// Generates key/subkey pair for the given value to provide effective sorting
/// of the values and better detection of the vectorizable values sequences. The
/// keys/subkeys can be used for better sorting of the values themselves (keys)
/// and in values subgroups (subkeys).
static std::pair<size_t, size_t> generateKeySubkey(
Value *V, const TargetLibraryInfo *TLI,
function_ref<hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator,
bool AllowAlternate) {
hash_code Key = hash_value(V->getValueID() + 2);
hash_code SubKey = hash_value(0);
// Sort the loads by the distance between the pointers.
if (auto *LI = dyn_cast<LoadInst>(V)) {
Key = hash_combine(LI->getType(), hash_value(Instruction::Load), Key);
if (LI->isSimple())
SubKey = hash_value(LoadsSubkeyGenerator(Key, LI));
else
Key = SubKey = hash_value(LI);
} else if (isVectorLikeInstWithConstOps(V)) {
// Sort extracts by the vector operands.
if (isa<ExtractElementInst, UndefValue>(V))
Key = hash_value(Value::UndefValueVal + 1);
if (auto *EI = dyn_cast<ExtractElementInst>(V)) {
if (!isUndefVector(EI->getVectorOperand()).all() &&
!isa<UndefValue>(EI->getIndexOperand()))
SubKey = hash_value(EI->getVectorOperand());
}
} else if (auto *I = dyn_cast<Instruction>(V)) {
// Sort other instructions just by the opcodes except for CMPInst.
// For CMP also sort by the predicate kind.
if ((isa<BinaryOperator, CastInst>(I)) &&
isValidForAlternation(I->getOpcode())) {
if (AllowAlternate)
Key = hash_value(isa<BinaryOperator>(I) ? 1 : 0);
else
Key = hash_combine(hash_value(I->getOpcode()), Key);
SubKey = hash_combine(
hash_value(I->getOpcode()), hash_value(I->getType()),
hash_value(isa<BinaryOperator>(I)
? I->getType()
: cast<CastInst>(I)->getOperand(0)->getType()));
// For casts, look through the only operand to improve compile time.
if (isa<CastInst>(I)) {
std::pair<size_t, size_t> OpVals =
generateKeySubkey(I->getOperand(0), TLI, LoadsSubkeyGenerator,
/*AllowAlternate=*/true);
Key = hash_combine(OpVals.first, Key);
SubKey = hash_combine(OpVals.first, SubKey);
}
} else if (auto *CI = dyn_cast<CmpInst>(I)) {
CmpInst::Predicate Pred = CI->getPredicate();
if (CI->isCommutative())
Pred = std::min(Pred, CmpInst::getInversePredicate(Pred));
CmpInst::Predicate SwapPred = CmpInst::getSwappedPredicate(Pred);
SubKey = hash_combine(hash_value(I->getOpcode()), hash_value(Pred),
hash_value(SwapPred),
hash_value(CI->getOperand(0)->getType()));
} else if (auto *Call = dyn_cast<CallInst>(I)) {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(Call, TLI);
if (isTriviallyVectorizable(ID)) {
if (ID == Intrinsic::fmuladd)
ID = Intrinsic::fma;
SubKey = hash_combine(hash_value(I->getOpcode()), hash_value(ID),
hash_value(I->getType()));
} else if (!VFDatabase(*Call).getMappings(*Call).empty()) {
SubKey = hash_combine(hash_value(I->getOpcode()),
hash_value(Call->getCalledFunction()),
hash_value(I->getType()));
} else {
Key = hash_combine(hash_value(Call), Key);
SubKey = hash_combine(hash_value(I->getOpcode()), hash_value(Call));
}
for (const CallBase::BundleOpInfo &Op : Call->bundle_op_infos())
SubKey = hash_combine(hash_value(Op.Begin), hash_value(Op.End),
hash_value(Op.Tag), SubKey);
} else if (auto *Gep = dyn_cast<GetElementPtrInst>(I)) {
if (Gep->getNumOperands() == 2 && isa<ConstantInt>(Gep->getOperand(1)))
SubKey = hash_value(Gep->getPointerOperand());
else
SubKey = hash_value(Gep);
} else if (BinaryOperator::isIntDivRem(I->getOpcode()) &&
!isa<ConstantInt>(I->getOperand(1))) {
// Do not try to vectorize instructions with potentially high cost.
SubKey = hash_value(I);
} else {
SubKey =
hash_combine(hash_value(I->getOpcode()), hash_value(I->getType()));
}
Key = hash_combine(hash_value(I->getParent()->getNumber()), Key);
}
return std::make_pair(Key, SubKey);
}
/// Checks if the specified instruction \p I is an main operation for the given
/// \p MainOp and \p AltOp instructions.
static bool isMainInstruction(Instruction *I, Instruction *MainOp,
Instruction *AltOp, const TargetLibraryInfo &TLI);
/// Builds the arguments types vector for the given call instruction with the
/// given \p ID for the specified vector factor.
static SmallVector<Type *>
buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
const unsigned VF, unsigned MinBW,
const TargetTransformInfo *TTI) {
SmallVector<Type *> ArgTys;
for (auto [Idx, Arg] : enumerate(CI->args())) {
if (ID != Intrinsic::not_intrinsic) {
if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, TTI)) {
ArgTys.push_back(Arg->getType());
continue;
}
if (MinBW > 0) {
ArgTys.push_back(
getWidenedType(IntegerType::get(CI->getContext(), MinBW), VF));
continue;
}
}
ArgTys.push_back(getWidenedType(Arg->getType(), VF));
}
return ArgTys;
}
/// Calculates the costs of vectorized intrinsic (if possible) and vectorized
/// function (if possible) calls. Returns invalid cost for the corresponding
/// calls, if they cannot be vectorized/will be scalarized.
static std::pair<InstructionCost, InstructionCost>
getVectorCallCosts(CallInst *CI, Type *VecTy, const TargetTransformInfo *TTI,
const TargetLibraryInfo *TLI, ArrayRef<Type *> ArgTys,
const TTI::TargetCostKind CostKind) {
auto Shape = VFShape::get(CI->getFunctionType(),
ElementCount::getFixed(getNumElements(VecTy)),
false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
auto LibCost = InstructionCost::getInvalid();
if (!CI->isNoBuiltin() && VecFunc) {
// Calculate the cost of the vector library call.
// If the corresponding vector call is cheaper, return its cost.
LibCost = TTI->getCallInstrCost(nullptr, VecTy, ArgTys, CostKind);
}
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
// Calculate the cost of the vector intrinsic call.
FastMathFlags FMF;
if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
FMF = FPCI->getFastMathFlags();
const InstructionCost ScalarLimit = 10000;
IntrinsicCostAttributes CostAttrs(ID, VecTy, ArgTys, FMF,
dyn_cast<IntrinsicInst>(CI),
LibCost.isValid() ? LibCost : ScalarLimit);
auto IntrinsicCost = TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
if (LibCost.isValid()) {
if (IntrinsicCost > LibCost)
IntrinsicCost = InstructionCost::getInvalid();
} else if (IntrinsicCost > ScalarLimit) {
// A type-based query always scalarizes struct-returning intrinsics (e.g.
// llvm.sincos), which do not have a VFDatabase name mapping. Retry with an
// argument-aware query (as the loop vectorizer does) so such lowerings are
// taken into account.
SmallVector<const Value *> Args(CI->args());
IntrinsicCostAttributes ArgAwareAttrs(
ID, VecTy, Args, ArgTys, FMF, dyn_cast<IntrinsicInst>(CI), ScalarLimit);
IntrinsicCost = TTI->getIntrinsicInstrCost(ArgAwareAttrs, CostKind);
if (IntrinsicCost > ScalarLimit)
IntrinsicCost = InstructionCost::getInvalid();
}
return {IntrinsicCost, LibCost};
}
/// \returns the reciprocal-throughput cost of \p I widened to \p VF lanes (an
/// arithmetic op or a vectorizable call).
static InstructionCost getVectorOpCost(Instruction *I, unsigned VF,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const TTI::TargetCostKind CostKind) {
assert((isa<BinaryOperator, CallInst>(I)) &&
"getVectorOpCost expects an arithmetic op or a vectorizable call.");
Type *VecTy = getWidenedType(I->getType(), VF);
if (auto *CI = dyn_cast<CallInst>(I)) {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &TLI);
SmallVector<Type *> ArgTys = buildIntrinsicArgTypes(CI, ID, VF, 0, &TTI);
auto [IntrCost, LibCost] =
getVectorCallCosts(CI, VecTy, &TTI, &TLI, ArgTys, CostKind);
return std::min(IntrCost, LibCost);
}
SmallVector<const Value *, 2> Args{I->getOperand(0), I->getOperand(1)};
return TTI.getArithmeticInstrCost(
I->getOpcode(), VecTy, CostKind, TTI::getOperandInfo(I->getOperand(0)),
TTI::getOperandInfo(I->getOperand(1)), Args, I);
}
/// Packs a type's kind and scalar width into one key, so an opcode/intrinsic
/// id reused across scalar widths (e.g. fdiv float vs. fdiv double) is cached
/// per width instead of sharing one verdict.
static unsigned getPoorThroughputTypeKey(Type *Ty) {
return (static_cast<unsigned>(Ty->getTypeID()) << 16) |
Ty->getScalarSizeInBits();
}
namespace {
/// Memoizes opcode / intrinsic / callee kinds already ruled out as seeds.
struct PoorThroughputOpCache {
SmallDenseSet<std::pair<unsigned, unsigned>> CheapOpcodes;
SmallDenseSet<std::pair<Intrinsic::ID, unsigned>> CheapIntrinsics;
// Keyed on the callee alone: a Function's signature is fixed, so its type
// never varies between calls.
SmallDenseSet<const Function *> CheapCallees;
};
/// Groups poor-throughput seeds for tryToVectorizeSequence.
struct SeedGroupKey {
unsigned Opcode = 0;
Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
StringRef CalleeName;
bool less(const SeedGroupKey &O) const {
if (Opcode != O.Opcode)
return Opcode < O.Opcode;
if (IntrID != O.IntrID)
return IntrID < O.IntrID;
return CalleeName < O.CalleeName;
}
};
static SeedGroupKey getSeedGroupKey(const Instruction *I,
const TargetLibraryInfo &TLI) {
SeedGroupKey K;
K.Opcode = I->getOpcode();
if (auto *CI = dyn_cast<CallInst>(I)) {
K.IntrID = getVectorIntrinsicIDForCall(CI, &TLI);
if (const Function *F = CI->getCalledFunction())
K.CalleeName = F->getName();
}
return K;
}
} // namespace
/// Returns true if \p I is an expensive scalar op whose vector form is cheaper
/// per lane (e.g. fdiv, frem, fsqrt). If \p MaxVecRegSize is non-zero, the
/// widest vector fitting into it is checked too.
static bool isPoorThroughputOp(const VectorizeOptions &Opts, Instruction *I,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
PoorThroughputOpCache &Cache,
const TTI::TargetCostKind CostKind,
unsigned MaxVecRegSize = 0) {
if (!isa<BinaryOperator, CallInst>(I))
return false;
Type *Ty = I->getType();
if ((Ty->isVectorTy() && !Opts.slp_revec) || Ty->isAggregateType() ||
!isValidElementType(Ty, Opts.slp_revec))
return false;
auto Analyze = [&]() {
InstructionCost ScalarCost = TTI.getInstructionCost(I, CostKind);
if (ScalarCost < TTI::TCC_Expensive)
return false;
constexpr unsigned MinVF = 2;
if (getVectorOpCost(I, MinVF, TTI, TLI, CostKind) < ScalarCost * MinVF)
return true;
// Some ops (e.g. div by constant) scalarize at VF=2 but have a cheap
// vector form at the full register width.
unsigned TyBits = Ty->getPrimitiveSizeInBits().getFixedValue();
unsigned MaxVF = TyBits ? llvm::bit_floor(MaxVecRegSize / TyBits) : 0;
return MaxVF > MinVF &&
getVectorOpCost(I, MaxVF, TTI, TLI, CostKind) < ScalarCost * MaxVF;
};
auto CheckCached = [&](bool IsCheap, llvm::function_ref<void()> MarkCheap) {
if (IsCheap)
return false;
if (Analyze())
return true;
MarkCheap();
return false;
};
if (auto *CI = dyn_cast<CallInst>(I)) {
if (any_of(CI->args(), [&](const Value *Arg) {
return !isValidElementType(Arg->getType(), Opts.slp_revec);
}))
return false;
// A call, used only as a select operand, may get the values rejected by the
// select condition, which the vector library call handles on its slow path.
if (!CI->use_empty() && !CI->hasNUsesOrMore(UsesLimit) &&
all_of(CI->users(), [&](User *U) {
return match(
U, m_Select(m_Unless(m_Specific(CI)), m_Value(), m_Value()));
}))
return false;
if (Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &TLI)) {
auto Key = std::make_pair(ID, getPoorThroughputTypeKey(Ty));
return CheckCached(Cache.CheapIntrinsics.contains(Key),
[&] { Cache.CheapIntrinsics.insert(Key); });
}
// A call without the vector variants is never vectorized.
if (VFDatabase::getMappings(*CI).empty())
return false;
const Function *Callee = CI->getCalledFunction();
if (!Callee)
return Analyze();
return CheckCached(Cache.CheapCallees.contains(Callee),
[&] { Cache.CheapCallees.insert(Callee); });
}
auto Key = std::make_pair(I->getOpcode(), getPoorThroughputTypeKey(Ty));
return CheckCached(Cache.CheapOpcodes.contains(Key),
[&] { Cache.CheapOpcodes.insert(Key); });
}
/// Get the loop nest for the given loop.
ArrayRef<const Loop *> BoUpSLP::getLoopNest(const Loop *L) {
assert(L && "Expected valid loop");
if (Opts.slp_cost_loop_trip_count == 0)
return {};
SmallVector<const Loop *> &Res =
LoopToLoopNest.try_emplace(L).first->getSecond();
if (!Res.empty())
return Res;
SmallVector<const Loop *> LoopNest;
while (L) {
LoopNest.push_back(L);
L = L->getParentLoop();
}
Res.assign(LoopNest.rbegin(), LoopNest.rend());
return Res;
}
/// Detects an extractvalue bundle that can be widened by vectorizing the
/// underlying struct-returning calls.
///
/// \p VL is a bundle whose state \p S is Instruction::ExtractValue. The
/// bundle is acceptable for widening into one struct-of-vectors call only
/// when:
/// - every element of \p VL is an ExtractValueInst,
/// - every ExtractValueInst extracts the same struct field (its
/// getIndices() matches the main op's indices),
/// - the aggregate operands form a uniform set of CallInsts (per
/// getSameOpcode) that is not an alt-shuffle and whose return type is
/// a literal struct, and
/// - every user of every such call is itself an ExtractValueInst, so the
/// external-use extraction code can rebuild scalars via extractvalue +
/// extractelement without needing an insertvalue chain.
///
/// On success returns true and fills \p Indices with the common field
/// index path and \p Calls with the per-lane aggregate calls (in VL order),
/// for the caller to feed as the operand of the new tree entry. Otherwise
/// returns false and leaves the output parameters untouched.
static bool checkEVsForVecCalls(ArrayRef<Value *> VL,
const InstructionsState &S,
const TargetLibraryInfo &TLI,
SmallVectorImpl<unsigned> &Indices,
SmallVectorImpl<Value *> &Calls) {
assert(S && S.getOpcode() == Instruction::ExtractValue &&
"Expected extractvalue instruction state.");
if (!all_of(VL, IsaPred<ExtractValueInst>))
return false;
auto *VL0 = cast<ExtractValueInst>(S.getMainOp());
ArrayRef<unsigned> VL0Indices = VL0->getIndices();
SmallVector<Value *> Aggregates;
for (Value *V : VL) {
if (V == VL0) {
Aggregates.push_back(VL0->getAggregateOperand());
continue;
}
auto *IV = cast<ExtractValueInst>(V);
if (IV->getIndices() != VL0Indices ||
isa<ScalableVectorType>(IV->getType()))
return false;
Value *Agg = IV->getAggregateOperand();
Aggregates.push_back(Agg);
}
const InstructionsState AggState = getSameOpcode(Aggregates, TLI);
if (AggState && AggState.getOpcode() == Instruction::Call &&
!AggState.isAltShuffle() &&
isa<StructType>(AggState.getMainOp()->getType()) &&
allSameBlock(Aggregates)) {
// The struct-returning call may have non-bundle users too. The external
// extraction code rebuilds scalars by extractvalue + extractelement,
// which only works when every user of the call is an ExtractValueInst.
// Bail out if any aggregate has a different kind of user.
for (Value *Agg : Aggregates) {
if (!all_of(Agg->users(), IsaPred<ExtractValueInst>))
return false;
}
Indices.assign(VL0Indices.begin(), VL0Indices.end());
Calls.swap(Aggregates);
return true;
}
return false;
}
static bool allStructUsersAreExtractValueInsts(ArrayRef<Value *> VL) {
return all_of(VL, [](Value *V) {
return !isa<StructType>(V->getType()) ||
all_of(V->users(), IsaPred<ExtractValueInst>);
});
}
BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
const InstructionsState &S, ArrayRef<Value *> VL,
bool IsScatterVectorizeUserTE, OrdersType &CurrentOrder,
SmallVectorImpl<Value *> &PointerOps, StridedPtrInfo &SPtrInfo,
SmallVectorImpl<int> &ReuseShuffleIndices) {
assert(S.getMainOp() &&
"Expected instructions with same/alternate opcodes only.");
unsigned ShuffleOrOp =
S.isAltShuffle() ? (unsigned)Instruction::ShuffleVector : S.getOpcode();
Instruction *VL0 = S.getMainOp();
switch (ShuffleOrOp) {
case Instruction::PHI: {
// Too many operands - gather, most probably won't be vectorized.
if (VL0->getNumOperands() > MaxPHINumOperands)
return TreeEntry::NeedToGather;
// Check for terminator values (e.g. invoke).
for (PHINode *PHI : make_isa_range<PHINode>(VL)) {
for (Instruction *Term :
make_isa_range<Instruction>(PHI->incoming_values())) {
if (Term->isTerminator()) {
LLVM_DEBUG(dbgs()
<< "SLP: Need to swizzle PHINodes (terminator use).\n");
return TreeEntry::NeedToGather;
}
}
}
return TreeEntry::Vectorize;
}
case Instruction::ExtractElement:
if (any_of(VL, [&](Value *V) {
if (S.isCopyableElement(V) || isa<PoisonValue>(V))
return false;
auto *EI = dyn_cast<ExtractElementInst>(V);
if (!EI)
return true;
Value *Op = EI->getOperand(0);
if (isVectorized(Op))
return true;
auto *OpI = dyn_cast<Instruction>(Op);
return OpI && OpI->isTerminator();
}))
return TreeEntry::NeedToGather;
[[fallthrough]];
case Instruction::ExtractValue: {
bool Reuse = canReuseExtract(VL, CurrentOrder);
// Copyable lanes are inserted into the reused source vector at their own
// index, which is correct only for the identity extract order (empty
// CurrentOrder) and without a reuse shuffle; other cases fall back to
// gather.
if (S.areInstructionsWithCopyableElements() &&
(!Reuse || !ReuseShuffleIndices.empty()))
return TreeEntry::NeedToGather;
if (Reuse || !CurrentOrder.empty())
return TreeEntry::Vectorize;
SmallVector<unsigned> Indices;
SmallVector<Value *> Calls;
if (ShuffleOrOp == Instruction::ExtractValue &&
checkEVsForVecCalls(VL, S, *TLI, Indices, Calls))
return TreeEntry::Vectorize;
LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
return TreeEntry::NeedToGather;
}
case Instruction::InsertValue:
// Handle only simple insertvalue building a homogeneous aggregate from
// scalars: exactly one index and an inserted value whose type is a valid
// vector element type. Vectors and aggregates (structs/arrays) are
// rejected, since the inserted operand type is later widened into the
// result FixedVectorType.
// TODO: Support more complex insertvalues.
if (any_of(make_isa_range<InsertValueInst>(VL),
[&](InsertValueInst *IV) {
return IV->getNumIndices() != 1 ||
!::isValidElementType(IV->getOperand(1)->getType(),
Opts.slp_revec) ||
IV->getOperand(1)->getType()->isVectorTy();
}) ||
none_of(make_isa_range<InsertValueInst>(VL), [](InsertValueInst *IV) {
return isa<UndefValue>(IV->getAggregateOperand());
}))
return TreeEntry::NeedToGather;
[[fallthrough]];
case Instruction::InsertElement: {
// Check that we have a buildvector and not a shuffle of 2 or more
// different vectors.
ValueSet SourceVectors;
for (Value *V : VL) {
if (isa<PoisonValue>(V)) {
LLVM_DEBUG(
dbgs()
<< "SLP: Gather of insertelement/insertvalue/poison vector.\n");
return TreeEntry::NeedToGather;
}
SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
assert(getElementIndex(V) != std::nullopt &&
"Non-constant or undef index?");
}
if (count_if(VL, [&SourceVectors](Value *V) {
return !SourceVectors.contains(V);
}) >= 2) {
// Found 2nd source vector - cancel.
LLVM_DEBUG(
dbgs() << "SLP: Gather of insertelement/insertvalue vectors with "
"different source vectors.\n");
return TreeEntry::NeedToGather;
}
if (any_of(VL, [&SourceVectors](Value *V) {
// The last InsertElement/InsertValue can have multiple uses.
return SourceVectors.contains(V) && !V->hasOneUse();
})) {
assert((Opts.slp_revec || ShuffleOrOp == Instruction::InsertValue) &&
"Only supported by REVEC or InsertValue.");
LLVM_DEBUG(
dbgs() << "SLP: Gather of insertelement/insertvalue vectors with "
"multiple uses.\n");
return TreeEntry::NeedToGather;
}
return TreeEntry::Vectorize;
}
case Instruction::Load: {
// Check that a vectorized load would load the same memory as a scalar
// load. For example, we don't want to vectorize loads that are smaller
// than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
// treats loading/storing it as an i8 struct. If we vectorize loads/stores
// from such a struct, we read/write packed bits disagreeing with the
// unvectorized version.
auto IsGatheredNode = [&]() {
if (!GatheredLoadsEntriesFirst)
return false;
return all_of(VL, [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
return any_of(getTreeEntries(V), [&](const TreeEntry *TE) {
return TE->Idx >= *GatheredLoadsEntriesFirst;
});
});
};
switch (canVectorizeLoads(VL, VL0, CurrentOrder, PointerOps, SPtrInfo)) {
case LoadsState::Vectorize:
return TreeEntry::Vectorize;
case LoadsState::CompressVectorize:
if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
// Delay slow vectorized nodes for better vectorization attempts.
LoadEntriesToVectorize.insert(VectorizableTree.size());
return TreeEntry::NeedToGather;
}
return IsGatheredNode() ? TreeEntry::NeedToGather
: TreeEntry::CompressVectorize;
case LoadsState::ScatterVectorize:
if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
// Delay slow vectorized nodes for better vectorization attempts.
LoadEntriesToVectorize.insert(VectorizableTree.size());
return TreeEntry::NeedToGather;
}
return IsGatheredNode() ? TreeEntry::NeedToGather
: TreeEntry::ScatterVectorize;
case LoadsState::StridedVectorize:
if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
// Delay slow vectorized nodes for better vectorization attempts.
LoadEntriesToVectorize.insert(VectorizableTree.size());
return TreeEntry::NeedToGather;
}
return IsGatheredNode() ? TreeEntry::NeedToGather
: TreeEntry::StridedVectorize;
case LoadsState::BlendedLoadVectorize:
if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
// Delay slow vectorized nodes for better vectorization attempts.
LoadEntriesToVectorize.insert(VectorizableTree.size());
return TreeEntry::NeedToGather;
}
return IsGatheredNode() ? TreeEntry::NeedToGather
: TreeEntry::BlendedLoadVectorize;
case LoadsState::Gather:
#ifndef NDEBUG
Type *ScalarTy = VL0->getType();
if (DL->getTypeSizeInBits(ScalarTy) !=
DL->getTypeAllocSizeInBits(ScalarTy))
LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
else if (any_of(VL, [](Value *V) {
auto *LI = dyn_cast<LoadInst>(V);
return !LI || !LI->isSimple();
}))
LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
else
LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
#endif // NDEBUG
registerNonVectorizableLoads(VL);
return TreeEntry::NeedToGather;
}
llvm_unreachable("Unexpected state of loads");
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::PtrToAddr:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
Type *SrcTy = VL0->getOperand(0)->getType();
for (Instruction *I : make_isa_range<Instruction>(VL)) {
Type *Ty = I->getOperand(0)->getType();
if (Ty != SrcTy || !isValidElementType(Ty, Opts.slp_revec)) {
LLVM_DEBUG(
dbgs() << "SLP: Gathering casts with different src types.\n");
return TreeEntry::NeedToGather;
}
}
return TreeEntry::Vectorize;
}
case Instruction::ICmp:
case Instruction::FCmp: {
// Check that all of the compares have the same predicate.
CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0);
Type *ComparedTy = VL0->getOperand(0)->getType();
for (CmpInst *Cmp : make_isa_range<CmpInst>(VL)) {
if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0 &&
!CmpSamePredicateHelper::canConvertTo(Cmp, P0)) ||
Cmp->getOperand(0)->getType() != ComparedTy) {
LLVM_DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
return TreeEntry::NeedToGather;
}
}
return TreeEntry::Vectorize;
}
case Instruction::Select:
if (Opts.slp_revec) {
SmallPtrSet<Type *, 4> CondTypes;
for (Value *V : VL) {
Value *Cond;
if (!match(V, m_Select(m_Value(Cond), m_Value(), m_Value())) &&
!match(V, m_ZExt(m_Value(Cond))))
continue;
CondTypes.insert(Cond->getType());
}
if (CondTypes.size() > 1) {
LLVM_DEBUG(
dbgs()
<< "SLP: Gathering select with different condition types.\n");
return TreeEntry::NeedToGather;
}
}
[[fallthrough]];
case Instruction::FNeg:
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
case Instruction::Freeze:
if (S.getMainOp()->getType()->isFloatingPointTy() &&
TTI->isFPVectorizationPotentiallyUnsafe() &&
any_of(make_isa_range<BinaryOperator>(VL),
[](BinaryOperator *BO) { return !BO->isFast(); }))
return TreeEntry::NeedToGather;
return TreeEntry::Vectorize;
case Instruction::GetElementPtr: {
auto IsMatchingLane = [&S](Value *V) { return !S.isCopyableElement(V); };
// We don't combine GEPs with complicated (nested) indexing.
for (GetElementPtrInst *GEP : make_filter_range(
make_isa_range<GetElementPtrInst>(VL), IsMatchingLane)) {
if (GEP->getNumOperands() != 2) {
LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
return TreeEntry::NeedToGather;
}
}
// We can't combine several GEPs into one vector if they operate on
// different types.
Type *Ty0 = cast<GEPOperator>(VL0)->getSourceElementType();
for (GEPOperator *GEP :
make_filter_range(make_isa_range<GEPOperator>(VL), IsMatchingLane)) {
Type *CurTy = GEP->getSourceElementType();
if (Ty0 != CurTy) {
LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
return TreeEntry::NeedToGather;
}
}
// We don't combine GEPs with non-constant indexes.
Type *Ty1 = VL0->getOperand(1)->getType();
for (GetElementPtrInst *GEP : make_filter_range(
make_isa_range<GetElementPtrInst>(VL), IsMatchingLane)) {
auto *Op = GEP->getOperand(1);
if ((!IsScatterVectorizeUserTE && !isa<ConstantInt>(Op)) ||
(Op->getType() != Ty1 &&
Op->getType()->getScalarSizeInBits() >
DL->getIndexSizeInBits(
GEP->getType()->getPointerAddressSpace()))) {
LLVM_DEBUG(
dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
return TreeEntry::NeedToGather;
}
}
// The indices must be representable in a single type, the operands
// builder casts the constants to it.
if (!getCommonGEPIndexType(
VL, VL0,
[&S](Value *V) {
return isa<GetElementPtrInst>(V) && !S.isCopyableElement(V);
},
*DL)) {
LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (mixed index types).\n");
return TreeEntry::NeedToGather;
}
return TreeEntry::Vectorize;
}
case Instruction::Store: {
// Check if the stores are consecutive or if we need to swizzle them.
llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType();
// Avoid types that are padded when being allocated as scalars, while
// being packed together in a vector (such as i1).
if (DL->getTypeSizeInBits(ScalarTy) !=
DL->getTypeAllocSizeInBits(ScalarTy)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n");
return TreeEntry::NeedToGather;
}
// Make sure all stores in the bundle are simple - we can't vectorize
// atomic or volatile stores.
for (Value *V : VL) {
auto *SI = cast<StoreInst>(V);
if (!SI->isSimple()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n");
return TreeEntry::NeedToGather;
}
PointerOps.push_back(SI->getPointerOperand());
}
// Check the order of pointer operands.
if (llvm::sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, CurrentOrder)) {
Value *Ptr0;
Value *PtrN;
if (CurrentOrder.empty()) {
Ptr0 = PointerOps.front();
PtrN = PointerOps.back();
} else {
Ptr0 = PointerOps[CurrentOrder.front()];
PtrN = PointerOps[CurrentOrder.back()];
}
Align CommonAlignment = computeCommonAlignment<StoreInst>(VL);
std::optional<int64_t> Dist =
getPointersDiff(ScalarTy, Ptr0, ScalarTy, PtrN, *DL, *SE);
// Check that the sorted pointer operands are consecutive.
if (static_cast<uint64_t>(*Dist) == VL.size() - 1)
return TreeEntry::Vectorize;
if (Opts.slp_enable_strided_stores &&
analyzeConstantStrideCandidate(PointerOps, ScalarTy, CommonAlignment,
CurrentOrder, *Dist, Ptr0, SPtrInfo))
return TreeEntry::StridedVectorize;
// If the stores are not consecutive but the target supports masked stores
// for the widened type, lower them as a masked store.
FixedVectorType *StoreVecTy = nullptr;
if (Opts.slp_enable_masked_stores &&
isMaskedStoreCompress(VL, PointerOps, CurrentOrder, *TTI, *DL, *SE,
CommonAlignment, ReuseShuffleIndices,
StoreVecTy, Cache)) {
SPtrInfo.Ty = StoreVecTy;
return TreeEntry::ExpandVectorize;
}
}
LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
return TreeEntry::NeedToGather;
}
case Instruction::Call: {
if (S.getMainOp()->getType()->isFloatingPointTy() &&
TTI->isFPVectorizationPotentiallyUnsafe() &&
any_of(make_isa_range<FPMathOperator>(VL),
[](FPMathOperator *FPOp) { return !FPOp->isFast(); }))
return TreeEntry::NeedToGather;
// Check if the calls are all to the same vectorizable intrinsic or
// library function.
CallInst *CI = cast<CallInst>(VL0);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
VFShape Shape = VFShape::get(
CI->getFunctionType(),
ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
if (!VecFunc && !isTriviallyVectorizable(ID)) {
LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
return TreeEntry::NeedToGather;
}
if (isa<StructType>(CI->getType()) &&
!allStructUsersAreExtractValueInsts(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Struct-returning calls have non-extractvalue "
"users.\n");
return TreeEntry::NeedToGather;
}
Function *F = CI->getCalledFunction();
unsigned NumArgs = CI->arg_size();
SmallVector<Value *, 4> ScalarArgs(NumArgs, nullptr);
for (unsigned J = 0; J != NumArgs; ++J)
if (isVectorIntrinsicWithScalarOpAtArg(ID, J, TTI))
ScalarArgs[J] = CI->getArgOperand(J);
// Skip copyables (idempotent stand-ins) and poisons.
for (Value *V : make_filter_range(VL, [&](Value *V) {
return !isa<PoisonValue>(V) && !S.isCopyableElement(V);
})) {
CallInst *CI2 = dyn_cast<CallInst>(V);
Intrinsic::ID ID2 = CI2 ? getVectorIntrinsicIDForCall(CI2, TLI)
: Intrinsic::not_intrinsic;
Intrinsic::ID Equivalent = isEquivalentIntrinsicID(ID, ID2);
if (!CI2 ||
(CI2->getCalledFunction() != F &&
isEquivalentIntrinsicID(Equivalent, Intrinsic::fmuladd) ==
Intrinsic::not_intrinsic) ||
(ID != ID2 && Equivalent == Intrinsic::not_intrinsic) ||
(VecFunc &&
VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) ||
!CI->hasIdenticalOperandBundleSchema(*CI2)) {
LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V
<< "\n");
return TreeEntry::NeedToGather;
}
// Some intrinsics have scalar arguments and should be same in order for
// them to be vectorized.
for (unsigned J = 0; J != NumArgs; ++J) {
if (isVectorIntrinsicWithScalarOpAtArg(ID, J, TTI)) {
Value *A1J = CI2->getArgOperand(J);
if (ScalarArgs[J] != A1J) {
LLVM_DEBUG(dbgs()
<< "SLP: mismatched arguments in call:" << *CI
<< " argument " << ScalarArgs[J] << "!=" << A1J << "\n");
return TreeEntry::NeedToGather;
}
}
}
// Verify that the bundle operands are identical between the two calls.
if (CI->hasOperandBundles() &&
!std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
CI->op_begin() + CI->getBundleOperandsEndIndex(),
CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI
<< "!=" << *V << '\n');
return TreeEntry::NeedToGather;
}
}
SmallVector<Type *> ArgTys =
buildIntrinsicArgTypes(CI, ID, VL.size(), 0, TTI);
auto *VecTy = getWidenedType(S.getMainOp()->getType(), VL.size());
auto VecCallCosts =
getVectorCallCosts(CI, VecTy, TTI, TLI, ArgTys, CostKind);
if (!VecCallCosts.first.isValid() && !VecCallCosts.second.isValid())
return TreeEntry::NeedToGather;
return TreeEntry::Vectorize;
}
case Instruction::ShuffleVector: {
if (!S.isAltShuffle()) {
// REVEC can support non alternate shuffle.
if (Opts.slp_revec && getShufflevectorNumGroups(VL))
return TreeEntry::Vectorize;
// If this is not an alternate sequence of opcode like add-sub
// then do not vectorize this instruction.
LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
return TreeEntry::NeedToGather;
}
return TreeEntry::Vectorize;
}
default:
LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
return TreeEntry::NeedToGather;
}
}
namespace {
/// Allows to correctly handle operands of the phi nodes based on the \p Main
/// PHINode order of incoming basic blocks/values.
class PHIHandler {
DominatorTree &DT;
PHINode *Main = nullptr;
SmallVector<Value *> Phis;
SmallVector<SmallVector<Value *>> Operands;
public:
PHIHandler() = delete;
PHIHandler(DominatorTree &DT, PHINode *Main, ArrayRef<Value *> Phis)
: DT(DT), Main(Main), Phis(Phis),
Operands(Main->getNumIncomingValues(),
SmallVector<Value *>(Phis.size(), nullptr)) {}
void buildOperands() {
constexpr unsigned FastLimit = 4;
if (Main->getNumIncomingValues() <= FastLimit) {
for (unsigned I : seq<unsigned>(0, Main->getNumIncomingValues())) {
BasicBlock *InBB = Main->getIncomingBlock(I);
if (!DT.isReachableFromEntry(InBB)) {
Operands[I].assign(Phis.size(), PoisonValue::get(Main->getType()));
continue;
}
// Prepare the operand vector.
for (auto [Idx, V] : enumerate(Phis)) {
auto *P = dyn_cast<PHINode>(V);
if (!P) {
assert(isa<PoisonValue>(V) &&
"Expected isa instruction or poison value.");
Operands[I][Idx] = V;
continue;
}
if (P->getIncomingBlock(I) == InBB)
Operands[I][Idx] = P->getIncomingValue(I);
else
Operands[I][Idx] = P->getIncomingValueForBlock(InBB);
}
}
return;
}
SmallMapVector<BasicBlock *, SmallVector<unsigned>, 4>
Blocks;
for (unsigned I : seq<unsigned>(Main->getNumIncomingValues())) {
BasicBlock *InBB = Main->getIncomingBlock(I);
if (!DT.isReachableFromEntry(InBB)) {
Operands[I].assign(Phis.size(), PoisonValue::get(Main->getType()));
continue;
}
Blocks.try_emplace(InBB).first->second.push_back(I);
}
for (auto [Idx, V] : enumerate(Phis)) {
if (isa<PoisonValue>(V)) {
for (unsigned I : seq<unsigned>(Main->getNumIncomingValues()))
Operands[I][Idx] = V;
continue;
}
auto *P = cast<PHINode>(V);
for (unsigned I : seq<unsigned>(P->getNumIncomingValues())) {
BasicBlock *InBB = P->getIncomingBlock(I);
if (InBB == Main->getIncomingBlock(I)) {
if (isa_and_nonnull<PoisonValue>(Operands[I][Idx]))
continue;
Operands[I][Idx] = P->getIncomingValue(I);
continue;
}
auto *It = Blocks.find(InBB);
if (It == Blocks.end())
continue;
Operands[It->second.front()][Idx] = P->getIncomingValue(I);
}
}
for (const auto &P : Blocks) {
ArrayRef<unsigned> IncomingValues = P.second;
if (IncomingValues.size() <= 1)
continue;
unsigned BasicI = IncomingValues.consume_front();
for (unsigned I : IncomingValues) {
assert(all_of(enumerate(Operands[I]),
[&](const auto &Data) {
return !Data.value() ||
Data.value() == Operands[BasicI][Data.index()];
}) &&
"Expected empty operands list.");
Operands[I] = Operands[BasicI];
}
}
}
ArrayRef<Value *> getOperands(unsigned I) const { return Operands[I]; }
};
} // namespace
/// Returns main/alternate instructions for the given \p VL. Unlike
/// getSameOpcode supports non-compatible instructions for better SplitVectorize
/// node support.
/// \returns first main/alt instructions, if only poisons and instruction with
/// only 2 opcodes exists. Returns pair of nullptr otherwise.
static std::pair<Instruction *, Instruction *>
getMainAltOpsNoStateVL(ArrayRef<Value *> VL) {
Instruction *MainOp = nullptr;
Instruction *AltOp = nullptr;
for (Value *V :
make_filter_range(VL, [](Value *V) { return !isa<PoisonValue>(V); })) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return {};
if (!MainOp) {
MainOp = I;
continue;
}
if (MainOp->getOpcode() == I->getOpcode()) {
if (I->getParent() != MainOp->getParent())
return {};
continue;
}
if (!AltOp) {
AltOp = I;
continue;
}
if (AltOp->getOpcode() == I->getOpcode()) {
if (I->getParent() != AltOp->getParent())
return {};
continue;
}
return {};
}
if (!AltOp)
return {};
assert(MainOp && AltOp && MainOp->getOpcode() != AltOp->getOpcode() &&
"Expected different main and alt instructions.");
return std::make_pair(MainOp, AltOp);
}
/// Checks that every instruction appears once in the list and if not, packs
/// them, building \p ReuseShuffleIndices mask and mutating \p VL. The list of
/// unique scalars is extended by poison values to the whole register size.
///
/// \returns false if \p VL could not be uniquified, in which case \p VL is
/// unchanged and \p ReuseShuffleIndices is empty.
static bool tryToFindDuplicates(const VectorizeOptions &Opts,
SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &ReuseShuffleIndices,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const InstructionsState &S,
const BoUpSLP::EdgeInfo &UserTreeIdx,
const BoUpSLP &R, bool BuildGatherOnly = true) {
// TODO: Reordering of struct types is not supported.
if (isa<StructType>(getValueType(VL.front(), Opts.slp_revec))) {
LLVM_DEBUG(dbgs() << "SLP: struct type in bundle.\n");
ReuseShuffleIndices.clear();
return true;
}
// Check that every instruction appears once in this bundle.
SmallVector<Value *> UniqueValues;
SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
for (Value *V : VL) {
if (isConstant(V)) {
// Constants are always considered distinct, even if the same constant
// appears multiple times in VL.
ReuseShuffleIndices.emplace_back(
isa<PoisonValue>(V) ? PoisonMaskElem : UniqueValues.size());
UniqueValues.emplace_back(V);
continue;
}
auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
ReuseShuffleIndices.emplace_back(Res.first->second);
if (Res.second)
UniqueValues.emplace_back(V);
}
// Check if we need to schedule the scalars. If no, can keep original scalars
// and avoid extra shuffles.
bool RequireScheduling = S && S.getOpcode() != Instruction::PHI &&
!isVectorLikeInstWithConstOps(S.getMainOp()) &&
(S.areInstructionsWithCopyableElements() ||
!doesNotNeedToSchedule(UniqueValues));
// Compute this flag BEFORE the tail-poison erase below - it must reflect
// the state of the original VL (for the InsertsCost call), not the
// potentially-shrunk UniqueValues.
bool AreAllValuesNonConst = UniquePositions.size() == UniqueValues.size();
// Drop tail poisons, if the values can be vectorized.
if (RequireScheduling) {
const auto EndIt =
find_if_not(make_range(UniqueValues.rbegin(), UniqueValues.rend()),
IsaPred<PoisonValue>);
assert(EndIt != UniqueValues.rend() && "Expected at least one non-poison.");
UniqueValues.erase(EndIt.base(), UniqueValues.end());
}
unsigned NumUniqueScalarValues = UniqueValues.size();
if (NumUniqueScalarValues == VL.size()) {
ReuseShuffleIndices.clear();
return true;
}
// For VL=4 with 3 unique values: keep originals. A <3 x T> vector is
// always widened to <4 x T> on hardware, so the packing just adds an
// extra expand shuffle. Does not apply to loads (a <3 x T> load is a
// single memory access) or PHIs (benefit from compact packing in loops).
constexpr unsigned SmallVecWidth = 4;
constexpr unsigned SmallVecUniqueThreshold = 3;
if (VL.size() == SmallVecWidth &&
NumUniqueScalarValues == SmallVecUniqueThreshold && !BuildGatherOnly &&
!(S && (S.getOpcode() == Instruction::Load ||
S.getOpcode() == Instruction::PHI))) {
// Keep originals with identity reuse - no packing, no extra shuffle.
ReuseShuffleIndices.clear();
return true;
}
// Checks if unique inserts + shuffle is more profitable than just inserts or
// vectorized values.
auto EstimatePackPlusShuffleVsInserts = [&]() {
// Single instruction/argument insert - no shuffle.
if (UniquePositions.size() == 1 &&
(NumUniqueScalarValues == 1 ||
all_of(UniqueValues, IsaPred<UndefValue, Instruction, Argument>)))
return std::make_pair(false, false);
// For large gathers with power-of-2 VL where packing would produce
// non-power-of-2, reject if most scalars are constants - the packing
// overhead (non-power-of-2 split + shuffles) outweighs the benefit.
constexpr unsigned MinVLForConstGatherCheck = 4;
if (BuildGatherOnly && VL.size() > MinVLForConstGatherCheck &&
has_single_bit(static_cast<unsigned>(VL.size())) &&
!has_single_bit(NumUniqueScalarValues) &&
UniquePositions.size() * 2 < NumUniqueScalarValues)
return std::make_pair(false, false);
auto GetLoadsState = [&](ArrayRef<Value *> Loads) {
assert(S && S.getOpcode() == Instruction::Load && "Expected load.");
BoUpSLP::OrdersType Order;
SmallVector<Value *> PointerOps;
BoUpSLP::StridedPtrInfo SPtrInfo;
return R.canVectorizeLoads(Loads, S.getMainOp(), Order, PointerOps,
SPtrInfo);
};
auto CheckLoads = [&](ArrayRef<Value *> Loads, bool IncludeGather) {
BoUpSLP::LoadsState Res = GetLoadsState(Loads);
return (IncludeGather && Res == BoUpSLP::LoadsState::Gather) ||
Res == BoUpSLP::LoadsState::ScatterVectorize ||
Res == BoUpSLP::LoadsState::StridedVectorize ||
Res == BoUpSLP::LoadsState::CompressVectorize ||
Res == BoUpSLP::LoadsState::BlendedLoadVectorize;
};
// Operand of the root tree entry on the vectorize path: always pack the
// scalars (PackProfitable=true). Choose between keeping the original VL
// and packing the unique values:
// - For loads, prefer the originals only when both the deduplicated and
// the full sequence can be vectorized non-Gather, or when the reuse
// mask is the identity (the shuffle is free).
// - For everything else (including !S, where RequireScheduling is forced
// to false above), keep originals iff no scheduling is required.
bool IsRootOperand =
UserTreeIdx.UserTE && UserTreeIdx.UserTE->Idx == 0 && !BuildGatherOnly;
if (IsRootOperand) {
if (S && S.getOpcode() == Instruction::Load) {
bool UseOrig = (CheckLoads(UniqueValues, /*IncludeGather=*/true) &&
CheckLoads(VL, /*IncludeGather=*/false)) ||
ShuffleVectorInst::isIdentityMask(
ReuseShuffleIndices, ReuseShuffleIndices.size());
return std::make_pair(true, UseOrig);
}
return std::make_pair(true, !RequireScheduling);
}
APInt DemandedElts = APInt::getZero(VL.size());
for (auto [Idx, Val] : enumerate(ReuseShuffleIndices))
if (Val != PoisonMaskElem && UniquePositions.contains(UniqueValues[Val]))
DemandedElts.setBit(Idx);
Type *ScalarTy = ::getValueType(UniqueValues.front(), Opts.slp_revec);
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
auto *UniquesVecTy =
cast<VectorType>(getWidenedType(ScalarTy, NumUniqueScalarValues));
const unsigned NumParts = R.getNumberOfParts(VecTy, ScalarTy);
const unsigned UniquesNumParts = R.getNumberOfParts(UniquesVecTy, ScalarTy);
// No need to schedule scalars and only single register used? Use original
// scalars, do not pack.
if (!RequireScheduling) {
if (VL.size() / NumUniqueScalarValues == 1 &&
(NumParts <= 1 || UniquesNumParts >= NumParts))
return std::make_pair(true, true);
// For PHI operands, prefer packing with reuse shuffle - the PHI
// carries the vector through the loop cheaply.
if (S && S.getOpcode() == Instruction::PHI && NumUniqueScalarValues > 1 &&
UniquesNumParts <= NumParts)
return std::make_pair(true, false);
}
const TTI::TargetCostKind CostKind = R.getCostKind();
InstructionCost ReusesCost = getShuffleCost(
TTI, TTI::SK_PermuteSingleSrc, VecTy, CostKind,
NumUniqueScalarValues > VL.size() / 2 ? ArrayRef<int>()
: ArrayRef(ReuseShuffleIndices),
/*Index=*/0, UniquesVecTy);
// For vectorizable (non-gather) nodes with low duplication, prefer keeping
// the original values over packing uniques + reshuffling:
// - A single duplicate (non-load) adds negligible overhead.
// - When most values are already unique (>50%), or exactly half are unique
// for some ops (GEPs, non-alt-shuffle casts), the reshuffle cost may
// exceed the savings from a smaller packed vector - check against a
// per-register-part threshold (stricter for wider vectors).
if (S && !BuildGatherOnly) {
bool HasOneDup = S.getOpcode() != Instruction::Load &&
NumUniqueScalarValues + 1 == VL.size();
bool MostlyUnique = NumUniqueScalarValues * 2 > VL.size();
bool IsHalfUniqueValues =
NumUniqueScalarValues * 2 == VL.size() &&
(S.getOpcode() == Instruction::GetElementPtr ||
(isa<CastInst>(S.getMainOp()) && !S.isAltShuffle()));
InstructionCost CostThreshold =
NumParts * (VL.size() > SmallVecWidth ? 1 : 2);
if (HasOneDup ||
((MostlyUnique || IsHalfUniqueValues) && ReusesCost > CostThreshold))
return std::make_pair(true, true);
}
// For loads, check if either the deduplicated or the full (with
// duplicates) set can be scatter/compress-vectorized. Prefer the unique
// loads (pack + reshuffle) when possible, otherwise use the originals.
if (S && S.getOpcode() == Instruction::Load) {
// The poison lanes are covered by a single wide load, nothing to pack.
if (any_of(VL, IsaPred<PoisonValue>) &&
GetLoadsState(VL) == BoUpSLP::LoadsState::Vectorize)
return std::make_pair(true, true);
bool UniquesVectorized =
CheckLoads(UniqueValues, /*IncludeGather=*/false);
if (UniquesVectorized || CheckLoads(VL, /*IncludeGather=*/false)) {
bool UseOriginal = !UniquesVectorized;
// The unique loads are consecutive: pack them unless the masked
// gather of the originals is cheaper than the wide load + reshuffle.
if (UseOriginal &&
GetLoadsState(UniqueValues) == BoUpSLP::LoadsState::Vectorize) {
auto *LI0 = cast<LoadInst>(S.getMainOp());
InstructionCost PackCost =
ReusesCost + TTI.getMemoryOpCost(
Instruction::Load, UniquesVecTy, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI0->getPointerOperand()));
InstructionCost GatherCost = TTI.getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::masked_gather, VecTy, LI0->getPointerOperand(),
/*VariableMask=*/false, computeCommonAlignment<LoadInst>(VL)),
CostKind);
UseOriginal = GatherCost < PackCost;
}
return std::make_pair(true, UseOriginal);
}
}
bool CanSkipBVCost =
(!BuildGatherOnly && !RequireScheduling) || R.hasSameNode(S, VL);
InstructionCost InsertsCost =
CanSkipBVCost
? InstructionCost(TTI::TCC_Free)
: getScalarizationOverhead(TTI, Opts.slp_revec, ScalarTy, VecTy,
DemandedElts,
/*Insert=*/true, /*Extract=*/false,
CostKind, AreAllValuesNonConst, VL);
APInt UniquesDemandedElts = APInt::getAllOnes(NumUniqueScalarValues);
for (const auto [Idx, V] : enumerate(UniqueValues))
if (isConstant(V))
UniquesDemandedElts.clearBit(Idx);
InstructionCost UniquesCost =
CanSkipBVCost
? InstructionCost(TTI::TCC_Free)
: getScalarizationOverhead(TTI, Opts.slp_revec, ScalarTy,
UniquesVecTy, UniquesDemandedElts,
/*Insert=*/true,
/*Extract=*/false, CostKind,
AreAllValuesNonConst, UniqueValues);
UniquesCost += ReusesCost;
if (UniquesCost <= InsertsCost) {
// Packing to a vector that occupies the same register width as the
// original only adds the reshuffle cost; keep the originals. Loads may
// instead benefit from a wider contiguous load.
if (S && S.getOpcode() != Instruction::Load) {
unsigned EltBits =
S.getMainOp()->getDataLayout().getTypeSizeInBits(ScalarTy);
unsigned MinVF = R.getMinVF(EltBits);
auto RegWidth = [&](unsigned N) {
return std::max(
getFullVectorNumberOfElements(TTI, ScalarTy, N, Opts.slp_revec),
MinVF);
};
// Keeping the originals just moves the reshuffle to the operand
// columns with duplicates; keep them only if at most one column has
// duplicates, so the total number of reshuffles does not grow.
auto HasExtraReshuffle = [&]() {
if (BuildGatherOnly)
return false;
unsigned NumDupColumns = 0;
for (unsigned OpIdx :
seq<unsigned>(S.getMainOp()->getNumOperands())) {
SmallPtrSet<const Value *, 16> UniqueOps;
auto IsNotPoison = [](Value *V) { return !isa<PoisonValue>(V); };
for (Value *V : make_filter_range(VL, IsNotPoison)) {
Value *Op;
if (S.isCopyableElement(V)) {
if (OpIdx != 0)
continue;
Op = V;
} else {
auto *I = dyn_cast<Instruction>(V);
if (!I || OpIdx >= I->getNumOperands())
continue;
Op = I->getOperand(OpIdx);
}
if (!isConstant(Op) && !UniqueOps.insert(Op).second) {
++NumDupColumns;
break;
}
}
if (NumDupColumns > 1)
return true;
}
return false;
};
return std::make_pair(true, RegWidth(NumUniqueScalarValues) >=
RegWidth(VL.size()) &&
!HasExtraReshuffle());
}
return std::make_pair(true, false);
}
InstructionCost CostDiff = UniquesCost - InsertsCost;
if (CostDiff < TTI::TCC_Expensive ||
(R.getTreeSize() == 0 && R.isReductionTree() &&
CostDiff == TTI::TCC_Expensive))
return std::make_pair(S && (!S.isAltShuffle() || !BuildGatherOnly),
false);
// Otherwise, use original values, if values do not require scheduling and
// pass still try to vectorize them.
bool KeepOriginal = !BuildGatherOnly && !RequireScheduling;
return std::make_pair(KeepOriginal, KeepOriginal);
};
const auto [PackProfitable, UseOriginal] = EstimatePackPlusShuffleVsInserts();
if (PackProfitable) {
if (UseOriginal) {
// Prefer original scalars - avoid shuffling.
ReuseShuffleIndices.clear();
} else {
// Better to use uniques + reshuffle.
LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
VL = std::move(UniqueValues);
}
return true;
}
// Buildvector/gather of the original scalars.
LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
ReuseShuffleIndices.clear();
return false;
}
/// The nodes with no more than this number of elements are not split.
static constexpr unsigned SmallNodeSize = 4;
bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
const InstructionsState &LocalState,
SmallVectorImpl<Value *> &Op1,
SmallVectorImpl<Value *> &Op2,
OrdersType &ReorderIndices) const {
if (VL.size() <= SmallNodeSize || TTI->preferAlternateOpcodeVectorization() ||
!Opts.slp_split_alternate_instructions)
return false;
// Split vectorization of struct types is not supported.
if (isa<StructType>(getValueType(VL.front(), Opts.slp_revec)))
return false;
// Check if this is a duplicate of another split entry.
LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *LocalState.getMainOp()
<< ".\n");
for (TreeEntry *E : getSplitTreeEntries(LocalState.getMainOp())) {
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at "
<< *LocalState.getMainOp() << ".\n");
return false;
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
return false;
}
}
ReorderIndices.assign(VL.size(), VL.size());
SmallBitVector Op1Indices(VL.size());
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I) {
Op1.push_back(V);
Op1Indices.set(Idx);
continue;
}
if ((LocalState.getAltOpcode() != LocalState.getOpcode() &&
isMainInstruction(I, LocalState.getMainOp(), LocalState.getAltOp(),
*TLI)) ||
(LocalState.getAltOpcode() == LocalState.getOpcode() &&
!isAlternateInstruction(I, LocalState.getMainOp(),
LocalState.getAltOp(), *TLI))) {
Op1.push_back(V);
Op1Indices.set(Idx);
continue;
}
Op2.push_back(V);
}
Type *ScalarTy = getValueType(VL.front(), Opts.slp_revec);
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
unsigned Opcode0 = LocalState.getOpcode();
unsigned Opcode1 = LocalState.getAltOpcode();
SmallBitVector OpcodeMask(getAltInstrMask(VL, ScalarTy, Opcode0, Opcode1));
// Enable split node, only if all nodes do not form legal alternate
// instruction (like X86 addsub).
SmallPtrSet<Value *, 4> UOp1(llvm::from_range, Op1);
SmallPtrSet<Value *, 4> UOp2(llvm::from_range, Op2);
if (UOp1.size() <= 1 || UOp2.size() <= 1 ||
TTI->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask))
return false;
unsigned Op1Cnt = 0, Op2Cnt = Op1.size();
for (unsigned Idx : seq<unsigned>(VL.size())) {
if (Op1Indices.test(Idx)) {
ReorderIndices[Op1Cnt] = Idx;
++Op1Cnt;
} else {
ReorderIndices[Op2Cnt] = Idx;
++Op2Cnt;
}
}
if (isIdentityOrder(ReorderIndices))
ReorderIndices.clear();
// When VL fills a power-of-2 register but the split halves do not, the
// reorder shuffle makes the split unprofitable - reject.
else if (hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), VL.size(),
Opts.slp_revec) &&
(!hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), Op1.size(),
Opts.slp_revec) ||
!hasFullVectorsOrPowerOf2(*TTI, Op2.front()->getType(), Op2.size(),
Opts.slp_revec)))
return false;
SmallVector<int> Mask;
if (!ReorderIndices.empty())
inversePermutation(ReorderIndices, Mask);
unsigned NumParts = TTI->getNumberOfParts(VecTy);
auto *Op1VecTy = cast<VectorType>(getWidenedType(ScalarTy, Op1.size()));
auto *Op2VecTy = cast<VectorType>(getWidenedType(ScalarTy, Op2.size()));
// Check non-profitable single register ops, which better to be represented
// as alternate ops.
if (NumParts >= VL.size())
return false;
InstructionCost InsertCost = getShuffleCost(
*TTI, TTI::SK_InsertSubvector, VecTy, CostKind, {}, Op1.size(), Op2VecTy);
auto *SubVecTy = cast<VectorType>(
getWidenedType(ScalarTy, std::max(Op1.size(), Op2.size())));
InstructionCost NewShuffleCost =
getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, SubVecTy, CostKind, Mask);
if (!LocalState.isCmpOp() && NumParts <= 1 &&
(Mask.empty() || InsertCost >= NewShuffleCost))
return false;
if ((LocalState.getMainOp()->isBinaryOp() &&
LocalState.getAltOp()->isBinaryOp() &&
(LocalState.isShiftOp() || LocalState.isBitwiseLogicOp() ||
LocalState.isAddSubLikeOp() || LocalState.isMulDivLikeOp())) ||
(LocalState.getMainOp()->isCast() && LocalState.getAltOp()->isCast()) ||
(LocalState.getMainOp()->isUnaryOp() &&
LocalState.getAltOp()->isUnaryOp())) {
SmallVector<Value *> Ops0, Ops1;
for (Instruction *I : make_isa_range<Instruction>(VL)) {
Ops0.push_back(I->getOperand(0));
if (I->isBinaryOp())
Ops1.push_back(I->getOperand(1));
}
TTI::OperandValueInfo Op0Info = TTI::getOperandInfo(Ops0);
TTI::OperandValueInfo Op1Info =
Ops1.empty() ? TTI::OperandValueInfo() : TTI::getOperandInfo(Ops1);
InstructionCost OriginalVecOpsCost =
TTI->getArithmeticInstrCost(Opcode0, VecTy, CostKind, Op0Info, Op1Info,
{}, LocalState.getMainOp()) +
TTI->getArithmeticInstrCost(Opcode1, VecTy, CostKind, Op0Info, Op1Info,
{}, LocalState.getAltOp());
SmallVector<int> OriginalMask(VL.size(), PoisonMaskElem);
for (unsigned Idx : seq<unsigned>(VL.size())) {
if (isa<PoisonValue>(VL[Idx]))
continue;
OriginalMask[Idx] = Idx + (Op1Indices.test(Idx) ? 0 : VL.size());
}
InstructionCost OriginalCost =
OriginalVecOpsCost + getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy,
CostKind, OriginalMask);
InstructionCost NewVecOpsCost =
TTI->getArithmeticInstrCost(Opcode0, Op1VecTy, CostKind, Op0Info,
Op1Info, {}, LocalState.getMainOp()) +
TTI->getArithmeticInstrCost(Opcode1, Op2VecTy, CostKind, Op0Info,
Op1Info, {}, LocalState.getAltOp());
InstructionCost NewCost =
NewVecOpsCost + InsertCost +
(!VectorizableTree.empty() && getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::Store
? NewShuffleCost
: 0);
// If not profitable to split - exit.
if (NewCost >= OriginalCost)
return false;
}
return true;
}
namespace {
/// Class accepts incoming list of values, checks if it is able to model
/// "copyable" values as compatible operations, and generates the list of values
/// for scheduling and list of operands doe the new nodes.
class InstructionsCompatibilityAnalysis {
DominatorTree &DT;
const DataLayout &DL;
const TargetTransformInfo &TTI;
const TargetLibraryInfo &TLI;
unsigned MainOpcode = 0;
Instruction *MainOp = nullptr;
/// Whether every copyable in the current value list is an absorbable
/// single-use fmul/fadd. Computed once per buildInstructionsState call.
bool AbsorbCopyableFMulOrFAdds = false;
/// Checks if the opcode is supported as the main opcode for copyable
/// elements.
static bool isSupportedOpcode(const unsigned Opcode) {
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
Opcode == Instruction::Mul || Opcode == Instruction::AShr ||
Opcode == Instruction::LShr || Opcode == Instruction::Shl ||
Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
Opcode == Instruction::And || Opcode == Instruction::Or ||
Opcode == Instruction::Xor || Opcode == Instruction::FAdd ||
Opcode == Instruction::FSub || Opcode == Instruction::FMul ||
Opcode == Instruction::FDiv;
}
/// Checks if \p I can be the main op for copyable analysis: a supported
/// binary operator, fmuladd, or an integer min/max intrinsic, the only
/// call with a well-defined idempotent value (FP min/max lacks one because of
/// NaNs). An extractelement with constant index from a fixed vector is also
/// supported: the matching lanes reuse the source vector and the copyable
/// lanes are inserted into it. A single-index scalar GEP is supported: the
/// copyable lanes are modeled as zero-index GEPs of the lane values
/// themselves. A scalar zext is supported: the copyable lanes are constants
/// modeled as zexts of truncated constants.
static bool isSupportedMainOp(Instruction *I) {
return isSupportedOpcode(I->getOpcode()) || isa<MinMaxIntrinsic>(I) ||
RecurrenceDescriptor::isFMulAddIntrinsic(I) ||
(isa<ExtractElementInst>(I) && isVectorLikeInstWithConstOps(I) &&
isa<FixedVectorType>(I->getOperand(0)->getType())) ||
(isa<GetElementPtrInst>(I) && I->getNumOperands() == 2 &&
!I->getType()->isVectorTy()) ||
(isa<ZExtInst>(I) && !I->getType()->isVectorTy() &&
!isa<Constant>(I->getOperand(0)));
}
/// Identifies the best candidate value, which represents main opcode
/// operation.
/// Currently the best candidate is the Add instruction with the parent
/// block with the highest DFS incoming number (block, that dominates other).
void findAndSetMainInstruction(ArrayRef<Value *> VL, const BoUpSLP &R) {
BasicBlock *Parent = nullptr;
// Checks if the instruction has supported opcode.
auto IsSupportedInstruction = [&](Instruction *I, bool AnyUndef) {
if (AnyUndef && (I->isIntDivRem() || I->isFPDivRem() || isa<CallInst>(I)))
return false;
return I && isSupportedMainOp(I) &&
(!doesNotNeedToBeScheduled(I) || !R.isVectorized(I));
};
// Exclude operands instructions immediately to improve compile time, it
// will be unable to schedule anyway.
SmallDenseSet<Value *, 8> Operands;
SmallMapVector<unsigned, SmallVector<Instruction *>, 4> Candidates;
bool AnyUndef = false;
for (Value *V : VL) {
auto *I = dyn_cast<Instruction>(V);
if (!I) {
AnyUndef |= isa<UndefValue>(V);
continue;
}
if (!DT.isReachableFromEntry(I->getParent()))
continue;
if (Candidates.empty()) {
Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
Parent = I->getParent();
Operands.insert(I->op_begin(), I->op_end());
continue;
}
if (Parent == I->getParent()) {
Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
Operands.insert(I->op_begin(), I->op_end());
continue;
}
auto *NodeA = DT.getNode(Parent);
auto *NodeB = DT.getNode(I->getParent());
assert(NodeA && "Should only process reachable instructions");
assert(NodeB && "Should only process reachable instructions");
assert((NodeA == NodeB) ==
(NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeA->getDFSNumIn() < NodeB->getDFSNumIn()) {
Candidates.clear();
Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
Parent = I->getParent();
Operands.clear();
Operands.insert(I->op_begin(), I->op_end());
}
}
unsigned BestOpcodeNum = 0;
MainOp = nullptr;
bool UsedOutside = false;
for (const auto &P : Candidates) {
bool PUsedOutside = all_of(P.second, isUsedOutsideBlock);
if (UsedOutside && !PUsedOutside)
continue;
if (!UsedOutside && PUsedOutside)
BestOpcodeNum = 0;
if (P.second.size() < BestOpcodeNum)
continue;
// If have inner dependencies - skip.
if (!PUsedOutside && any_of(P.second, [&](Instruction *I) {
return Operands.contains(I);
}))
continue;
// On a tie, keep the outer binary op as MainOp rather than replacing it
// with an inner op that appears as its direct operand. For example, in
// (2.0f * A) + B the fadd and fmul each appear once in VL; without this
// check the fmul could win and prevent vectorization of the fadd pair.
if (P.second.size() == BestOpcodeNum) {
auto *I = P.second.front();
if (auto *MainBO = dyn_cast<BinaryOperator>(MainOp)) {
auto *MainBOOp0 = dyn_cast<Instruction>(MainBO->getOperand(0));
auto *MainBOOp1 = dyn_cast<Instruction>(MainBO->getOperand(1));
if (MainBOOp0 && MainBOOp0->getOpcode() == I->getOpcode() &&
MainBOOp0->getParent() == I->getParent())
continue;
if (MainBOOp1 && MainBOOp1->getOpcode() == I->getOpcode() &&
MainBOOp1->getParent() == I->getParent())
continue;
}
// Keep fmuladd over fmul/fadd on a tie only when every copyable is
// an absorbed fmul/fadd.
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) &&
(I->getOpcode() == Instruction::FMul ||
I->getOpcode() == Instruction::FAdd) &&
AbsorbCopyableFMulOrFAdds)
continue;
// Same check when fmuladd replaces fmul/fadd on a tie.
if ((MainOp->getOpcode() == Instruction::FMul ||
MainOp->getOpcode() == Instruction::FAdd) &&
RecurrenceDescriptor::isFMulAddIntrinsic(I) &&
!AbsorbCopyableFMulOrFAdds)
continue;
}
UsedOutside = PUsedOutside;
if (P.first == Instruction::GetElementPtr) {
// Only the GEPs of the main op shape (source element type and index
// type) are matching lanes, the others are copyable. Select the most
// frequent shape to minimize the number of copyable lanes.
SmallDenseMap<std::pair<Type *, Type *>, unsigned, 4> ShapeCounts;
auto GetShape = [](Instruction *I) {
return std::make_pair(cast<GEPOperator>(I)->getSourceElementType(),
I->getOperand(1)->getType());
};
for (Instruction *I : P.second)
if (IsSupportedInstruction(I, AnyUndef))
++ShapeCounts[GetShape(I)];
unsigned BestShapeNum = 0;
for (Instruction *I : P.second) {
if (!IsSupportedInstruction(I, AnyUndef))
continue;
unsigned ShapeNum = ShapeCounts.lookup(GetShape(I));
if (ShapeNum > BestShapeNum) {
MainOp = I;
BestOpcodeNum = P.second.size();
BestShapeNum = ShapeNum;
}
}
continue;
}
for (Instruction *I : P.second) {
if (IsSupportedInstruction(I, AnyUndef)) {
MainOp = I;
BestOpcodeNum = P.second.size();
break;
}
}
}
if (MainOp) {
// Do not match, if any copyable is a terminator from the same block as
// the main operation.
if (any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
return I->getParent() == MainOp->getParent() && I->isTerminator();
})) {
MainOp = nullptr;
return;
}
MainOpcode = MainOp->getOpcode();
}
}
/// Returns the idempotent value for MainOp. For Add, returns 0. For Or, it
/// should choose between false and the operand itself, since V or V == V.
/// For the min/max intrinsics, returns the corresponding limit value (e.g. 0
/// for umax, since umax(V, 0) == V). For smin and smax the type limit is
/// kept when it stays same after narrowing, otherwise the limit is deduced
/// from the signed width of V.
Value *selectBestIdempotentValue(Value *V) const {
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
if (auto *MinMax = dyn_cast<MinMaxIntrinsic>(MainOp);
MinMax && MinMax->isSigned() &&
!isTypeLimitNeutral(V, MinMax->getIntrinsicID() == Intrinsic::smax,
DL)) {
// TODO: add AC to both width queries once the analysis has one.
unsigned Width = ComputeMaxSignificantBits(V, DL, nullptr, nullptr, &DT);
APInt Limit = MinMax->getIntrinsicID() == Intrinsic::smax
? APInt::getSignedMinValue(Width)
: APInt::getSignedMaxValue(Width);
return Constant::getIntegerValue(
MainOp->getType(),
Limit.sext(MainOp->getType()->getScalarSizeInBits()));
}
return ConstantExpr::getIdentity(MainOp, MainOp->getType(),
!MainOp->isCommutative());
}
/// Returns the value and operands for the \p V, considering if it is original
/// instruction and its actual operands should be returned, or it is a
/// copyable element and its should be represented as idempotent instruction.
/// \p SelfOp selects the op(V, V) modeling for a copyable lane instead of
/// op(V, identity). \p PoisonPlaceholder selects op(V, poison) for the
/// absorbing constant V.
SmallVector<Value *> getOperands(const InstructionsState &S, Value *V,
bool SelfOp,
bool PoisonPlaceholder = false) const {
if (SelfOp)
return {V, V};
if (isa<PoisonValue>(V)) {
// Poison operands of the main op operand types: a GEP index or a cast
// source has a type, different from the lane type. Excludes the
// trailing callee operand of a call.
auto *CI = dyn_cast<CallInst>(MainOp);
return map_to_vector(
seq<unsigned>(CI ? CI->arg_size() : MainOp->getNumOperands()),
[&](unsigned Idx) -> Value * {
return PoisonValue::get(MainOp->getOperand(Idx)->getType());
});
}
if (!S.isCopyableElement(V))
return convertTo(cast<Instruction>(V), S).second;
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp)) {
Type *Ty = MainOp->getType();
if (S.hasAbsorbedCopyableFMulOrFAdd() &&
isAbsorbableCopyableFMulOrFAdd(S, V)) {
auto *I = cast<Instruction>(V);
// fmuladd(a, b, -0.0) == fmul a, b.
if (I->getOpcode() == Instruction::FMul)
return {I->getOperand(0), I->getOperand(1),
ConstantFP::getNegativeZero(Ty)};
// fmuladd(1.0, a, b) == fadd a, b.
return {ConstantFP::get(Ty, 1.0), I->getOperand(0), I->getOperand(1)};
}
// fmuladd(V, 1.0, -0.0) == V.
if (S.getCopyableOpIdx() == 0)
return {V, ConstantFP::get(Ty, 1.0), ConstantFP::getNegativeZero(Ty)};
// fmuladd(0.0, -0.0, V) == V.
return {ConstantFP::getZero(Ty), ConstantFP::getNegativeZero(Ty), V};
}
// gep V, 0 == V.
if (MainOpcode == Instruction::GetElementPtr)
return {V, ConstantInt::get(MainOp->getOperand(1)->getType(), 0)};
// zext(trunc(V)) == V for the round-trippable constant lanes.
if (MainOpcode == Instruction::ZExt)
return {ConstantExpr::getTrunc(cast<Constant>(V),
cast<CastInst>(MainOp)->getSrcTy())};
assert(isSupportedMainOp(MainOp) && "Unsupported opcode");
Type *Ty = V->getType();
if (PoisonPlaceholder &&
V == ConstantExpr::getBinOpAbsorber(MainOpcode, Ty))
return {V, PoisonValue::get(Ty)};
if (isa<UndefValue>(V) && isUndefTolerantBinOp(MainOp))
return {V, V};
return {V, selectBestIdempotentValue(V)};
}
/// Checks if the lanes with the absorbing constant may use poison as the
/// other operand: every other non-poison lane must be a binary operator with
/// one constant operand and an undef-tolerant binary operator of a distinct
/// load as the other operand.
bool canUsePoisonPlaceholders(const InstructionsState &S,
ArrayRef<Value *> VL) const {
Constant *Absorber =
ConstantExpr::getBinOpAbsorber(MainOpcode, VL.front()->getType());
if (!Absorber || !is_contained(VL, Absorber))
return false;
SmallPtrSet<const LoadInst *, 8> Loads;
return all_of(VL, [&](Value *V) {
if (isa<PoisonValue>(V) || S.isCopyableElement(V))
return true;
SmallVector<Value *> Ops = convertTo(cast<Instruction>(V), S).second;
return count_if(Ops, IsaPred<Constant>) == 1 &&
any_of(Ops, [&](Value *Op) {
const LoadInst *LI = getLoadOfUndefTolerantBinOp(Op);
return LI && Loads.insert(LI).second;
});
});
}
/// Returns the copyable lanes of an idempotent binop to be modeled as
/// op(V, V) == V instead of op(V, identity): when the operand column the
/// lane joins also holds a constant lane, the reorder would align that
/// column as constants and move the lane out, breaking the opcode-peer
/// node, while the symmetric form is reorder-proof. The other column must
/// be an instruction node so the lane is absorbed there as a copyable
/// rather than breaking a constant column. A lane is modeled this way only
/// when it has a strong opcode-peer pair of its own.
SmallPtrSet<const Value *, 4> findSelfOpLanes(const InstructionsState &S,
ArrayRef<Value *> VL,
const BoUpSLP &R) const {
SmallPtrSet<const Value *, 4> SelfOpLanes;
if (!Instruction::isIdempotent(MainOpcode) ||
!S.areInstructionsWithCopyableElements() || none_of(VL, [&](Value *V) {
if (isa<PoisonValue>(V))
return false;
if (S.isCopyableElement(V))
return isa<Constant>(V);
return isa<Constant>(cast<Instruction>(V)->getOperand(0));
}))
return SelfOpLanes;
// Index the instruction operands of the non-copyable lanes by opcode
// and block, so the copyable lanes find their opcode peers without a
// quadratic scan.
SmallDenseMap<std::pair<unsigned, BasicBlock *>, SmallVector<Value *>>
PeerOps;
for (Instruction *RI : make_isa_range<Instruction>(VL)) {
if (S.isCopyableElement(RI))
continue;
assert(RI->getNumOperands() == 2 && "Expected binary operation.");
for (unsigned OpIdx : seq<unsigned>(RI->getNumOperands())) {
auto *OpI = dyn_cast<Instruction>(RI->getOperand(OpIdx));
if (OpI &&
isa<Instruction>(RI->getOperand(RI->getNumOperands() - 1 - OpIdx)))
PeerOps[{OpI->getOpcode(), OpI->getParent()}].push_back(OpI);
}
}
for (Instruction *I : make_isa_range<Instruction>(VL)) {
if (!S.isCopyableElement(I))
continue;
auto It = PeerOps.find({I->getOpcode(), I->getParent()});
if (It == PeerOps.end())
continue;
SmallVector<std::pair<Value *, Value *>> Candidates;
for (Value *OpI : It->second)
Candidates.emplace_back(I, OpI);
if (R.findBestRootPair(Candidates).first)
SelfOpLanes.insert(I);
}
return SelfOpLanes;
}
/// Builds the index operands of a GEP node with the main op \p VL0. The
/// indices of the matching GEP lanes (for which \p IsGEPLane returns true)
/// are cast to the common index type, all other lanes (non-GEP pointers,
/// copyable and poison lanes, modeled as gep V, 0) get a zero index of that
/// type. Required to be able to find correct matches between different
/// gather nodes and reuse the vectorized values rather than trying to
/// gather them again. The vectorized nodes always have the common index
/// type (a legality invariant); the operands built for the profitability
/// analysis before that check may have none, the constants are cast to the
/// pointer index type then.
void buildGEPIndexOperands(ArrayRef<Value *> VL, Instruction *VL0,
function_ref<bool(Value *)> IsGEPLane,
BoUpSLP::ValueList &Indices) const {
constexpr unsigned IndexIdx = 1;
Type *Ty = getCommonGEPIndexType(VL, VL0, IsGEPLane, DL);
if (!Ty)
Ty = DL.getIndexType(VL0->getOperand(0)->getType()->getScalarType());
for (auto [Idx, V] : enumerate(VL)) {
if (!IsGEPLane(V)) {
Indices[Idx] = ConstantInt::getNullValue(Ty);
continue;
}
auto *Op = cast<GetElementPtrInst>(V)->getOperand(IndexIdx);
auto *CI = dyn_cast<ConstantInt>(Op);
Indices[Idx] = CI ? ConstantFoldIntegerCast(
CI, Ty, CI->getValue().isSignBitSet(), DL)
: Op;
}
}
/// Builds operands for the original instructions.
void
buildOriginalOperands(const InstructionsState &S, ArrayRef<Value *> VL,
SmallVectorImpl<BoUpSLP::ValueList> &Operands) const {
unsigned ShuffleOrOp =
S.isAltShuffle() ? (unsigned)Instruction::ShuffleVector : S.getOpcode();
Instruction *VL0 = S.getMainOp();
switch (ShuffleOrOp) {
case Instruction::PHI: {
auto *PH = cast<PHINode>(VL0);
// Keeps the reordered operands to avoid code duplication.
PHIHandler Handler(DT, PH, VL);
Handler.buildOperands();
Operands.assign(PH->getNumOperands(), {});
for (unsigned I : seq<unsigned>(PH->getNumOperands()))
Operands[I].assign(Handler.getOperands(I).begin(),
Handler.getOperands(I).end());
return;
}
case Instruction::ExtractValue: {
SmallVector<unsigned> Indices;
SmallVector<Value *> Calls;
if (checkEVsForVecCalls(VL, S, TLI, Indices, Calls)) {
Operands.assign(1, {});
Operands[0].swap(Calls);
return;
}
[[fallthrough]];
}
case Instruction::ExtractElement:
// This is a special case, as it does not gather, but at the same time
// we are not extending buildTree_rec() towards the operands.
Operands.assign(1, {VL.size(), VL0->getOperand(0)});
return;
case Instruction::InsertValue:
case Instruction::InsertElement:
Operands.assign(2, {VL.size(), nullptr});
for (auto [Idx, V] : enumerate(VL)) {
auto *IE = cast<Instruction>(V);
for (auto [OpIdx, Ops] : enumerate(Operands))
Ops[Idx] = IE->getOperand(OpIdx);
}
return;
case Instruction::Load:
Operands.assign(
1, {VL.size(),
PoisonValue::get(cast<LoadInst>(VL0)->getPointerOperandType())});
for (auto [V, Op] : zip(VL, Operands.back())) {
auto *LI = dyn_cast<LoadInst>(V);
if (!LI)
continue;
Op = LI->getPointerOperand();
}
return;
case Instruction::ICmp:
// With trunc X to i1 lanes the node is the bit tests of and A, B: the
// compares give the operands of their ands, trunc X gives X and 1.
if (any_of(VL, IsaPred<TruncInst>)) {
Type *Ty = VL0->getOperand(0)->getType();
Operands.assign(2, {VL.size(), PoisonValue::get(Ty)});
for (auto [Idx, V] : enumerate(VL)) {
if (auto *TI = dyn_cast<TruncInst>(V)) {
Operands[0][Idx] = TI->getOperand(0);
Operands[1][Idx] = ConstantInt::get(Ty, 1);
} else if (auto *CI = dyn_cast<ICmpInst>(V)) {
auto *And = cast<BinaryOperator>(CI->getOperand(0));
Operands[0][Idx] = And->getOperand(0);
Operands[1][Idx] = And->getOperand(1);
}
}
return;
}
[[fallthrough]];
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::PtrToAddr:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast:
case Instruction::FCmp:
case Instruction::FNeg:
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
case Instruction::Freeze:
case Instruction::Store:
case Instruction::ShuffleVector:
Operands.assign(VL0->getNumOperands(), {VL.size(), nullptr});
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I) {
for (auto [OpIdx, Ops] : enumerate(Operands))
Ops[Idx] = PoisonValue::get(VL0->getOperand(OpIdx)->getType());
continue;
}
SmallVector<Value *> ConvertedOps = convertTo(I, S).second;
for (auto [OpIdx, Ops] : enumerate(Operands))
Ops[Idx] = ConvertedOps[OpIdx];
}
return;
case Instruction::Select:
Operands.assign(VL0->getNumOperands(), {VL.size(), nullptr});
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I) {
for (auto [OpIdx, Ops] : enumerate(Operands))
Ops[Idx] = PoisonValue::get(VL0->getOperand(OpIdx)->getType());
continue;
}
if (isa<ZExtInst>(I)) {
// Special case for select + zext i1 to avoid explosion of different
// types. We want to keep the condition as i1 to be able to match
// different selects together and reuse the vectorized condition
// rather than trying to gather it.
Operands[0][Idx] = I->getOperand(0);
Operands[1][Idx] = ConstantInt::get(I->getType(), 1);
Operands[2][Idx] = ConstantInt::getNullValue(I->getType());
continue;
}
SmallVector<Value *> ConvertedOps = convertTo(I, S).second;
for (auto [OpIdx, Ops] : enumerate(Operands))
Ops[Idx] = ConvertedOps[OpIdx];
}
return;
case Instruction::GetElementPtr: {
Operands.assign(2, {VL.size(), nullptr});
for (auto [Idx, V] : enumerate(VL)) {
auto *GEP = dyn_cast<GetElementPtrInst>(V);
Operands[0][Idx] = GEP ? GEP->getPointerOperand() : V;
}
buildGEPIndexOperands(VL, VL0, IsaPred<GetElementPtrInst>, Operands[1]);
return;
}
case Instruction::Call: {
auto *CI = cast<CallInst>(VL0);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &TLI);
for (unsigned Idx : seq<unsigned>(CI->arg_size())) {
if (isVectorIntrinsicWithScalarOpAtArg(ID, Idx, &TTI))
continue;
auto &Ops = Operands.emplace_back();
for (Value *V : VL) {
auto *I = dyn_cast<Instruction>(V);
Ops.push_back(I ? I->getOperand(Idx)
: PoisonValue::get(VL0->getOperand(Idx)->getType()));
}
}
return;
}
default:
break;
}
llvm_unreachable("Unexpected vectorization of the instructions.");
}
/// Check if the specified \p VL list of values is better to represent as
/// uniform with copyables, as modeled via \p CopyableS, or as alternate (or
/// uniform with compatible ops), modeled via \p S.
/// Performs the analysis of the operands, choosing the preferred main
/// instruction and checking the matching of the operands for the main
/// instruction and copyable elements.
bool isCopyablePreferable(ArrayRef<Value *> VL, const BoUpSLP &R,
const InstructionsState &S,
const InstructionsState &CopyableS) {
// If all elements are vectorized already - keep as is.
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || R.isVectorized(V);
}))
return false;
Instruction *SMain = S.getMainOp();
Instruction *SAlt = S.isAltShuffle() ? S.getAltOp() : nullptr;
const bool IsCommutative = isCommutative(SMain);
const bool IsAltCommutative =
S.isAltShuffle() ? isCommutative(SAlt) : false;
const bool IsMainCommutative = isCommutative(MainOp);
SmallVector<BoUpSLP::ValueList> Ops;
buildOriginalOperands(S, SMain, Ops);
// Support only binary operations for now.
if (Ops.size() != 2)
return false;
// Try to find better candidate for S main instruction, which operands have
// better matching.
auto CheckOperands = [](Value *Op, Value *SMainOp) {
auto *OpI = dyn_cast<BinaryOperator>(Op);
if (!OpI)
return false;
auto *SMainOpI = dyn_cast<BinaryOperator>(SMainOp);
if (!SMainOpI)
return true;
return any_of(make_isa_range<Instruction>(OpI->operands()),
[&](Instruction *I) {
return I->getOpcode() == SMainOpI->getOpcode();
});
};
SmallPtrSet<Value *, 8> Operands;
for (Instruction *I : make_isa_range<Instruction>(VL)) {
if (I == SMain)
continue;
Instruction *MatchingOp = S.getMatchingMainOpOrAltOp(I);
if (MatchingOp != SMain)
continue;
SmallVector<BoUpSLP::ValueList> VOps;
buildOriginalOperands(S, I, VOps);
Operands.insert(I->op_begin(), I->op_end());
assert(VOps.size() == 2 && Ops.size() == 2 &&
"Expected binary operations only.");
if (CheckOperands(VOps[0][0], Ops[0][0]) ||
CheckOperands(VOps[1][0], Ops[1][0]) ||
(IsCommutative && (CheckOperands(VOps[0][0], Ops[1][0]) ||
CheckOperands(VOps[1][0], Ops[0][0])))) {
SMain = I;
Ops.swap(VOps);
break;
}
}
SmallVector<BoUpSLP::ValueList> MainOps;
buildOriginalOperands(S, MainOp, MainOps);
auto BuildFirstOperandCandidates =
[&](SmallVectorImpl<std::pair<Value *, Value *>> &Candidates,
ArrayRef<BoUpSLP::ValueList> Ops, Value *Op0, Value *Op1,
bool IsCommutative) {
Candidates.emplace_back(Ops[0][0], Op0);
if (IsCommutative)
Candidates.emplace_back(Ops[0][0], Op1);
};
auto BuildSecondOperandCandidates =
[&](SmallVectorImpl<std::pair<Value *, Value *>> &Candidates,
ArrayRef<BoUpSLP::ValueList> Ops, int PrevBestIdx, Value *Op0,
Value *Op1, bool IsCommutative) {
if (PrevBestIdx != 1)
Candidates.emplace_back(Ops[1][0], Op1);
if (PrevBestIdx != 0 && IsCommutative)
Candidates.emplace_back(Ops[1][0], Op0);
};
auto FindBestCandidate =
[&](ArrayRef<std::pair<Value *, Value *>> Candidates, bool &IsConst,
int &Score) {
auto Res = R.findBestRootPair(Candidates);
Score = Res.second;
IsConst =
Res.second == BoUpSLP::LookAheadHeuristics::ScoreConstants &&
isConstant(Candidates[Res.first.value_or(0)].first) &&
isConstant(Candidates[Res.first.value_or(0)].second);
if (IsConst) {
// Check if there are splat candidates and consider them better
// option.
for (const auto [Idx, P] : enumerate(Candidates)) {
if (!isConstant(P.first) && !isConstant(P.second) &&
P.second == P.first) {
Res.first = Idx;
IsConst = false;
Score = isa<LoadInst>(Candidates[Res.first.value_or(0)].first)
? BoUpSLP::LookAheadHeuristics::ScoreSplatLoads
: BoUpSLP::LookAheadHeuristics::ScoreSplat;
break;
}
}
}
return Res.first;
};
for (Instruction *I : make_isa_range<Instruction>(VL)) {
if (I == SMain && (I == MainOp || !S.isAltShuffle()))
continue;
SmallVector<BoUpSLP::ValueList> VOps;
buildOriginalOperands(S, I == SMain ? MainOp : I, VOps);
SmallVector<Value *> CopyableOps =
getOperands(CopyableS, I == MainOp ? SMain : I, /*SelfOp=*/false);
if (CopyableOps.size() == VOps.size() &&
all_of(zip(CopyableOps, VOps), [&](const auto &P) {
return std::get<0>(P) == std::get<1>(P)[0];
}))
continue;
SmallVector<std::pair<Value *, Value *>> Candidates;
BuildFirstOperandCandidates(Candidates, MainOps, CopyableOps[0],
CopyableOps[1], IsMainCommutative);
const unsigned OpSize = Candidates.size();
Instruction *MatchingOp =
S.getMatchingMainOpOrAltOp(I) == S.getMainOp() ? SMain : SAlt;
const bool IsCommutativeInst =
(MatchingOp == SMain ? IsCommutative : IsAltCommutative) ||
isCommutative(I, MatchingOp);
if (S.isAltShuffle() && MatchingOp == SAlt &&
any_of(VOps, [&](const BoUpSLP::ValueList &Ops) {
auto *I = dyn_cast<BinaryOperator>(Ops[0]);
return I && Operands.contains(I);
}))
return false;
if (S.isAltShuffle() && MatchingOp == SMain)
Operands.insert(I->op_begin(), I->op_end());
BuildFirstOperandCandidates(Candidates, Ops, VOps[0][0], VOps[1][0],
IsCommutativeInst);
bool IsBestConst;
int Score;
std::optional<int> BestOp =
FindBestCandidate(Candidates, IsBestConst, Score);
const bool IsOriginalBetter =
static_cast<unsigned>(BestOp.value_or(OpSize)) >= OpSize;
Candidates.clear();
BuildSecondOperandCandidates(
Candidates, MainOps, IsOriginalBetter ? -1 : *BestOp, CopyableOps[0],
CopyableOps[1], IsMainCommutative);
const unsigned SecondOpSize = Candidates.size();
BuildSecondOperandCandidates(
Candidates, Ops,
IsOriginalBetter ? BestOp.value_or(OpSize - 1) - OpSize : -1,
VOps[0][0], VOps[1][0], IsCommutativeInst);
bool IsSecondBestConst;
int SecondScore;
std::optional<int> SecondBestOp =
FindBestCandidate(Candidates, IsSecondBestConst, SecondScore);
// No best candidates.
if (!BestOp && !SecondBestOp)
return false;
// Original better in both ops combinations.
const bool IsSecondOriginalBetter =
static_cast<unsigned>(SecondBestOp.value_or(SecondOpSize)) >=
SecondOpSize;
if (IsOriginalBetter && IsSecondOriginalBetter)
return false;
// Original is better in second combination, but in the first combination
// no best candidates.
if (!BestOp && IsSecondOriginalBetter)
return false;
// Original is better in first combination, but in the second combination
// no best candidates.
if (!SecondBestOp && IsOriginalBetter)
return false;
// Copyable is best in the first combination, but it is constant, but
// original is better in second non-constant combination.
if (!IsOriginalBetter && IsBestConst && IsSecondOriginalBetter &&
!IsSecondBestConst)
return false;
// Copyable is best in the second combination, but it is constant, but
// original is better in the first non-constant combination.
if (BestOp && IsOriginalBetter && !IsBestConst &&
!IsSecondOriginalBetter && IsSecondBestConst)
return false;
// Original combination score is better.
if (((Score > SecondScore ||
(Score <= BoUpSLP::LookAheadHeuristics::ScoreAltOpcodes &&
Score == SecondScore)) &&
IsOriginalBetter) ||
(IsSecondOriginalBetter &&
(SecondScore > Score ||
(Score <= BoUpSLP::LookAheadHeuristics::ScoreAltOpcodes &&
Score == SecondScore))))
return false;
}
return true;
}
public:
InstructionsCompatibilityAnalysis(DominatorTree &DT, const DataLayout &DL,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI)
: DT(DT), DL(DL), TTI(TTI), TLI(TLI) {}
InstructionsState buildInstructionsState(ArrayRef<Value *> VL,
const BoUpSLP &R,
bool WithProfitabilityCheck = false,
bool SkipSameCodeCheck = false) {
InstructionsState S = (SkipSameCodeCheck || !allSameBlock(VL))
? InstructionsState::invalid()
: getSameOpcode(VL, TLI);
// Check if series of selects + zext i1 %x to in can be combined into
// selects + select %x, i32 1, i32 0.
Instruction *SelectOp = nullptr;
if (!S && allSameBlock(VL) && all_of(VL, [&](Value *V) {
if (match(V, m_Select(m_Value(), m_Value(), m_Value()))) {
if (!SelectOp)
SelectOp = cast<Instruction>(V);
return true;
}
auto *ZExt = dyn_cast<ZExtInst>(V);
return (ZExt && ZExt->getSrcTy()->isIntegerTy(1)) ||
isa<PoisonValue>(V);
})) {
if (SelectOp)
return InstructionsState(SelectOp, SelectOp);
}
// The undef lanes among the loads are the copyable elements, covered by a
// single wide load.
if (!S && R.Opts.slp_copyable_elements &&
all_of(VL, IsaPred<LoadInst, UndefValue>) && any_of(VL, [](Value *V) {
return isa<UndefValue>(V) && !isa<PoisonValue>(V);
})) {
SmallVector<Value *> Loads(make_isa_range<LoadInst>(VL));
if (Loads.size() > 1 && allSameBlock(Loads)) {
auto *LI = cast<LoadInst>(Loads.front());
return InstructionsState(LI, LI, /*HasCopyables=*/true);
}
}
if (S && S.isAltShuffle()) {
Type *ScalarTy = S.getMainOp()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
unsigned Opcode0 = S.getOpcode();
unsigned Opcode1 = S.getAltOpcode();
SmallBitVector OpcodeMask(
getAltInstrMask(VL, ScalarTy, Opcode0, Opcode1));
// If this pattern is supported by the target then we consider the order.
if (TTI.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask))
return S;
} else if (S &&
(!R.Opts.slp_copyable_elements ||
!isSupportedMainOp(S.getMainOp()) ||
all_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
return I->getOpcode() == S.getOpcode() ||
(S.getOpcode() == Instruction::Add &&
I->getOpcode() == Instruction::Shl);
}))) {
return S;
}
if (!R.Opts.slp_copyable_elements)
return S;
AbsorbCopyableFMulOrFAdds = hasOnlyAbsorbableCopyableFMulOrFAdds(VL);
findAndSetMainInstruction(VL, R);
if (!MainOp)
return S;
InstructionsState OrigS = S;
S = InstructionsState(MainOp, MainOp, /*HasCopyables=*/true);
// Cast nodes support only matching lanes from the main op block and
// copyable (round-trippable constant) lanes.
if (isa<CastInst>(MainOp) && any_of(VL, [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
return !isa<PoisonValue>(V) &&
!(I && I->getOpcode() == MainOpcode &&
I->getParent() == MainOp->getParent()) &&
!S.isCopyableElement(V);
}))
return OrigS;
if (OrigS && !isCopyablePreferable(VL, R, OrigS, S))
return OrigS;
if (!WithProfitabilityCheck)
return S;
// ExtractElement copyable nodes reuse the source vector and insert the
// copyable lanes; the binary-operator operand heuristics below do not
// apply, so defer profitability to the full tree cost.
if (isa<ExtractElementInst>(MainOp)) {
// A load as a copyable lane would be pulled out of the consecutive-load
// vectorization; keep the original state so the node can be split.
if (any_of(VL, [&](Value *V) {
return S.isCopyableElement(V) && isa<LoadInst>(V);
}))
return OrigS;
return S;
}
// Check if it is profitable to vectorize the instruction.
unsigned CopyableNum =
count_if(VL, [&](Value *V) { return S.isCopyableElement(V); });
// A cast copyable node just extends the sources of the matching lanes:
// the constant lanes are free in a gather of the original values, so it
// cannot be better unless the matching lanes are the majority and are not
// vectorized already (the gather reuses the vector otherwise).
if (isa<CastInst>(MainOp) &&
(CopyableNum * 2 >= VL.size() || none_of(VL, [&](Value *V) {
return !S.isCopyableElement(V) && !isa<PoisonValue>(V) &&
!R.isVectorized(V);
})))
return OrigS;
// Absorb copyable single-use fmuls/fadds as fmuladd(a, b, -0.0) or
// fmuladd(1.0, a, b) when every copyable is such a binop: the binops die
// instead of being computed and gathered.
if (RecurrenceDescriptor::isFMulAddIntrinsic(MainOp) &&
AbsorbCopyableFMulOrFAdds)
S.setAbsorbCopyableFMulOrFAdd(true);
SmallVector<BoUpSLP::ValueList> Operands = buildOperands(S, VL, R);
auto CheckOperand = [&](ArrayRef<Value *> Ops) {
if (allConstant(Ops) || isSplat(Ops))
return true;
// Non-instruction operands of a call (args, constants) are always a
// trivial gather, same as the constant/splat cases above.
if (MainOpcode == Instruction::Call && none_of(Ops, IsaPred<Instruction>))
return true;
// Check if it is "almost" splat, i.e. has >= 4 elements and only single
// one is different.
constexpr unsigned Limit = 4;
if (Operands.front().size() >= Limit) {
SmallDenseMap<const Value *, unsigned> Counters;
auto IsNotUndef = [](Value *V) { return !isa<UndefValue>(V); };
for (Value *V : make_filter_range(Ops, IsNotUndef))
++Counters[V];
if (Counters.size() == 2 &&
any_of(Counters, [&](const auto &C) { return C.second == 1; }))
return true;
}
// First operand not a constant or splat? Last attempt - check for
// potential vectorization.
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
InstructionsState OpS = Analysis.buildInstructionsState(Ops, R);
if (!OpS || (OpS.getOpcode() == Instruction::PHI && !allSameBlock(Ops)))
return false;
unsigned CopyableNum =
count_if(Ops, [&](Value *V) { return OpS.isCopyableElement(V); });
return CopyableNum <= VL.size() / 2;
};
// A copyable GEP node is a vector GEP over the base pointers of the
// matching lanes and the values of the copyable lanes. It is profitable
// only if that base vector is cheap (a splat, constants or vectorizable),
// otherwise it is a gather not cheaper than the gather of the pointers.
if (MainOpcode == Instruction::GetElementPtr &&
!CheckOperand(Operands.front()))
return OrigS;
auto BuildCandidates =
[](SmallVectorImpl<std::pair<Value *, Value *>> &Candidates, Value *V1,
Value *V2) {
if (V1 != V2 && isa<PHINode>(V1))
return;
auto *I1 = dyn_cast<Instruction>(V1);
auto *I2 = dyn_cast<Instruction>(V2);
if (I1 && I2 && I1->getOpcode() == I2->getOpcode() &&
I1->getParent() != I2->getParent())
return;
Candidates.emplace_back(V1, (I1 || I2) ? V2 : V1);
};
if (VL.size() == 2) {
// The operand-pairing heuristic below does not apply to calls, GEPs
// and casts; defer to the full tree cost computation instead of
// pre-rejecting here.
if (MainOpcode == Instruction::Call ||
MainOpcode == Instruction::GetElementPtr || isa<CastInst>(MainOp))
return S;
// Check if the operands allow better vectorization.
SmallVector<std::pair<Value *, Value *>, 4> Candidates1, Candidates2;
BuildCandidates(Candidates1, Operands[0][0], Operands[0][1]);
BuildCandidates(Candidates2, Operands[1][0], Operands[1][1]);
bool Res = !Candidates1.empty() && !Candidates2.empty() &&
R.findBestRootPair(Candidates1).first &&
R.findBestRootPair(Candidates2).first;
if (!Res && isCommutative(MainOp)) {
Candidates1.clear();
Candidates2.clear();
BuildCandidates(Candidates1, Operands[0][0], Operands[1][1]);
BuildCandidates(Candidates2, Operands[1][0], Operands[0][1]);
Res = !Candidates1.empty() && !Candidates2.empty() &&
R.findBestRootPair(Candidates1).first &&
R.findBestRootPair(Candidates2).first;
}
if (!Res)
return OrigS;
const TTI::TargetCostKind Kind = R.getCostKind();
InstructionCost ScalarCost = TTI.getInstructionCost(S.getMainOp(), Kind);
InstructionCost VectorCost;
auto *VecTy = getWidenedType(S.getMainOp()->getType(), VL.size());
switch (MainOpcode) {
case Instruction::Add:
case Instruction::Sub:
case Instruction::Mul:
case Instruction::AShr:
case Instruction::LShr:
case Instruction::Shl:
case Instruction::SDiv:
case Instruction::UDiv:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
case Instruction::FAdd:
case Instruction::FMul:
case Instruction::FSub:
case Instruction::FDiv:
VectorCost = TTI.getArithmeticInstrCost(
MainOpcode, VecTy, Kind,
TTI::commonOperandInfo(Operands[0][0], Operands[0][1]),
TTI::commonOperandInfo(Operands[1][0], Operands[1][1]), {},
S.getMainOp(), &TLI);
break;
default:
// Calls (min/max, fmuladd) return above before reaching this switch.
llvm_unreachable("Unexpected instruction.");
}
if (VectorCost > ScalarCost)
return OrigS;
return S;
}
// Casts are the only 1-operand and fmuladd is the only 3-operand
// copyable.
assert((Operands.size() == 2 ||
(Operands.size() == 1 && isa<CastInst>(MainOp)) ||
(Operands.size() == 3 &&
RecurrenceDescriptor::isFMulAddIntrinsic(MainOp))) &&
"Unexpected number of operands!");
if (CopyableNum < VL.size() / 2)
return S;
// Too many phi copyables - exit.
const unsigned Limit = VL.size() / 24;
if ((CopyableNum >= VL.size() - Limit ||
(CopyableNum >= VL.size() - 1 && VL.size() > 4) ||
CopyableNum >= MaxPHINumOperands) &&
all_of(VL, [&](Value *V) {
return isa<PHINode>(V) || !S.isCopyableElement(V);
}))
return OrigS;
// Check profitability if number of copyables > VL.size() / 2.
// 1. Reorder operands for better matching.
if (Operands.size() == 2 && isCommutative(MainOp)) {
Value *BestFrontOp = nullptr;
for (auto [OpL, OpR] : zip(Operands.front(), Operands.back())) {
// Make instructions the first operands.
if (!isa<Instruction>(OpL) && isa<Instruction>(OpR)) {
BestFrontOp = OpR;
std::swap(OpL, OpR);
continue;
}
// Make constants the second operands.
if ((isa<Constant>(OpL) && !match(OpR, m_Zero())) ||
match(OpL, m_Zero())) {
if (isa<Instruction>(OpR))
BestFrontOp = OpR;
std::swap(OpL, OpR);
continue;
}
if (isa<Instruction>(OpL))
BestFrontOp = OpL;
}
// If some of the RHS operands better match most of LHS - swap such
// operands to increase matching rate.
if (auto *BestLHS = dyn_cast_if_present<Instruction>(BestFrontOp)) {
const unsigned BestOpcode = BestLHS->getOpcode();
for (auto [OpL, OpR] : zip(Operands.front(), Operands.back())) {
auto *OpRI = dyn_cast<Instruction>(OpR);
if (!OpRI)
continue;
if (OpRI->getOpcode() == BestOpcode)
std::swap(OpL, OpR);
}
}
}
// 2. Check, if operands can be vectorized. Skip for fmuladd; the
// copyable operand is checked below and may legitimately hold many
// instructions.
if (Operands.size() == 2 &&
count_if(Operands.back(), IsaPred<Instruction>) > 1)
return OrigS;
// Check the operand holding the copyable values.
if (!CheckOperand(Operands[S.getCopyableOpIdx()])) {
if (Operands.size() == 2)
return OrigS;
// Retry with the copyable modeled as the first multiplicand.
S.setCopyableOpIdx(0);
Operands = buildOperands(S, VL, R);
if (!CheckOperand(Operands[S.getCopyableOpIdx()]))
return OrigS;
}
return S;
}
SmallVector<BoUpSLP::ValueList>
buildOperands(const InstructionsState &S, ArrayRef<Value *> VL,
const BoUpSLP &R, bool AllowPoisonPlaceholders = true) {
assert(S && "Invalid state!");
SmallVector<BoUpSLP::ValueList> Operands;
if (S.areInstructionsWithCopyableElements()) {
MainOp = S.getMainOp();
MainOpcode = S.getOpcode();
// ExtractElement copyable nodes carry a single operand (the shared
// source vector); copyable lanes are inserted during codegen and do
// not contribute an operand column.
if (MainOpcode == Instruction::ExtractElement) {
Operands.assign(1,
BoUpSLP::ValueList(VL.size(), MainOp->getOperand(0)));
return Operands;
}
// The undef lanes of the loads have no pointer operands.
if (MainOpcode == Instruction::Load) {
buildOriginalOperands(S, VL, Operands);
return Operands;
}
// Excludes the trailing callee operand (2 for min/max, 3 for fmuladd).
// getNumberOfPotentiallyCommutativeOps collapses fmuladd to 2 and must
// not be used here. Only the 2-operand case is commutative-normalized.
auto *CI = dyn_cast<CallInst>(MainOp);
const unsigned NumMainOpOperands =
CI ? CI->arg_size() : MainOp->getNumOperands();
const bool IsCommutative =
isCommutative(MainOp) && NumMainOpOperands == 2;
Operands.assign(NumMainOpOperands,
BoUpSLP::ValueList(VL.size(), nullptr));
SmallPtrSet<const Value *, 4> SelfOpLanes = findSelfOpLanes(S, VL, R);
const bool PoisonPlaceholders =
AllowPoisonPlaceholders && canUsePoisonPlaceholders(S, VL);
// Populate operands for every lane.
for (auto [Idx, V] : enumerate(VL)) {
SmallVector<Value *> OperandsForValue =
getOperands(S, V, SelfOpLanes.contains(V), PoisonPlaceholders);
for (auto [OperandIdx, Operand] : enumerate(OperandsForValue))
Operands[OperandIdx][Idx] = Operand;
}
// The copyable lanes of a GEP node are gep V, 0; cast the indices of
// the matching lanes and the zeroes to the common index type.
if (MainOpcode == Instruction::GetElementPtr)
buildGEPIndexOperands(
VL, MainOp,
[&](Value *V) {
return isa<GetElementPtrInst>(V) && !S.isCopyableElement(V);
},
Operands[1]);
// Operand-order normalization below swaps OpIdx 0 and OpIdx 1
// of non-copyable lanes. That is only safe when the main op is
// commutative (e.g. 0 - X is not X - 0, so `sub` must be
// excluded). With absorbed fmul/fadd copyables the fmuladd
// multiplicands are commutative per lane and get normalized too;
// the 0/1 swaps never touch the addend column.
if (IsCommutative || S.hasAbsorbedCopyableFMulOrFAdd()) {
// IsCommutative can hold for MainOp (e.g. a Sub/FSub feeding only
// fabs/icmp-eq-0) without every lane sharing that property, so
// re-check the specific lane before swapping it. Absorbed fmul/fadd
// lanes are always commutative.
auto CanSwap = [&](Value *V) {
if (S.hasAbsorbedCopyableFMulOrFAdd() &&
isAbsorbableCopyableFMulOrFAdd(S, V))
return true;
return isCommutative(S.getMatchingMainOpOrAltOp(cast<Instruction>(V)),
V);
};
// Absorbed fmul/fadd copyables do not vote for the majority operand
// pattern (their multiplicand order is arbitrary) but take part
// in the swaps.
auto SwappableLane = [&](Value *V) {
return !isa<PoisonValue>(V) &&
(!S.isCopyableElement(V) ||
(S.hasAbsorbedCopyableFMulOrFAdd() &&
isAbsorbableCopyableFMulOrFAdd(S, V)));
};
// Count (ID0, ID1) pair frequencies for operand normalization.
// Pairs and their inverses are tracked under a canonical key
// so that (Load, Add) and (Add, Load) contribute to the same
// bucket.
struct PairInfo {
unsigned FwdCount = 0;
unsigned RevCount = 0;
};
SmallMapVector<std::pair<unsigned, unsigned>, PairInfo, 8> PairCounts;
SmallMapVector<unsigned, unsigned, 4> AddendIDCounts;
unsigned MajID0 = 0, MajID1 = 0;
for (const auto &P :
make_filter_range(enumerate(VL), [&](const auto &P) {
return !S.isCopyableElement(P.value()) &&
!isa<PoisonValue>(P.value());
})) {
size_t Idx = P.index();
unsigned ID0 = Operands[0][Idx]->getValueID();
unsigned ID1 = Operands[1][Idx]->getValueID();
if (S.hasAbsorbedCopyableFMulOrFAdd())
++AddendIDCounts[Operands[2][Idx]->getValueID()];
if (ID0 == ID1)
continue;
unsigned MinID = std::min(ID0, ID1);
unsigned MaxID = std::max(ID0, ID1);
auto *It = PairCounts.try_emplace(std::make_pair(MinID, MaxID)).first;
PairInfo &Info = It->second;
if (ID0 < ID1)
++Info.FwdCount;
else
++Info.RevCount;
}
// Find the most frequent (ID0, ID1) pair across non-copyable
// lanes. Select the orientation (original or inverse) that
// has more votes as the majority pattern.
unsigned BestCount = 0;
for (const auto &P : PairCounts) {
const PairInfo &Info = P.second;
unsigned Total = Info.FwdCount + Info.RevCount;
if (Total > BestCount) {
BestCount = Total;
if (Info.FwdCount >= Info.RevCount) {
MajID0 = P.first.first;
MajID1 = P.first.second;
} else {
MajID0 = P.first.second;
MajID1 = P.first.first;
}
}
}
// Absorbed fadd copyables are fmuladd(1.0, a, b): pick the
// addend/multiplicand assignment matching the majority operand
// kinds of the non-copyable lanes in the multiplicand (1) and
// addend (2) columns.
if (S.hasAbsorbedCopyableFMulOrFAdd()) {
unsigned MajID2 = 0, Best2 = 0;
for (const auto &P : AddendIDCounts) {
if (P.second > Best2) {
Best2 = P.second;
MajID2 = P.first;
}
}
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I || I->getOpcode() != Instruction::FAdd ||
!isAbsorbableCopyableFMulOrFAdd(S, I))
continue;
unsigned ID1 = Operands[1][Idx]->getValueID();
unsigned ID2 = Operands[2][Idx]->getValueID();
unsigned Cur = (ID1 == MajID1) + (ID2 == MajID2);
unsigned Swapped = (ID2 == MajID1) + (ID1 == MajID2);
if (Swapped > Cur)
std::swap(Operands[1][Idx], Operands[2][Idx]);
}
}
// Normalize swappable lanes in two steps:
// 1) Swap lanes whose operand types are the exact inverse of
// the majority pattern, making the non-copyable lanes
// consistent.
// 2) Independently, if a strict majority of swappable lanes
// have loads at OpIdx 1, swap those lanes to put loads at
// OpIdx 0 for better downstream vectorization.
unsigned LAt0 = 0, LAt1 = 0, TotalNC = 0;
for (auto [Idx, V] :
make_filter_range(enumerate(VL), [&](const auto &P) {
return SwappableLane(P.value());
})) {
// Step 1: swap exact-inverse lanes.
if (BestCount > 0) {
unsigned ID0 = Operands[0][Idx]->getValueID();
unsigned ID1 = Operands[1][Idx]->getValueID();
if (ID0 == MajID1 && ID1 == MajID0 && CanSwap(V))
std::swap(Operands[0][Idx], Operands[1][Idx]);
}
++TotalNC;
LAt0 += isa<LoadInst>(Operands[0][Idx]);
LAt1 += isa<LoadInst>(Operands[1][Idx]);
}
// Step 2: if most non-copyable lanes have loads at OpIdx 1,
// swap those lanes to put loads at OpIdx 0.
if (TotalNC > 1 && LAt1 > LAt0 && LAt1 * 2 > TotalNC) {
for (auto [Idx, V] :
make_filter_range(enumerate(VL), [&](const auto &P) {
return SwappableLane(P.value());
})) {
if (!isa<LoadInst>(Operands[0][Idx]) &&
isa<LoadInst>(Operands[1][Idx]) && CanSwap(V))
std::swap(Operands[0][Idx], Operands[1][Idx]);
}
}
}
} else {
buildOriginalOperands(S, VL, Operands);
}
return Operands;
}
};
} // namespace
void BoUpSLP::tryToVectorizeSplatGatheredScalars() {
unsigned PrevTreeSize = VectorizableTree.size();
auto LoadsSubkey = [](size_t /*Key*/, LoadInst *LI) {
return hash_value(getUnderlyingObject(LI->getPointerOperand()));
};
// The key includes the value type: the opcode-based key does not always
// distinguish types (e.g. all extractvalue instructions share one key),
// while the tree requires same-typed scalars.
SmallMapVector<std::tuple<size_t, size_t, Type *>, SmallSetVector<Value *, 4>,
4>
Groups;
// Only gathers with vectorized (non-gather) users can reuse the broadcast.
for (const std::unique_ptr<TreeEntry> &TE : make_filter_range(
VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && TE->UserTreeIndex &&
!TE->UserTreeIndex.UserTE->isGather() &&
isSplat(TE->Scalars);
})) {
auto *I = dyn_cast<Instruction>(TE->Scalars.front());
// Skip shuffle-like instructions: their splat gathers are already emitted
// as cheap shuffles of the source vector.
if (!I ||
isa<ExtractElementInst, InsertElementInst, ShuffleVectorInst>(I) ||
I->getType()->isVoidTy() || isVectorized(I) || isDeleted(I) ||
(UserIgnoreList && UserIgnoreList->contains(I)))
continue;
// Scheduling new memory bundles in a to-be-versioned tree records extra
// alias-check pairs and can push the region over the versioning limits.
if (isTryingRuntimeAliasChecks() && I->mayReadOrWriteMemory())
continue;
auto [Key, SubKey] =
generateKeySubkey(I, TLI, LoadsSubkey, /*AllowAlternate=*/true);
Groups[std::make_tuple(Key, SubKey, I->getType())].insert(I);
}
// Values left in singleton groups cannot form a bundle on their own;
// regroup them by the opcode-insensitive key so alternate/copyable
// bundles still form.
SmallMapVector<std::pair<size_t, Type *>, SmallSetVector<Value *, 4>, 4>
FallbackGroups;
for (auto &[Key, Group] : make_filter_range(
Groups, [](const auto &P) { return P.second.size() < 2; })) {
FallbackGroups[std::make_pair(std::get<0>(Key), Group.front()->getType())]
.insert(Group.front());
}
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
auto BuildSubtree = [&](const auto &GroupMap) {
auto IsMultiElement = [](const auto &P) { return P.second.size() >= 2; };
for (const auto &Entry : make_filter_range(GroupMap, IsMultiElement)) {
const auto &Group = Entry.second;
// Copyable-aware check so bundles with copyable lanes are not skipped.
if (!Analysis.buildInstructionsState(Group.getArrayRef(), *this))
continue;
unsigned PrevSize = VectorizableTree.size();
buildTreeRec(Group.getArrayRef(), 0, EdgeInfo());
if (PrevSize == VectorizableTree.size())
continue;
TreeEntry *NewRoot = VectorizableTree[PrevSize].get();
if (NewRoot->isGather()) {
// Failed to vectorize the bundle: drop the added gather entry, it has
// no users and only adds cost.
for (Value *V : NewRoot->Scalars) {
auto It = ValueToGatherNodes.find(V);
if (It != ValueToGatherNodes.end())
It->second.remove(NewRoot);
}
LoadEntriesToVectorize.remove(PrevSize);
VectorizableTree.pop_back();
continue;
}
// The root has no users, the gathers reuse its lanes by value: keep the
// loads in memory order instead of permuting them into the group order.
if (NewRoot->State == TreeEntry::Vectorize &&
NewRoot->getOpcode() == Instruction::Load)
NewRoot->ReorderIndices.clear();
SplatGatheredScalarsRoots.push_back(NewRoot);
}
};
BuildSubtree(Groups);
BuildSubtree(FallbackGroups);
NumCanonicalSplatSubtreeEntries = VectorizableTree.size() - PrevTreeSize;
}
BoUpSLP::ScalarsVectorizationLegality
BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx) const {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
InstructionsState S = Analysis.buildInstructionsState(
VL, *this, /*WithProfitabilityCheck=*/true);
bool AreScatterAllGEPSameBlock = false;
if (!S) {
SmallVector<unsigned> SortedIndices;
BasicBlock *BB = nullptr;
bool IsScatterVectorizeUserTE =
UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
AreScatterAllGEPSameBlock =
(IsScatterVectorizeUserTE && VL.front()->getType()->isPointerTy() &&
VL.size() > 2 &&
all_of(VL,
[&BB](Value *V) {
auto *I = dyn_cast<GetElementPtrInst>(V);
if (!I)
return doesNotNeedToBeScheduled(V);
if (!BB)
BB = I->getParent();
return BB == I->getParent() && I->getNumOperands() == 2;
}) &&
BB &&
sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL,
*SE, SortedIndices));
if (!AreScatterAllGEPSameBlock) {
LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
"C,S,B,O, small shuffle. \n";
dbgs() << "[";
interleaveComma(VL, dbgs(), [&](Value *V) { dbgs() << *V; });
dbgs() << "]\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/true,
/*TrySplitVectorize=*/true);
}
// Reset S to make it GetElementPtr kind of node.
const auto *It = find_if(VL, IsaPred<GetElementPtrInst>);
assert(It != VL.end() && "Expected at least one GEP.");
S = getSameOpcode(*It, *TLI);
}
assert(S && "Must be valid.");
// Gather very wide PHI bundles. Wide PHIs (e.g. produced by
// jump threading) are not profitable to vectorize and make this analysis
// explode, so gather them to keep the compile time bounded.
if (S.getOpcode() == Instruction::PHI) {
unsigned NumIncomingValues =
cast<PHINode>(S.getMainOp())->getNumIncomingValues();
if (static_cast<uint64_t>(VL.size()) * NumIncomingValues >
Opts.slp_phi_vectorization_budget) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to wide PHI operand fan-out ("
<< VL.size() << " lanes x " << NumIncomingValues
<< " incoming values).\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
// Don't handle vectors.
if (!Opts.slp_revec &&
getValueType(VL.front(), Opts.slp_revec)->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
// Do not try to pack to avoid extra instructions here.
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}
// Check that all of the users of the scalars that we want to vectorize are
// schedulable.
BasicBlock *BB = S.getMainOp()->getParent();
if (BB->isEHPad() || isa<UnreachableInst>(BB->getTerminator()) ||
!DT->isReachableFromEntry(BB)) {
// Don't go into unreachable blocks. They may contain instructions with
// dependency cycles which confuse the final scheduling.
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't go into catchswitch blocks, which can happen with PHIs.
// Such blocks can only have PHIs and the catchswitch. There is no
// place to insert a shuffle if we need to, so just avoid that issue.
if (isa<CatchSwitchInst>(BB->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
// Do not try to pack to avoid extra instructions here.
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}
// Don't handle scalable vectors
if (S.getOpcode() == Instruction::ExtractElement &&
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Gather if we hit the RecursionMaxDepth, unless this is a load (or z/sext of
// a load), in which case peek through to include it in the tree, without
// ballooning over-budget.
if (Depth >= Opts.slp_recursion_max_depth &&
(S.isAltShuffle() || VL.size() < 4 ||
!(match(S.getMainOp(), m_Load(m_Value())) ||
all_of(VL, [&S](const Value *I) {
return match(I,
m_OneUse(m_ZExtOrSExt(m_OneUse(m_Load(m_Value()))))) &&
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Check if this is a duplicate of another entry.
LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.getMainOp() << ".\n");
// Cache invariants to avoid recomputing for every V in VL (and every E).
const bool IsPHIWithLoop =
S.getOpcode() == Instruction::PHI &&
LI->getLoopFor(S.getMainOp()->getParent()) != nullptr;
for (TreeEntry *E : getTreeEntries(S.getMainOp())) {
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V) ||
(IsPHIWithLoop && isa<PHINode>(V) && isVectorized(V));
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
bool AreAllSameBlock = !AreScatterAllGEPSameBlock;
bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
if (!AreAllSameInsts || isSplat(VL) ||
(isa<InsertElementInst, ExtractValueInst, ExtractElementInst>(
S.getMainOp()) &&
!all_of(VL, isVectorLikeInstWithConstOps))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O conditions. \n";
dbgs() << "[";
interleaveComma(VL, dbgs(), [&](Value *V) { dbgs() << *V; });
dbgs() << "]\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
// Don't vectorize ephemeral values.
if (!EphValues.empty()) {
for (Value *V : VL) {
if (EphValues.count(V)) {
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
// Do not try to pack to avoid extra instructions here.
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}
}
}
// We now know that this is a vector of instructions of the same type from
// the same block.
// Check that none of the instructions in the bundle are already in the tree
// and the node may be not profitable for the vectorization as the small
// alternate node.
if (S.isAltShuffle()) {
auto GetNumVectorizedExtracted = [&]() {
APInt Extracted = APInt::getZero(VL.size());
APInt Vectorized = APInt::getAllOnes(VL.size());
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I || doesNotNeedToBeScheduled(I) ||
all_of(I->operands(), [&](const Use &U) {
return isa<ExtractElementInst>(U.get());
}))
continue;
if (isVectorized(I))
Vectorized.clearBit(Idx);
else if (!I->hasOneUser() && !areAllUsersVectorized(I, UserIgnoreList))
Extracted.setBit(Idx);
}
return std::make_pair(Vectorized, Extracted);
};
auto [Vectorized, Extracted] = GetNumVectorizedExtracted();
bool PreferScalarize = !Vectorized.isAllOnes() && VL.size() == 2;
if (!Vectorized.isAllOnes() && !PreferScalarize) {
// Rough cost estimation, if the vector code (+ potential extracts) is
// more profitable than the scalar + buildvector.
Type *ScalarTy = VL.front()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
InstructionCost VectorizeCostEstimate =
getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, CostKind) +
getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, VecTy, Extracted,
/*Insert=*/false, /*Extract=*/true, CostKind);
InstructionCost ScalarizeCostEstimate = getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, VecTy, Vectorized,
/*Insert=*/true, /*Extract=*/false, CostKind, /*ForPoisonSrc=*/false);
PreferScalarize = VectorizeCostEstimate > ScalarizeCostEstimate;
}
if (PreferScalarize) {
LLVM_DEBUG(dbgs() << "SLP: The instructions are in tree and alternate "
"node is not profitable.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
// The reduction nodes (stored in UserIgnoreList) also should stay scalar.
if (UserIgnoreList && !UserIgnoreList->empty()) {
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}
/// Peel associative single-use binary chains into operand columns. Peeled
/// instructions go to \p ReassocScalars; sub-operands are inserted after the
/// peeled column so real pairs stay adjacent for flag reuse.
/// Poison lanes are allowed through; a lane that is not itself a matching
/// real instruction can still be peeled as a copyable identity leaf, as
/// long as some other lane anchors the opcode.
/// Add/sub (and fadd/fsub) chains peel together: a peeled subtract keeps
/// the column sign for its first operand and flips it for the second one,
/// recorded per column in \p NegatedColumns, so subtracted leaves stay out
/// of the positive total.
static void scanAssociativeOperands(
const InstructionsState &S, DominatorTree &DT, const DataLayout &DL,
const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
const BoUpSLP &R, SmallVectorImpl<BoUpSLP::ValueList> &Operands,
SmallBitVector &NegatedColumns, SmallVectorImpl<Value *> &ReassocScalars) {
assert(Operands.size() == 2 && NegatedColumns.empty() &&
"Expected the initial 2 operand columns.");
// The subtract family flips the sign of its second operand column.
auto IsSub = [](unsigned Opcode) {
return Opcode == Instruction::Sub || Opcode == Instruction::FSub;
};
// Signs ride with the columns so the parallel lists cannot drift apart.
struct SignedColumn {
BoUpSLP::ValueList Col;
bool Negated;
};
SmallVector<SignedColumn, 4> Columns = {
{std::move(Operands[0]), false},
{std::move(Operands[1]), IsSub(S.getOpcode())}};
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
// The opcode family a column may peel into: subtracts flatten as adds of
// a negated leaf, other opcodes peel only into their own kind.
const unsigned RootFamily = getReassocCombineOpcode(S.getOpcode());
// A lane whose value is itself a single-use chain link; block placement
// does not matter here (buildTreeRec re-checks that later for whatever
// columns end up as real leaves).
auto IsRealLane = [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
return I && I->hasOneUse() &&
getReassocCombineOpcode(I->getOpcode()) == RootFamily &&
isReassocChainLink(I);
};
// Builds on InstructionsCompatibilityAnalysis's own opcode matching
// (rather than the more permissive getSameOpcode()) so a column that
// falls back to a copyable identity leaf only ever anchors on a MainOp
// that InstructionsCompatibilityAnalysis::isSupportedOpcode() accepts.
auto CanPeel = [&](ArrayRef<Value *> Column) {
InstructionsState ColS = Analysis.buildInstructionsState(Column, R);
if (!ColS || getReassocCombineOpcode(ColS.getOpcode()) != RootFamily ||
ColS.isAltShuffle())
return InstructionsState::invalid();
// Every lane must be poison, a genuine matching real lane, or (only for
// a copyable ColS) stand in as a copyable identity leaf
// (Opcode(V, identity) == V).
if (!all_of(Column, [&](Value *V) {
return isa<PoisonValue>(V) || IsRealLane(V) ||
(ColS.areInstructionsWithCopyableElements() &&
ColS.isCopyableElement(V));
}))
return InstructionsState::invalid();
return ColS;
};
for (unsigned Idx = 0; Idx != Columns.size();) {
InstructionsState ColS = CanPeel(Columns[Idx].Col);
if (!ColS) {
++Idx;
continue;
}
BoUpSLP::ValueList Column = std::move(Columns[Idx].Col);
// The absorbing constants of the peeled lanes keep the identity as the
// other operand: the poison operand is not frozen in the flattened node.
SmallVector<BoUpSLP::ValueList> SubOperands = Analysis.buildOperands(
ColS, Column, R, /*AllowPoisonPlaceholders=*/false);
assert(SubOperands.size() == 2 && "Expected 2 operand columns.");
// Poison and copyable lanes have no real instruction left to erase
// later: a copyable V is used as-is, not subsumed by the flattened
// combine.
for (Value *V : Column)
if (!isa<PoisonValue>(V) && !ColS.isCopyableElement(V))
ReassocScalars.push_back(V);
// A peeled subtract negates its second operand column.
const bool Negated = Columns[Idx].Negated;
Columns[Idx].Col = std::move(SubOperands.front());
Columns.insert(
std::next(Columns.begin(), Idx + 1),
{std::move(SubOperands.back()), IsSub(ColS.getOpcode()) != Negated});
// Do not advance Idx: re-examine the column that was just placed here.
}
Operands.clear();
for (auto &[Col, Negated] : Columns) {
Operands.push_back(std::move(Col));
NegatedColumns.push_back(Negated);
}
}
/// Realign flattened operand columns so matching value families share a
/// column across lanes. Lane 0 is canonical; other lanes match by key equality
/// only (never compare hash values). Columns sharing a key are paired by the
/// family of their first operand where available, so e.g. shifts fed by the
/// same load family land in one column instead of pairing by encounter order.
/// Values move between columns only within the same sign: a subtracted leaf
/// never lands in an added column. The sign is queried per lane and column
/// with \p IsNegated: alternate add/sub nodes negate only the non-leading
/// columns of their subtract lanes.
static SmallVector<BoUpSLP::ValueList> alignReassociatedOperandsByKey(
ArrayRef<BoUpSLP::ValueList> Operands,
function_ref<bool(unsigned Lane, unsigned Col)> IsNegated,
const TargetLibraryInfo &TLI) {
const unsigned NumCols = Operands.size();
const unsigned NumLanes = Operands.front().size();
auto LoadsSubkey = [](size_t /*Key*/, LoadInst *LI) {
return hash_value(getUnderlyingObject(LI->getPointerOperand()));
};
auto GetKey = [&](Value *V) {
return generateKeySubkey(V, &TLI, LoadsSubkey, /*AllowAlternate=*/false);
};
const std::pair<size_t, size_t> NoSecondaryKey{0, 0};
// Family of the value's first operand; loads already carry their family in
// the primary key.
auto GetSecondaryKey = [&](Value *V) -> std::pair<size_t, size_t> {
auto *I = dyn_cast<Instruction>(V);
if (!I || isa<LoadInst>(I) || I->getNumOperands() == 0)
return NoSecondaryKey;
return GetKey(I->getOperand(0));
};
SmallVector<std::pair<size_t, size_t>> Lane0Keys = map_to_vector(
Operands, [&](const BoUpSLP::ValueList &Col) { return GetKey(Col[0]); });
SmallVector<std::pair<size_t, size_t>> Lane0SecondaryKeys =
map_to_vector(Operands, [&](const BoUpSLP::ValueList &Col) {
return GetSecondaryKey(Col[0]);
});
SmallVector<BoUpSLP::ValueList> Aligned(NumCols,
BoUpSLP::ValueList(NumLanes));
for (unsigned Col : seq<unsigned>(NumCols))
Aligned[Col][0] = Operands[Col][0];
for (unsigned Lane : seq<unsigned>(1, NumLanes)) {
// Buckets are keyed by the value key and the column sign.
using Key = std::pair<std::pair<size_t, size_t>, unsigned>;
auto Sign = [&](unsigned Col) { return IsNegated(Lane, Col) ? 1U : 0U; };
SmallDenseMap<Key, SmallVector<unsigned, 2>, 8> Buckets;
for (unsigned Col : seq<unsigned>(NumCols))
Buckets[{GetKey(Operands[Col][Lane]), Sign(Col)}].push_back(Col);
SmallDenseMap<Key, unsigned, 8> NextInBucket;
SmallVector<unsigned> SlotSrcCol(NumCols, NumCols);
SmallBitVector ColClaimed(NumCols, false);
for (unsigned Slot : seq<unsigned>(NumCols)) {
const auto SlotKey = std::make_pair(Lane0Keys[Slot], Sign(Slot));
auto BucketIt = Buckets.find(SlotKey);
if (BucketIt == Buckets.end())
continue;
unsigned &Cursor = NextInBucket[SlotKey];
if (Cursor >= BucketIt->second.size())
continue;
// Among the remaining same-key columns prefer the one whose operand
// family matches lane 0's; no match keeps the bucket order.
unsigned PickPos = Cursor;
if (Lane0SecondaryKeys[Slot] != NoSecondaryKey)
for (unsigned P = Cursor, E = BucketIt->second.size(); P != E; ++P)
if (GetSecondaryKey(Operands[BucketIt->second[P]][Lane]) ==
Lane0SecondaryKeys[Slot]) {
PickPos = P;
break;
}
std::swap(BucketIt->second[Cursor], BucketIt->second[PickPos]);
unsigned SrcCol = BucketIt->second[Cursor++];
SlotSrcCol[Slot] = SrcCol;
ColClaimed[SrcCol] = true;
}
// Leftover slots take leftover columns of the same sign; per-lane sign
// counts match, so every slot finds one.
for (unsigned Slot : seq<unsigned>(NumCols)) {
if (SlotSrcCol[Slot] != NumCols)
continue;
for (unsigned Col : seq<unsigned>(NumCols)) {
if (!ColClaimed[Col] && IsNegated(Lane, Col) == IsNegated(Lane, Slot)) {
SlotSrcCol[Slot] = Col;
ColClaimed[Col] = true;
break;
}
}
}
for (unsigned Slot : seq<unsigned>(NumCols))
Aligned[Slot][Lane] = Operands[SlotSrcCol[Slot]][Lane];
}
return Aligned;
}
/// Rank reassociated operand layouts by vectorizable load columns, other load
/// patterns, broadcast/constant columns, then fewer unique values per column.
/// Identity columns are scored by the opcode of the emitted combines.
static std::tuple<unsigned, unsigned, unsigned, int>
getReassocColumnsQuality(ArrayRef<BoUpSLP::ValueList> Columns, const BoUpSLP &R,
unsigned Opcode) {
unsigned NumConsecutiveLoadCols = 0;
unsigned NumOtherVecLoadCols = 0;
unsigned NumBroadcastOrConstCols = 0;
int NumUniqueValues = 0;
const unsigned CombineOpcode = getReassocCombineOpcode(Opcode);
for (ArrayRef<Value *> Col : Columns) {
if (all_of(Col, IsaPred<Constant>)) {
++NumBroadcastOrConstCols;
continue;
}
// Identity constants are free beyond the base they are inserted into.
SmallPtrSet<Value *, 8> UniqueValues;
for (Value *V : Col)
if (!isBinOpIdentityConstant(V, CombineOpcode))
UniqueValues.insert(V);
NumUniqueValues += UniqueValues.size();
if (UniqueValues.size() <= 1) {
++NumBroadcastOrConstCols;
continue;
}
if (all_of(Col, IsaPred<LoadInst>)) {
BoUpSLP::OrdersType Order;
SmallVector<Value *> PointerOps;
BoUpSLP::StridedPtrInfo SPtrInfo;
BoUpSLP::LoadsState LS =
R.canVectorizeLoads(Col, Col.front(), Order, PointerOps, SPtrInfo);
if (LS == BoUpSLP::LoadsState::Vectorize)
++NumConsecutiveLoadCols;
else if (LS != BoUpSLP::LoadsState::Gather)
++NumOtherVecLoadCols;
}
}
// Greater is better; negate unique count as a tie-breaker.
return std::make_tuple(NumConsecutiveLoadCols, NumOtherVecLoadCols,
NumBroadcastOrConstCols, -NumUniqueValues);
}
void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
const EdgeInfo &UserTreeIdx,
unsigned InterleaveFactor) {
assert((allConstant(VLRef) || allSameType(VLRef)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef);
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState,
bool SplitInHalves = false) {
SmallVector<Value *> Op1, Op2;
OrdersType ReorderIndices;
if (SplitInHalves) {
auto *Mid = std::next(VL.begin(), VL.size() / 2);
Op1.assign(VL.begin(), Mid);
Op2.assign(Mid, VL.end());
} else if (!canBuildSplitNode(VL, LocalState, Op1, Op2, ReorderIndices)) {
return false;
}
auto Invalid = ScheduleBundle::invalid();
auto *TE = newTreeEntry(VL, TreeEntry::SplitVectorize, Invalid, LocalState,
UserTreeIdx, {}, ReorderIndices);
LLVM_DEBUG(dbgs() << "SLP: split alternate node.\n"; TE->dump());
auto AddNode = [&](ArrayRef<Value *> Op, unsigned Idx) {
InstructionsState S = getSameOpcode(Op, *TLI);
if (S && (isa<LoadInst>(S.getMainOp()) ||
getSameValuesTreeEntry(S.getMainOp(), Op, /*SameVF=*/true))) {
// Build gather node for loads, they will be gathered later.
TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
Idx == 0 ? 0 : Op1.size());
(void)newTreeEntry(Op, TreeEntry::NeedToGather, Invalid, S, {TE, Idx});
} else {
TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
Idx == 0 ? 0 : Op1.size());
buildTreeRec(Op, Depth, {TE, Idx});
}
};
AddNode(Op1, 0);
AddNode(Op2, 1);
return true;
};
auto AreOnlyConstsWithPHIs = [](ArrayRef<Value *> VL) {
bool AreConsts = false;
auto IsNotPoison = [](Value *V) { return !isa<PoisonValue>(V); };
for (Value *V : make_filter_range(VL, IsNotPoison)) {
if (isa<Constant>(V)) {
AreConsts = true;
continue;
}
if (!isa<PHINode>(V))
return false;
}
return AreConsts;
};
if (AreOnlyConstsWithPHIs(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to all constants and PHIs.\n");
newGatherTreeEntry(VL, InstructionsState::invalid(), UserTreeIdx);
return;
}
ScalarsVectorizationLegality Legality =
getScalarsVectorizationLegality(VL, Depth, UserTreeIdx);
InstructionsState S = Legality.getInstructionsState();
if (!Legality.isLegal()) {
if (Legality.trySplitVectorize()) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(VL);
// Last chance to try to vectorize alternate node.
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
if (Legality.tryToFindDuplicates())
(void)tryToFindDuplicates(Opts, VL, ReuseShuffleIndices, *TTI, *TLI, S,
UserTreeIdx, *this);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
if (SplitStoredValues && S && UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->Idx == 0 && TrySplitNode(S, /*SplitInHalves=*/true))
return;
// FIXME: investigate if there are profitable cases for VL.size() <= 4.
if (S.isAltShuffle() && TrySplitNode(S))
return;
// Cmp nodes with interchangeable lanes (e.g. x == 0 mixed with x <u C)
// try the split by the original predicates, like the alternate nodes. The
// trunc X to i1 lanes have no original predicate.
if (S.getOpcode() == Instruction::ICmp && !S.isAltShuffle() &&
none_of(VL, IsaPred<TruncInst>)) {
auto SameOrSwapped = [](const ICmpInst *CI, CmpInst::Predicate P) {
return CI->getPredicate() == P ||
CI->getPredicate() == CmpInst::getSwappedPredicate(P);
};
ICmpInst *MainI = cast<ICmpInst>(*find_if(VL, IsaPred<ICmpInst>));
auto *AltIt = find_if(VL, [&](Value *V) {
auto *CI = dyn_cast<ICmpInst>(V);
return CI && !SameOrSwapped(CI, MainI->getPredicate());
});
ICmpInst *AltI = AltIt == VL.end() ? nullptr : cast<ICmpInst>(*AltIt);
if (AltI &&
all_of(make_isa_range<ICmpInst>(VL),
[&](ICmpInst *CI) {
return SameOrSwapped(CI, MainI->getPredicate()) ||
SameOrSwapped(CI, AltI->getPredicate());
}) &&
TrySplitNode(InstructionsState(MainI, AltI)))
return;
}
// Check that every instruction appears once in this bundle.
if (!tryToFindDuplicates(Opts, VL, ReuseShuffleIndices, *TTI, *TLI, S,
UserTreeIdx, *this, /*BuildGatherOnly=*/false)) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
// Perform specific checks for each particular instruction kind.
bool IsScatterVectorizeUserTE =
UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
OrdersType CurrentOrder;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
SmallVector<int> ExpandShuffleMask;
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps, SPtrInfo,
ExpandShuffleMask);
// The zero-extended sub-fields of the same wider integer scalar are
// gathered and emitted as a bitcast to the field vector plus a permutation.
// For 2 lanes the emission is not cheaper than the insertelement chain.
if (State == TreeEntry::NeedToGather ||
(VL.size() > 2 && ReuseShuffleIndices.empty() &&
matchGatheredExtractedFields(VL, *DL))) {
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
// Check the loop nest. We need to be sure we handle a single loop nest at a
// time to avoid incorrect cost estimation because of the loop aware cost
// model.
if (VectorizableTree.empty()) {
assert(CurrentLoopNest.empty() && "Expected empty loop nest");
// Process the first node? Initial fill of the loop nest.
BasicBlock *Parent = S.getMainOp()->getParent();
if (const Loop *L = LI->getLoopFor(Parent)) {
L = findInnermostNonInvariantLoop(L, VL);
if (L)
CurrentLoopNest.assign(getLoopNest(L));
}
} else if (!UserTreeIdx ||
UserTreeIdx.UserTE->State == TreeEntry::SplitVectorize ||
UserTreeIdx.UserTE->isGather() ||
UserTreeIdx.UserTE->getMainOp()->getParent() !=
S.getMainOp()->getParent()) {
BasicBlock *Parent = S.getMainOp()->getParent();
if (const Loop *L = LI->getLoopFor(Parent)) {
// Check that the new loop nest shares the same outer structure as the
// tree's current loop nest. Completely disjoint nests (different
// outermost loops) are forced to gather because their scales cannot be
// meaningfully combined. Sibling inner loops (inside a common outer
// loop or outside any loops at all) are allowed: the cost model scales
// each entry by its own loop via getScaleToLoopIterations(), so a tree
// that spans sibling inner loops (e.g. a PHI at their merge block) can
// still be costed correctly. Contract CurrentLoopNest to the longest
// common prefix with the new entry's nest so subsequent entries in yet
// another sibling can also be admitted.
L = findInnermostNonInvariantLoop(L, VL);
if (L) {
SmallVector<const Loop *> NewLoopNest(getLoopNest(L));
unsigned CommonLen = 0;
for (const auto [L1, L2] : zip(CurrentLoopNest, NewLoopNest)) {
if (L1 != L2)
break;
++CommonLen;
}
auto ValidateMergedBTCs = [&](unsigned StartDepth) -> bool {
unsigned EndDepth =
std::min<unsigned>(NewLoopNest.size(), MergedLoopBTCs.size());
for (unsigned D = StartDepth; D < EndDepth; ++D) {
const SCEV *Constraint = MergedLoopBTCs[D];
if (!Constraint)
continue;
const SCEV *NewBTC = SE->getBackedgeTakenCount(NewLoopNest[D]);
if (isa<SCEVCouldNotCompute>(NewBTC) || NewBTC != Constraint)
return false;
}
return true;
};
auto BailOutToGather = [&]() {
LLVM_DEBUG(dbgs()
<< "SLP: Sibling loops have different trip counts.\n");
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
};
if (CurrentLoopNest.empty()) {
if (!ValidateMergedBTCs(0)) {
BailOutToGather();
return;
}
CurrentLoopNest.assign(NewLoopNest);
} else if (CommonLen < CurrentLoopNest.size() &&
CommonLen < NewLoopNest.size()) {
// Divergence below the common prefix: the tree now spans sibling
// loops at depth CommonLen. Admitting them into one tree makes
// the profitability decision JOINT across both siblings, so a
// very hot sibling could otherwise let an unprofitable cold
// sibling ride along "for free" (per-entry scaling of the cold
// sibling's entries would be dwarfed by the hot one). Require
// SCEV-proven equal backedge-taken counts for the diverging
// siblings before joining; otherwise force gather.
const Loop *SibA = CurrentLoopNest[CommonLen];
const Loop *SibB = NewLoopNest[CommonLen];
const SCEV *BecA = SE->getBackedgeTakenCount(SibA);
const SCEV *BecB = SE->getBackedgeTakenCount(SibB);
if (isa<SCEVCouldNotCompute>(BecA) || BecA != BecB) {
BailOutToGather();
return;
}
if (!ValidateMergedBTCs(CommonLen + 1)) {
BailOutToGather();
return;
}
if (MergedLoopBTCs.size() <= CommonLen)
MergedLoopBTCs.resize(CommonLen + 1, nullptr);
MergedLoopBTCs[CommonLen] = BecA;
CurrentLoopNest.truncate(CommonLen);
} else if (NewLoopNest.size() > CurrentLoopNest.size()) {
if (!ValidateMergedBTCs(CurrentLoopNest.size())) {
BailOutToGather();
return;
}
CurrentLoopNest.append(
std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
NewLoopNest.end());
}
// Otherwise NewLoopNest is a prefix of CurrentLoopNest: keep as-is.
}
}
}
Instruction *VL0 = S.getMainOp();
BasicBlock *BB = VL0->getParent();
auto &BSRef = BlocksSchedules[BB];
if (!BSRef)
BSRef = std::make_unique<BlockScheduling>(BB, Opts.slp_schedule_budget);
BlockScheduling &BS = *BSRef;
SetVector<Value *> UniqueValues(llvm::from_range, VL);
std::optional<ScheduleBundle *> BundlePtr =
BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S, UserTreeIdx);
#ifdef EXPENSIVE_CHECKS
// Make sure we didn't break any internal invariants
BS.verify();
#endif
if (!BundlePtr || (*BundlePtr && !*BundlePtr.value())) {
LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
// Last chance to try to vectorize copyable node.
if (S.areInstructionsWithCopyableElements() &&
ReuseShuffleIndices.empty()) {
InstructionsState AltS = getSameOpcode(VL, *TLI);
if (AltS && AltS.isAltShuffle() && TrySplitNode(AltS))
return;
}
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
registerNonVectorizableLoads(ArrayRef(VL));
return;
}
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
SmallVector<ValueList> Operands = Analysis.buildOperands(S, VL, *this);
// Flatten associative binary chains into operand columns. Only the peeled
// chain links are required to be single-use (they are erased); the root
// being flattened may have other uses. Skip lanes that are neither chain
// links nor copyable identity leaves. Alternate nodes flatten too, but
// each lane keeps its own opcode on every combine level. Restricted to
// BinaryOperator: isAssociative() is also true for associative intrinsics
// (e.g. smax/smin/umax/umin), which are CallInst, not BinaryOperator, and
// are not supported by the copyable-identity machinery used below
// (ConstantExpr::getBinOpIdentity, isSupportedOpcode,
// isBinOpIdentityConstant).
SmallVector<Value *> ReassocScalars;
// Sign of each flattened operand column (a subtracted leaf is negated).
SmallBitVector NegatedColumns;
// Per-lane subtract markers for flattened alternate nodes.
SmallBitVector SubLanes;
// Cached below (when the peel is kept) so the reorder step further down
// does not need to redo the aligning/scoring work.
SmallVector<ValueList> ReassocAlignedOperands;
// Snapshot of the pre-flatten operand columns, used by both revert points.
SmallVector<ValueList> NaturalTwoColumns;
std::tuple<unsigned, unsigned, unsigned, int> ReassocPeeledQuality;
if (Opts.slp_reassociate_ops && Operands.size() == 2 &&
all_of(VL, [&](Value *V) {
if (!S.isAltShuffle() && S.isCopyableElement(V))
return true;
auto *I = dyn_cast<BinaryOperator>(V);
return I && isReassocChainLink(I);
})) {
NaturalTwoColumns = Operands;
if (S.isAltShuffle()) {
if (SmallVector<SmallVector<Value *>> Flattened =
scanAltAssociativeOperands(S, *TLI, VL, Operands[0], Operands[1],
ReassocScalars, SubLanes);
!Flattened.empty()) {
Operands.clear();
for (auto &Col : Flattened)
Operands.emplace_back(std::move(Col));
}
} else {
scanAssociativeOperands(S, *DT, *DL, *TTI, *TLI, *this, Operands,
NegatedColumns, ReassocScalars);
}
// Drop flattening unless realigning improves load or broadcast column
// structure; an unimproved peel ties and reverts to natural columns.
if (!ReassocScalars.empty()) {
auto IsNegated = [&](unsigned Lane, unsigned Col) {
return S.isAltShuffle() ? Col > 0 && SubLanes.test(Lane)
: NegatedColumns.test(Col);
};
ReassocAlignedOperands =
alignReassociatedOperandsByKey(Operands, IsNegated, *TLI);
// Alternate nodes compare against the natural two-column form rather
// than the raw peel: the natural form pays one lane-select shuffle per
// level, so the flatten is worth keeping whenever the realigned
// columns improve on the natural ones.
ReassocPeeledQuality = getReassocColumnsQuality(
S.isAltShuffle() ? NaturalTwoColumns : Operands, *this,
S.getOpcode());
// The unique-value count (4th field) is only a tie-break for the
// later reorder-or-not decision, not for this one.
auto DropUniqueCount = [](const auto &Quality) {
return std::make_tuple(std::get<0>(Quality), std::get<1>(Quality),
std::get<2>(Quality));
};
if (DropUniqueCount(getReassocColumnsQuality(ReassocAlignedOperands,
*this, S.getOpcode())) <=
DropUniqueCount(ReassocPeeledQuality)) {
Operands = std::move(NaturalTwoColumns);
ReassocScalars.clear();
}
}
}
// Registers peeled chain links on the node so they are erased with it and
// their scheduling deps are released as reassociated operands.
auto RegisterReassocScalars = [&](TreeEntry *TE) {
for (Value *V : ReassocScalars) {
TE->addReassocScalar(V);
SmallVectorImpl<const TreeEntry *> &Owners =
ReassocScalarToTreeEntries.try_emplace(V).first->second;
if (!is_contained(Owners, TE))
Owners.push_back(TE);
}
};
// A value routed into several columns is consumed by several combines;
// that distorts both the cost and the vector structure (duplicated loads,
// partial masked columns), so prefer the natural two-column shape.
auto HasDupColumnValues = [&]() {
SmallPtrSet<const Value *, 16> ColumnValues;
return any_of(Operands, [&](const ValueList &Col) {
return any_of(Col, [&](const Value *V) {
return !isa<Constant>(V) && !ColumnValues.insert(V).second;
});
});
};
ScheduleBundle Empty;
ScheduleBundle &Bundle = BundlePtr.value() ? *BundlePtr.value() : Empty;
LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
unsigned ShuffleOrOp =
S.isAltShuffle() ? (unsigned)Instruction::ShuffleVector : S.getOpcode();
auto CreateOperandNodes = [&](TreeEntry *TE, const auto &Operands) {
// Postpone PHI nodes creation
SmallVector<unsigned> PHIOps;
for (unsigned I : seq<unsigned>(Operands.size())) {
ArrayRef<Value *> Op = Operands[I];
if (Op.empty())
continue;
InstructionsState S = getSameOpcode(Op, *TLI);
if ((!S || S.getOpcode() != Instruction::PHI) || S.isAltShuffle())
buildTreeRec(Op, Depth + 1, {TE, I});
else
PHIOps.push_back(I);
}
for (unsigned I : PHIOps)
buildTreeRec(Operands[I], Depth + 1, {TE, I});
};
switch (ShuffleOrOp) {
case Instruction::PHI: {
TreeEntry *TE =
newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (PHINode).\n";
TE->dump());
TE->setOperands(Operands);
CreateOperandNodes(TE, Operands);
return;
}
case Instruction::ExtractValue:
case Instruction::ExtractElement: {
if (CurrentOrder.empty()) {
LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
} else {
LLVM_DEBUG({
dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order";
for (unsigned Idx : CurrentOrder)
dbgs() << " " << Idx;
dbgs() << "\n";
});
fixupOrderingIndices(CurrentOrder);
}
// Insert new order with initial value 0, if it does not exist,
// otherwise return the iterator to the existing one.
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices, CurrentOrder);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry "
"(ExtractValueInst/ExtractElementInst).\n";
TE->dump());
// This is a special case, as it does not gather, but at the same time
// we are not extending buildTreeRec() towards the operands.
TE->setOperands(Operands);
if (ShuffleOrOp == Instruction::ExtractValue) {
SmallVector<unsigned> Indices;
SmallVector<Value *> Calls;
if (checkEVsForVecCalls(VL, S, *TLI, Indices, Calls)) {
TE->StructEVIndices = std::move(Indices);
buildTreeRec(Operands.front(), Depth + 1, {TE, 0});
}
}
return;
}
case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(ReuseShuffleIndices.empty() && "All inserts should be unique");
auto OrdCompare = [](const std::pair<int, int> &P1,
const std::pair<int, int> &P2) {
return P1.first > P2.first;
};
PriorityQueue<std::pair<int, int>, SmallVector<std::pair<int, int>>,
decltype(OrdCompare)>
Indices(OrdCompare);
for (int I = 0, E = VL.size(); I < E; ++I) {
unsigned Idx = *getElementIndex(VL[I]);
Indices.emplace(Idx, I);
}
OrdersType CurrentOrder(VL.size(), VL.size());
bool IsIdentity = true;
for (int I = 0, E = VL.size(); I < E; ++I) {
CurrentOrder[Indices.top().second] = I;
IsIdentity &= Indices.top().second == I;
Indices.pop();
}
if (IsIdentity)
CurrentOrder.clear();
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
{}, CurrentOrder);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (InsertElementInst).\n";
TE->dump());
TE->setOperands(Operands);
buildTreeRec(TE->getOperand(1), Depth + 1, {TE, 1});
return;
}
case Instruction::Load: {
// Check that a vectorized load would load the same memory as a scalar
// load. For example, we don't want to vectorize loads that are smaller
// than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
// treats loading/storing it as an i8 struct. If we vectorize loads/stores
// from such a struct, we read/write packed bits disagreeing with the
// unvectorized version.
TreeEntry *TE = nullptr;
fixupOrderingIndices(CurrentOrder);
switch (State) {
case TreeEntry::Vectorize:
TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices, CurrentOrder, InterleaveFactor);
if (CurrentOrder.empty())
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (LoadInst).\n";
TE->dump());
else
LLVM_DEBUG(dbgs()
<< "SLP: added a new TreeEntry (jumbled LoadInst).\n";
TE->dump());
break;
case TreeEntry::CompressVectorize:
// Vectorizing non-consecutive loads with (masked)load + compress.
TE = newTreeEntry(VL, TreeEntry::CompressVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
LLVM_DEBUG(
dbgs()
<< "SLP: added a new TreeEntry (masked LoadInst + compress).\n";
TE->dump());
break;
case TreeEntry::StridedVectorize:
// Vectorizing non-consecutive loads with `llvm.masked.gather`.
TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
TreeEntryToStridedPtrInfoMap[TE] = SPtrInfo;
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (strided LoadInst).\n";
TE->dump());
break;
case TreeEntry::ScatterVectorize:
// Vectorizing non-consecutive loads with `llvm.masked.gather`.
TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices);
LLVM_DEBUG(
dbgs()
<< "SLP: added a new TreeEntry (non-consecutive LoadInst).\n";
TE->dump());
break;
case TreeEntry::BlendedLoadVectorize: {
// Two masked loads from the candidate bases, blended by the per-lane
// select conditions. The conditions are added as the operand node, so
// the blend mask is vectorized through the tree.
TE = newTreeEntry(VL, TreeEntry::BlendedLoadVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (blended LoadInst).\n";
TE->dump());
Value *TrueBase = nullptr;
Value *FalseBase = nullptr;
SmallVector<Value *> Conditions;
[[maybe_unused]] bool Found = isSelectedBaseLoad(
VL0->getType(), PointerOps, *DL, TrueBase, FalseBase, Conditions);
assert(Found && "Expected a valid blended-load pattern.");
Operands.assign(1, ValueList(Conditions.begin(), Conditions.end()));
TE->setOperands(Operands);
buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
return;
}
case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
case TreeEntry::NeedToGather:
llvm_unreachable("Unexpected loads state.");
}
if (!CurrentOrder.empty() && State != TreeEntry::ScatterVectorize) {
assert(Operands.size() == 1 && "Expected a single operand only");
SmallVector<int> Mask;
inversePermutation(CurrentOrder, Mask);
reorderScalars(Operands.front(), Mask);
}
TE->setOperands(Operands);
if (State == TreeEntry::ScatterVectorize)
buildTreeRec(PointerOps, Depth + 1, {TE, 0});
return;
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::PtrToAddr:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or(
std::make_pair(std::numeric_limits<unsigned>::min(),
std::numeric_limits<unsigned>::max()));
if (ShuffleOrOp == Instruction::ZExt ||
ShuffleOrOp == Instruction::SExt) {
CastMaxMinBWSizes = std::make_pair(
std::max<unsigned>(DL->getTypeSizeInBits(VL0->getType()),
PrevMaxBW),
std::min<unsigned>(
DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
PrevMinBW));
} else if (ShuffleOrOp == Instruction::Trunc) {
CastMaxMinBWSizes = std::make_pair(
std::max<unsigned>(
DL->getTypeSizeInBits(VL0->getOperand(0)->getType()),
PrevMaxBW),
std::min<unsigned>(DL->getTypeSizeInBits(VL0->getType()),
PrevMinBW));
}
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CastInst).\n";
TE->dump());
TE->setOperands(Operands);
for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth, {TE, I});
if (ShuffleOrOp == Instruction::Trunc) {
ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
} else if (ShuffleOrOp == Instruction::SIToFP ||
ShuffleOrOp == Instruction::UIToFP) {
unsigned NumSignBits =
ComputeNumSignBits(VL0->getOperand(0), *DL, AC, nullptr, DT);
if (auto *OpI = dyn_cast<Instruction>(VL0->getOperand(0))) {
APInt Mask = DB->getDemandedBits(OpI);
NumSignBits = std::max(NumSignBits, Mask.countl_zero());
}
if (NumSignBits * 2 >=
DL->getTypeSizeInBits(VL0->getOperand(0)->getType()))
ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
}
return;
}
case Instruction::ICmp:
case Instruction::FCmp: {
// Check that all of the compares have the same predicate.
CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CmpInst).\n";
TE->dump());
// The ands of the bit tests are peeled into the operands of the node.
if (any_of(VL, IsaPred<TruncInst>)) {
for (ICmpInst *CI : make_isa_range<ICmpInst>(VL))
ReassocScalars.push_back(CI->getOperand(0));
RegisterReassocScalars(TE);
}
VLOperands Ops(VL, Operands, S, *this);
if (cast<CmpInst>(VL0)->isCommutative()) {
// Commutative predicate - collect + sort operands of the instructions
// so that each side is more likely to have the same opcode.
assert(P0 == CmpInst::getSwappedPredicate(P0) &&
"Commutative Predicate mismatch");
Ops.reorder();
Operands.front() = Ops.getVL(0);
Operands.back() = Ops.getVL(1);
} else {
// Collect operands - commute if it uses the swapped predicate.
// Lanes interchangeable with P0 (e.g. x == 0 in an x <u C bundle)
// already have their operands adjusted, no need to commute them.
for (auto [Idx, V] :
make_filter_range(enumerate(VL), [](const auto &P) {
return isa<CmpInst>(P.value());
})) {
auto *Cmp = cast<CmpInst>(V);
if (Cmp->getPredicate() != P0 &&
!CmpSamePredicateHelper::canConvertTo(Cmp, P0))
std::swap(Operands.front()[Idx], Operands.back()[Idx]);
}
}
TE->setOperands(Operands);
buildTreeRec(Operands.front(), Depth, {TE, 0});
buildTreeRec(Operands.back(), Depth, {TE, 1});
if (ShuffleOrOp == Instruction::ICmp) {
unsigned NumSignBits0 =
ComputeNumSignBits(VL0->getOperand(0), *DL, AC, nullptr, DT);
if (NumSignBits0 * 2 >=
DL->getTypeSizeInBits(VL0->getOperand(0)->getType()))
ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
unsigned NumSignBits1 =
ComputeNumSignBits(VL0->getOperand(1), *DL, AC, nullptr, DT);
if (NumSignBits1 * 2 >=
DL->getTypeSizeInBits(VL0->getOperand(1)->getType()))
ExtraBitWidthNodes.insert(getOperandEntry(TE, 1)->Idx);
}
return;
}
case Instruction::Select:
case Instruction::FNeg:
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
case Instruction::Freeze: {
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
LLVM_DEBUG(
dbgs() << "SLP: added a new TreeEntry "
"(SelectInst/UnaryOperator/BinaryOperator/FreezeInst).\n";
TE->dump());
if (!ReassocScalars.empty()) {
// Realign columns by value family (cached above as
// ReassocAlignedOperands), then let VLOperands polish the order,
// keeping the polish only if the column-quality score does not regress
// vs the peeled order. That polish is quadratic in the column count, so
// past ReassocReorderColumnLimit keep the realigned seed, which already
// groups the vectorizable columns. The polish swaps values between
// columns per lane, which must not mix added and subtracted leaves in
// one column, so signed nodes polish each sign group separately.
if (Operands.size() <= Opts.slp_reassociate_reorder_limit) {
SmallVector<ValueList> Reordered = ReassocAlignedOperands;
SmallVector<unsigned> GroupIds[2];
for (unsigned I : seq<unsigned>(NegatedColumns.size()))
GroupIds[NegatedColumns[I]].push_back(I);
for (ArrayRef<unsigned> Group :
make_filter_range(GroupIds, [](ArrayRef<unsigned> Group) {
return Group.size() > 1;
})) {
SmallVector<ValueList> GroupCols = map_to_vector(
Group, [&](unsigned I) { return ReassocAlignedOperands[I]; });
VLOperands Ops(GroupCols, VL0->getParent(), *this);
Ops.reorder();
for (unsigned Pos : seq<unsigned>(Group.size()))
Reordered[Group[Pos]] = Ops.getVL(Pos);
}
if (getReassocColumnsQuality(Reordered, *this, S.getOpcode()) >=
ReassocPeeledQuality)
Operands = std::move(Reordered);
else
LLVM_DEBUG(dbgs() << "SLP: reassociation reorder loses column "
"structure, keeping the natural operand "
"order.\n");
} else {
Operands = std::move(ReassocAlignedOperands);
}
if (HasDupColumnValues()) {
Operands = std::move(NaturalTwoColumns);
ReassocScalars.clear();
}
}
if (!ReassocScalars.empty())
RegisterReassocScalars(TE);
else if (isa<BinaryOperator>(VL0) && isCommutative(VL0)) {
VLOperands Ops(VL, Operands, S, *this);
Ops.reorder();
Operands[0] = Ops.getVL(0);
Operands[1] = Ops.getVL(1);
}
TE->setOperands(Operands);
if (!ReassocScalars.empty() && NegatedColumns.any())
TE->setReassocNegatedOps(NegatedColumns);
for (unsigned I : seq<unsigned>(TE->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
return;
}
case Instruction::GetElementPtr: {
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (GetElementPtrInst).\n";
TE->dump());
TE->setOperands(Operands);
for (unsigned I = 0, Ops = Operands.size(); I < Ops; ++I)
buildTreeRec(Operands[I], Depth + 1, {TE, I});
return;
}
case Instruction::Store: {
assert(CurrentOrder.empty() &&
"Expected ordered store during tree building");
if (State == TreeEntry::StridedVectorize) {
TreeEntry *TE =
newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
TreeEntryToStridedPtrInfoMap[TE] = SPtrInfo;
LLVM_DEBUG(
dbgs() << "SLP: added a new TreeEntry (strided StoreInst).\n";
TE->dump());
TE->setOperands(Operands);
buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
return;
}
if (State == TreeEntry::ExpandVectorize) {
assert(ReuseShuffleIndices.empty() &&
"Expected no reuse shuffle for an expanded masked store.");
TreeEntry *TE =
newTreeEntry(VL, TreeEntry::ExpandVectorize, Bundle, S, UserTreeIdx,
ExpandShuffleMask, CurrentOrder);
TreeEntryToStridedPtrInfoMap[TE] = SPtrInfo;
LLVM_DEBUG(
dbgs()
<< "SLP: added a new TreeEntry (expanded masked StoreInst).\n";
TE->dump());
TE->setOperands(Operands);
buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
return;
}
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices, CurrentOrder);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (StoreInst).\n";
TE->dump());
TE->setOperands(Operands);
buildTreeRec(TE->getOperand(0), Depth + 1, {TE, 0});
return;
}
case Instruction::Call: {
// Check if the calls are all to the same vectorizable intrinsic or
// library function.
CallInst *CI = cast<CallInst>(VL0);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CallInst).\n";
TE->dump());
if (isCommutative(VL0)) {
VLOperands Ops(VL, Operands, S, *this);
Ops.reorder();
Operands[0] = Ops.getVL(0);
Operands[1] = Ops.getVL(1);
}
TE->setOperands(Operands);
for (unsigned I : seq<unsigned>(CI->arg_size())) {
// For scalar operands no need to create an entry since no need to
// vectorize it.
if (isVectorIntrinsicWithScalarOpAtArg(ID, I, TTI))
continue;
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
}
return;
}
case Instruction::ShuffleVector: {
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
if (S.isAltShuffle()) {
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (isAltShuffle).\n";
TE->dump());
} else {
assert(Opts.slp_revec && "Only supported by REVEC.");
LLVM_DEBUG(
dbgs() << "SLP: added a new TreeEntry (ShuffleVectorInst).\n";
TE->dump());
}
// Reorder operands if reordering would enable vectorization.
auto *CI = dyn_cast<CmpInst>(VL0);
if (CI && any_of(make_isa_range<CmpInst>(VL),
[](CmpInst *Cmp) { return !Cmp->isCommutative(); })) {
auto *MainCI = cast<CmpInst>(S.getMainOp());
auto *AltCI = cast<CmpInst>(S.getAltOp());
CmpInst::Predicate MainP = MainCI->getPredicate();
CmpInst::Predicate AltP = AltCI->getPredicate();
assert(MainP != AltP &&
"Expected different main/alternate predicates.");
// Collect operands - commute if it uses the swapped predicate or
// alternate operation.
for (auto [Idx, V] :
make_filter_range(enumerate(VL), [](const auto &P) {
return !isa<PoisonValue>(P.value());
})) {
auto *Cmp = cast<CmpInst>(V);
if (isAlternateInstruction(Cmp, MainCI, AltCI, *TLI)) {
if (AltP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
std::swap(Operands.front()[Idx], Operands.back()[Idx]);
} else {
if (MainP == CmpInst::getSwappedPredicate(Cmp->getPredicate()))
std::swap(Operands.front()[Idx], Operands.back()[Idx]);
}
}
TE->setOperands(Operands);
buildTreeRec(Operands.front(), Depth + 1, {TE, 0});
buildTreeRec(Operands.back(), Depth + 1, {TE, 1});
return;
}
if (!ReassocScalars.empty()) {
// Peeled alternate chains take the realigned columns; the operand
// reorder below does not preserve the per-lane sign constraints.
Operands = std::move(ReassocAlignedOperands);
if (HasDupColumnValues()) {
Operands = std::move(NaturalTwoColumns);
ReassocScalars.clear();
}
}
if (ReassocScalars.empty() && (isa<BinaryOperator>(VL0) || CI)) {
VLOperands Ops(VL, Operands, S, *this);
Ops.reorder();
Operands[0] = Ops.getVL(0);
Operands[1] = Ops.getVL(1);
}
if (!ReassocScalars.empty())
RegisterReassocScalars(TE);
TE->setOperands(Operands);
for (unsigned I : seq<unsigned>(TE->getNumOperands()))
buildTreeRec(TE->getOperand(I), Depth + 1, {TE, I});
return;
}
default:
break;
}
llvm_unreachable("Unexpected vectorization of the instructions.");
}
unsigned BoUpSLP::canMapToVector(Type *T) const {
unsigned N = 1;
Type *EltTy = T;
while (isa<StructType, ArrayType, FixedVectorType>(EltTy)) {
if (EltTy->isEmptyTy())
return 0;
if (auto *ST = dyn_cast<StructType>(EltTy)) {
// Check that struct is homogeneous.
for (const auto *Ty : ST->elements())
if (Ty != *ST->element_begin())
return 0;
N *= ST->getNumElements();
EltTy = *ST->element_begin();
} else if (auto *AT = dyn_cast<ArrayType>(EltTy)) {
N *= AT->getNumElements();
EltTy = AT->getElementType();
} else {
auto *VT = cast<FixedVectorType>(EltTy);
N *= VT->getNumElements();
EltTy = VT->getElementType();
}
}
if (!isValidElementType(EltTy, Opts.slp_revec))
return 0;
size_t VTSize = DL->getTypeStoreSizeInBits(getWidenedType(EltTy, N));
if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize ||
VTSize != DL->getTypeStoreSizeInBits(T))
return 0;
return N;
}
bool BoUpSLP::canVectorStoreInsertValue(const TreeEntry *E) const {
if (E->getOpcode() != Instruction::InsertValue ||
E->State != TreeEntry::Vectorize)
return false;
bool HasStore = false;
for (Value *V : E->Scalars) {
auto *IV = dyn_cast<InsertValueInst>(V);
if (!IV || isDeleted(IV))
return false;
bool HasExternalUser = false;
for (User *U : IV->users()) {
if (auto *UI = dyn_cast<Instruction>(U); UI && isDeleted(UI))
continue;
if (is_contained(E->Scalars, U))
continue;
HasExternalUser = true;
// Only a plain (non-volatile, non-atomic) store may consume the vector
// directly. Volatile/atomic stores must keep their original aggregate
// type and access semantics, so fall back to the store + load roundtrip.
auto *SI = dyn_cast<StoreInst>(U);
if (!SI || !SI->isSimple())
return false;
HasStore = true;
}
if (HasExternalUser && !IV->hasOneUse())
return false;
}
return HasStore;
}
FixedVectorType *BoUpSLP::getInsertBuildVectorSrcTy(const TreeEntry *E) const {
Value *VL0 = E->getMainOp();
if (E->getOpcode() == Instruction::InsertElement)
return cast<FixedVectorType>(VL0->getType());
assert(E->getOpcode() == Instruction::InsertValue &&
"Expected InsertElement or InsertValue node.");
unsigned MaxIdx = E->Scalars.size() - 1;
for (InsertValueInst *I : make_isa_range<InsertValueInst>(E->Scalars))
MaxIdx = std::max(MaxIdx, I->getIndices().front());
return cast<FixedVectorType>(
getWidenedType(getValueType(VL0, Opts.slp_revec), MaxIdx + 1));
}
bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL,
SmallVectorImpl<unsigned> &CurrentOrder,
bool ResizeAllowed) const {
const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
assert(It != VL.end() && "Expected at least one extract instruction.");
auto *E0 = cast<Instruction>(*It);
// Check if all of the extracts come from the same vector and from the
// correct offset.
Value *Vec = E0->getOperand(0);
CurrentOrder.clear();
// We have to extract from a vector/aggregate with the same number of elements.
unsigned NElts;
if (E0->getOpcode() == Instruction::ExtractValue) {
NElts = canMapToVector(Vec->getType());
if (!NElts)
return false;
// Check if load can be rewritten as load of vector.
LoadInst *LI = dyn_cast<LoadInst>(Vec);
if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
return false;
} else {
NElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
}
unsigned E = VL.size();
if (!ResizeAllowed && NElts != E)
return false;
SmallVector<int> Indices(E, PoisonMaskElem);
unsigned MinIdx = NElts, MaxIdx = 0;
// Non-extract lanes (copyable elements modeled on an extract main op, or
// undefs) are treated as holes.
for (auto [I, V] : make_filter_range(enumerate(VL), [](const auto &P) {
return isa<ExtractElementInst, ExtractValueInst>(P.value());
})) {
auto *Inst = cast<Instruction>(V);
if (Inst->getOperand(0) != Vec)
return false;
if (auto *EE = dyn_cast<ExtractElementInst>(Inst))
if (isa<UndefValue>(EE->getIndexOperand()))
continue;
std::optional<unsigned> Idx = getExtractIndex(Inst);
if (!Idx)
return false;
const unsigned ExtIdx = *Idx;
if (ExtIdx >= NElts)
continue;
Indices[I] = ExtIdx;
if (MinIdx > ExtIdx)
MinIdx = ExtIdx;
if (MaxIdx < ExtIdx)
MaxIdx = ExtIdx;
}
if (MaxIdx - MinIdx + 1 > E)
return false;
if (MaxIdx + 1 <= E)
MinIdx = 0;
// Check that all of the indices extract from the correct offset.
bool ShouldKeepOrder = true;
// Assign to all items the initial value E + 1 so we can check if the extract
// instruction index was used already.
// Also, later we can check that all the indices are used and we have a
// consecutive access in the extract instructions, by checking that no
// element of CurrentOrder still has value E + 1.
CurrentOrder.assign(E, E);
for (unsigned I = 0; I < E; ++I) {
if (Indices[I] == PoisonMaskElem)
continue;
const unsigned ExtIdx = Indices[I] - MinIdx;
if (CurrentOrder[ExtIdx] != E) {
CurrentOrder.clear();
return false;
}
ShouldKeepOrder &= ExtIdx == I;
CurrentOrder[ExtIdx] = I;
}
if (ShouldKeepOrder)
CurrentOrder.clear();
return ShouldKeepOrder;
}
bool BoUpSLP::areAllUsersVectorized(
Instruction *I, const SmallDenseSet<Value *> *VectorizedVals) const {
return (I->hasOneUse() && (!VectorizedVals || VectorizedVals->contains(I))) ||
all_of(I->users(), [this](User *U) {
return isVectorized(U) || isVectorLikeInstWithConstOps(U) ||
(isa<ExtractElementInst>(U) && MustGather.contains(U));
});
}
static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
const InstructionsState &S,
DominatorTree &DT, const DataLayout &DL,
TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const BoUpSLP &R);
/// \returns true if \p Op is a contractable single-use fmul, which the backend
/// fuses with its fadd/fsub user \p I: \p Op is in the block of \p I or the
/// target sinks it there.
static bool isFusableFMulOperand(Value *Op, Instruction *I,
const TargetTransformInfo &TTI) {
Instruction *FMul;
if (!match(Op, m_Instruction(FMul, m_OneUse(m_AllowContract(
m_FMul(m_Value(), m_Value()))))))
return false;
if (FMul->getParent() == I->getParent())
return true;
SmallVector<Use *> OpsToSink;
return TTI.isProfitableToSinkOperands(I, OpsToSink) &&
any_of(OpsToSink, [&](const Use *U) { return U->get() == FMul; });
}
/// \returns the fmul operand of the fadd/fsub \p I that the backend fuses with
/// \p I into an fmuladd: the first fusable one from either operand, c - a*b
/// included. \p I must allow contraction.
static Instruction *getFusableFMulOperand(Instruction *I,
const TargetTransformInfo &TTI) {
if (!match(I, m_AllowContract(m_CombineOr(m_FAdd(m_Value(), m_Value()),
m_FSub(m_Value(), m_Value())))))
return nullptr;
for (Value *Op : I->operands())
if (isFusableFMulOperand(Op, I, TTI))
return cast<Instruction>(Op);
return nullptr;
}
/// \returns the cost of the binary operator \p I, priced as not fused. No
/// context instruction is passed: targets that model the fmuladd fusion would
/// discount the unfused side of the comparison as well.
static InstructionCost getUnfusedBinOpCost(Instruction *I,
TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
TTI::TargetCostKind CostKind) {
assert(match(I, m_BinOp()) && "Expected a binary operator.");
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
return TTI.getArithmeticInstrCost(
I->getOpcode(), I->getType(), CostKind, Op1Info, Op2Info,
{I->getOperand(0), I->getOperand(1)}, /*CxtI=*/nullptr, &TLI);
}
std::pair<Instruction *, InstructionCost>
BoUpSLP::getFMulFusingUser(Instruction *I, bool InTree) const {
if (!match(I, m_OneUse(m_FMul(m_Value(), m_Value()))))
return {nullptr, InstructionCost::getInvalid()};
auto *U = cast<Instruction>(I->user_back());
// The reduction cost accounts for the fusion of the reduced values into the
// reduction operations itself. The lanes of the combined fmuladd node are
// fused with its own fmul operand node, not with the other operand.
if (getFusableFMulOperand(U, *TTI) != I ||
(InTree && ((UserIgnoreList && UserIgnoreList->contains(U)) ||
any_of(getTreeEntries(U), [](const TreeEntry *TE) {
return TE->CombinedOp == TreeEntry::FMulAdd;
}))))
return {nullptr, InstructionCost::getInvalid()};
// The target must actually prefer the fused form.
InstructionCost FMACost =
canConvertToFMA(U, InstructionsState(U, U), *DT, *DL, *TTI, *TLI, *this);
return {FMACost.isValid() ? U : nullptr, FMACost};
}
bool BoUpSLP::isFusedVectorizedFAddSub(Instruction *U) const {
return any_of(getTreeEntries(U), [&](const TreeEntry *TE) {
return !DeletedNodes.contains(TE) && !TransformedToGatherNodes.contains(TE);
});
}
InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const {
if (!TE.hasState() || TE.isGather() ||
(TE.getOpcode() != Instruction::FMul &&
TE.getAltOpcode() != Instruction::FMul))
return TTI::TCC_Free;
InstructionCost Penalty = TTI::TCC_Free;
for (Value *V : TE.Scalars) {
Instruction *I;
if (!match(V, m_Instruction(I, m_FMul(m_Value(), m_Value()))) ||
(TE.hasCopyableElements() && TE.isCopyableElement(I)))
continue;
if (Instruction *U = getFMulFusingUser(I).first;
U && isFusedVectorizedFAddSub(U))
Penalty += getUnfusedBinOpCost(I, *TTI, *TLI, CostKind);
}
return Penalty;
}
// A poor-throughput entry's real vector-vs-scalar savings (fdiv/frem/fsqrt)
// are already folded into TreeCost like any other entry, including all
// shuffle/insert/extract overhead elsewhere in the tree. So bypassing the
// instruction-count veto just has to trust that already-complete TreeCost
// instead of guessing at extra savings: heavy gather/shuffle overhead raises
// TreeCost too, and will still block the bypass.
bool BoUpSLP::bypassesInstCountCheck(InstructionCost TreeCost) const {
if (!Opts.slp_vectorize_poor_throughput || TreeCost >= -getCostThreshold())
return false;
PoorThroughputOpCache Cache;
return any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &Ptr) {
const TreeEntry &TE = *Ptr;
return TE.hasState() && !DeletedNodes.contains(&TE) && !TE.isGather() &&
!TransformedToGatherNodes.contains(&TE) &&
TE.State != TreeEntry::CombinedVectorize &&
isPoorThroughputOp(Opts, TE.getMainOp(), *TTI, *TLI, Cache,
CostKind);
});
}
bool BoUpSLP::collectGatherUserOps(
const TreeEntry &E, SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) const {
UserOps.clear();
for (const auto &TE :
make_filter_range(VectorizableTree, [&](const auto &TE) {
return !DeletedNodes.contains(TE.get());
})) {
if (!(TE->isGather() || TransformedToGatherNodes.contains(TE.get())) ||
!E.isSame(TE->Scalars))
continue;
const TreeEntry *UserTE = TE->UserTreeIndex.UserTE;
if (!UserTE || !UserTE->hasState() || UserTE->isAltShuffle() ||
TransformedToGatherNodes.contains(UserTE))
return false;
Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
if (auto *II = dyn_cast<IntrinsicInst>(UserTE->getMainOp()))
IntrinsicID = II->getIntrinsicID();
UserOps.emplace_back(UserTE->getOpcode(), IntrinsicID,
TE->UserTreeIndex.EdgeIdx);
}
return !UserOps.empty();
}
uint64_t BoUpSLP::getNumScalarInsts(bool HasTreeLoop) {
uint64_t Total = 0;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
const TreeEntry &TE = *Ptr;
if (DeletedNodes.contains(&TE))
continue;
// CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the
// cmp child of a MinMax select) are absorbed into the parent on both
// scalar and vector sides. The backend fuses fadd+fmul -> fma and
// select+cmp -> smin/smax even for scalar code, so skip to avoid
// double-counting. Checked before the scale lookup below: some of
// these entries started out as constant-only gather nodes and have no
// valid instruction state to key the loop-nest scale off of.
if (TE.State == TreeEntry::CombinedVectorize)
continue;
uint64_t Scale = getEntryEffectiveScale(TE);
if (HasTreeLoop && Scale <= 1)
continue;
unsigned Count = 0;
if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) {
// Count extractelement scalars in gathers - they exist in the scalar
// code regardless of vectorization. ExtractElement instructions
// become free when the vector input is used directly.
for (Value *V : TE.Scalars)
if (isa<ExtractElementInst>(V))
++Count;
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
// Each vectorize entry represents a bundle of scalar instructions.
// Count per-entry without cross-entry deduplication, since shared
// scalars across entries still represent separate work in scalar code.
for (Instruction *I : make_isa_range<Instruction>(TE.Scalars)) {
if (TE.hasCopyableElements() && TE.isCopyableElement(I))
continue;
++Count;
// Calculate calls/divs/rems twice, they may cost higher, so better to
// include their count twice to mimic slightly real cost here.
if (I->isIntDivRem() || I->isFPDivRem())
++Count;
if (auto *CI = dyn_cast<CallInst>(I)) {
Intrinsic::ID BaseID = getVectorIntrinsicIDForCall(CI, TLI);
if (!isTriviallyVectorizable(BaseID))
++Count;
}
}
// Count peeled intermediates in the scalar baseline (not in Scalars).
Count += TE.getReassocScalars().size();
// Even when the whole node is not combined, individual scalar
// instructions may be fused by the backend. Each fused pair (e.g.
// fadd+fmul -> fma, select+cmp -> smin/smax) becomes a single scalar
// instruction, absorbing the operand instruction. Subtract 1 for each
// such match to avoid over-counting the scalar side.
if (TE.CombinedOp == TreeEntry::NotCombinedOp && TE.hasState()) {
unsigned Opcode = TE.getOpcode();
if (Opcode == Instruction::Select) {
for (Value *V : make_filter_range(TE.Scalars, [&](Value *V) {
return !TE.hasCopyableElements() || !TE.isCopyableElement(V);
})) {
auto *SI = dyn_cast<SelectInst>(V);
if (!SI)
continue;
auto [ID, _] = canConvertToMinOrMaxIntrinsic({V});
if (ID != Intrinsic::not_intrinsic) {
assert(Count > 0 && "Underflow in scalar inst count (minmax)");
--Count;
}
}
} else {
for (Instruction *I : make_filter_range(
make_isa_range<Instruction>(TE.Scalars), [&](Instruction *V) {
return !TE.hasCopyableElements() || !TE.isCopyableElement(V);
})) {
if (match(I, m_CombineOr(m_FAdd(m_Value(), m_Value()),
m_FSub(m_Value(), m_Value())))) {
if (canConvertToFMA(I, InstructionsState(I, I), *DT, *DL, *TTI,
*TLI, *this)
.isValid()) {
assert(Count > 0 && "Underflow in scalar inst count (fma)");
--Count;
}
} else if (match(I, m_FMul(m_Value(), m_Value()))) {
// The fmul, fused into a scalar user, disappears into the user's
// fmuladd. A vectorized fadd/fsub user lane already accounts for
// the fusion.
Instruction *U = getFMulFusingUser(I).first;
if (U && !isFusedVectorizedFAddSub(U)) {
assert(Count > 0 && "Underflow in scalar inst count (fma)");
--Count;
}
}
}
}
}
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
}
return Total;
}
uint64_t BoUpSLP::getNumVectorInsts(bool HasTreeLoop, bool CountExtracts) {
uint64_t Total = 0;
// The alternate op, fused with the multiplication by the target, is counted
// with the multiplication node, the same way as the scalar fmas.
auto IsFusedAlt = [&](const TreeEntry &TE) {
if (!TE.hasState() || TE.State != TreeEntry::Vectorize ||
DeletedNodes.contains(&TE) || TransformedToGatherNodes.contains(&TE) ||
!TE.isAltShuffle() || TE.hasReassocScalars() ||
!TE.getMainOp()->getType()->isFloatingPointTy() ||
!TE.getOperations().isAddSubLikeOp())
return false;
Type *ScalarTy = TE.getMainOp()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, TE.Scalars.size()));
SmallBitVector OpcodeMask(getAltInstrMask(
TE.Scalars, ScalarTy, TE.getOpcode(), TE.getAltOpcode()));
return TTI->isLegalAltInstr(VecTy, TE.getOpcode(), TE.getAltOpcode(),
OpcodeMask, TE.Scalars) &&
canConvertToFMA(TE.Scalars, TE.getOperations(), *DT, *DL, *TTI, *TLI,
*this)
.isValid();
};
// The scalar count already credits the fused scalars as fmas, so only a tree
// with the fused alternate op is counted at the level of the machine
// instructions. Doing the same for the other trees changes their VF=2
// decisions.
const bool HasFusedAlt =
any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &Ptr) {
return IsFusedAlt(*Ptr);
});
// Source vector -> max scale among the gather entries sharing it, so the
// combined shufflevector is still weighted like an in-loop entry below.
SmallDenseMap<Value *, uint64_t, 4> GatherExtractSourceVecs;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
const TreeEntry &TE = *Ptr;
if (DeletedNodes.contains(&TE))
continue;
if (TE.State == TreeEntry::CombinedVectorize)
continue;
uint64_t Scale = getEntryEffectiveScale(TE);
if (HasTreeLoop && Scale <= 1)
continue;
unsigned Count = 0;
if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) {
if (TE.hasState()) {
if (const TreeEntry *E =
getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars);
E && E != &TE && E->getVectorFactor() == TE.getVectorFactor())
continue;
SmallVector<Value *> RevScalars(TE.Scalars.rbegin(), TE.Scalars.rend());
if (const TreeEntry *E =
getSameValuesTreeEntry(TE.getMainOp(), RevScalars);
E && E->getVectorFactor() == TE.getVectorFactor()) {
Total = SaturatingAdd(Total, Scale);
continue;
}
}
// ExtractElement gathers from the same source vector become a single
// shufflevector. Collect source vectors globally across all gather
// entries and count once at the end.
if (all_of(TE.Scalars,
IsaPred<ExtractElementInst, UndefValue, Constant>)) {
for (ExtractElementInst *EE :
make_isa_range<ExtractElementInst>(TE.Scalars)) {
uint64_t &VecScale =
GatherExtractSourceVecs.try_emplace(EE->getVectorOperand(), 0)
.first->second;
VecScale = std::max(VecScale, Scale);
}
} else {
// A splat is a single broadcast. In loops, its per-lane count, scaled
// by the trip count, rejects the profitable trees.
if ((HasFusedAlt || HasTreeLoop) && isSplat(TE.Scalars))
Count = !isConstant(TE.Scalars.front());
else
Count = TE.Scalars.size() - count_if(TE.Scalars, isConstant);
}
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
// InsertElement/ExtractElement vectorize entries don't produce real
// vector instructions - InsertElement at root IS the result, and
// ExtractElement entries reference the input vector directly.
if (TE.getOpcode() == Instruction::InsertElement ||
TE.getOpcode() == Instruction::ExtractElement)
continue;
if (TE.getOpcode() == Instruction::InsertValue) {
// InsertValue materializes via store + load unless it is stored directly
// as a vector.
if (!canVectorStoreInsertValue(&TE))
Count += 2;
if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
++Count;
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
if (IsFusedAlt(TE))
continue;
if (TE.State == TreeEntry::SplitVectorize)
Count += 2;
else if (TE.hasReassocScalars())
// N operand columns need N-1 binary combines.
Count += TE.getNumOperands() - 1;
else
++Count;
if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
++Count;
Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
}
for (const auto &VecAndScale : GatherExtractSourceVecs)
Total = SaturatingAdd(Total, VecAndScale.second);
if (!CountExtracts)
return Total;
// Count extract instructions from ExternalUses, skipping insertelements
// (those get folded into shuffles, not real extracts).
SmallPtrSet<Value *, 8> CountedExtracts;
for (const ExternalUser &EU :
make_filter_range(ExternalUses, [](const ExternalUser &EU) {
return !isa_and_nonnull<InsertElementInst>(EU.User);
})) {
if (EU.User && EphValues.count(EU.User))
continue;
// Vector-typed scalars are consumed as whole vectors, no extract
// instruction is emitted for them.
if (isVectorizedTy(EU.Scalar->getType()) &&
(!Opts.slp_revec ||
(EU.E.hasState() && EU.E.getOpcode() == Instruction::InsertElement)))
continue;
if (ExternalUsesAsOriginalScalar.contains(EU.Scalar))
continue;
if (!CountedExtracts.insert(EU.Scalar).second)
continue;
// The extract of the lane, which is free in the target, is not emitted.
Type *ScalarTy = EU.Scalar->getType();
if (HasFusedAlt && !MinBWs.contains(&EU.E) &&
VectorType::isValidElementType(ScalarTy) &&
TTI->getVectorInstrCost(
Instruction::ExtractElement,
getWidenedType(ScalarTy, EU.E.getVectorFactor()), CostKind,
EU.Lane) == TTI::TCC_Free)
continue;
++Total;
}
return Total;
}
void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
const function_ref<bool(Instruction *)> IsAltOp, SmallVectorImpl<int> &Mask,
SmallVectorImpl<Value *> *OpScalars,
SmallVectorImpl<Value *> *AltScalars) const {
unsigned Sz = Scalars.size();
Mask.assign(Sz, PoisonMaskElem);
SmallVector<int> OrderMask;
if (!ReorderIndices.empty())
inversePermutation(ReorderIndices, OrderMask);
for (unsigned I = 0; I < Sz; ++I) {
unsigned Idx = I;
if (!ReorderIndices.empty())
Idx = OrderMask[I];
if (isa<PoisonValue>(Scalars[Idx]))
continue;
auto *OpInst = cast<Instruction>(Scalars[Idx]);
if (IsAltOp(OpInst)) {
Mask[I] = Sz + Idx;
if (AltScalars)
AltScalars->push_back(OpInst);
} else {
Mask[I] = Idx;
if (OpScalars)
OpScalars->push_back(OpInst);
}
}
if (!ReuseShuffleIndices.empty()) {
SmallVector<int> NewMask(ReuseShuffleIndices.size(), PoisonMaskElem);
transform(ReuseShuffleIndices, NewMask.begin(), [&Mask](int Idx) {
return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem;
});
Mask.swap(NewMask);
}
}
static bool isMainInstruction(Instruction *I, Instruction *MainOp,
Instruction *AltOp,
const TargetLibraryInfo &TLI) {
return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == MainOp;
}
void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
assert(TE.isGather() && TE.ReorderIndices.empty() &&
"Expected gather node without reordering.");
DenseMap<std::pair<size_t, Value *>, SmallVector<LoadInst *>> LoadsMap;
SmallSet<size_t, 2> LoadKeyUsed;
// Do not reorder nodes if it small (just 2 elements), all-constant or all
// instructions have same opcode already.
if (TE.Scalars.size() == 2 || (TE.hasState() && !TE.isAltShuffle()) ||
all_of(TE.Scalars, isConstant))
return;
if (any_of(seq<unsigned>(TE.Idx), [&](unsigned Idx) {
return VectorizableTree[Idx]->isSame(TE.Scalars);
}))
return;
auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
Key = hash_combine(hash_value(LI->getParent()->getNumber()), Key);
Value *Ptr = getUnderlyingObject(LI->getPointerOperand(),
Opts.slp_recursion_max_depth);
if (LoadKeyUsed.contains(Key)) {
auto LIt = LoadsMap.find(std::make_pair(Key, Ptr));
if (LIt != LoadsMap.end()) {
for (LoadInst *RLI : LIt->second) {
if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(),
LI->getType(), LI->getPointerOperand(), *DL, *SE,
/*StrictCheck=*/true))
return hash_value(RLI->getPointerOperand());
}
for (LoadInst *RLI : LIt->second) {
if (arePointersCompatible(RLI->getPointerOperand(),
LI->getPointerOperand(), *TLI,
Opts.slp_recursion_max_depth)) {
hash_code SubKey = hash_value(RLI->getPointerOperand());
return SubKey;
}
}
if (LIt->second.size() > 2) {
hash_code SubKey =
hash_value(LIt->second.back()->getPointerOperand());
return SubKey;
}
}
}
LoadKeyUsed.insert(Key);
LoadsMap.try_emplace(std::make_pair(Key, Ptr)).first->second.push_back(LI);
return hash_value(LI->getPointerOperand());
};
MapVector<size_t, MapVector<size_t, SmallVector<Value *>>> SortedValues;
SmallDenseMap<Value *, SmallVector<unsigned>, 8> KeyToIndex;
bool IsOrdered = true;
unsigned NumInstructions = 0;
// Try to "cluster" scalar instructions, to be able to build extra vectorized
// nodes.
for (auto [I, V] : enumerate(TE.Scalars)) {
size_t Key = 1, Idx = 1;
if (auto *Inst = dyn_cast<Instruction>(V);
Inst && !isa<ExtractElementInst, LoadInst, CastInst>(V) &&
!isDeleted(Inst) && !isVectorized(V)) {
std::tie(Key, Idx) = generateKeySubkey(V, TLI, GenerateLoadsSubkey,
/*AllowAlternate=*/false);
++NumInstructions;
}
auto &Container = SortedValues[Key];
if (IsOrdered && !KeyToIndex.contains(V) &&
!(isa<Constant, ExtractElementInst>(V) ||
isVectorLikeInstWithConstOps(V)) &&
((Container.contains(Idx) &&
KeyToIndex.at(Container[Idx].back()).back() != I - 1) ||
(!Container.empty() && !Container.contains(Idx) &&
KeyToIndex.at(Container.back().second.back()).back() != I - 1)))
IsOrdered = false;
auto &KTI = KeyToIndex[V];
if (KTI.empty())
Container[Idx].push_back(V);
KTI.push_back(I);
}
SmallVector<std::pair<unsigned, unsigned>> SubVectors;
APInt DemandedElts = APInt::getAllOnes(TE.Scalars.size());
if (!IsOrdered && NumInstructions > 1) {
unsigned Cnt = 0;
TE.ReorderIndices.resize(TE.Scalars.size(), TE.Scalars.size());
for (const auto &D : SortedValues) {
for (const auto &P : D.second) {
unsigned Sz = 0;
for (Value *V : P.second) {
ArrayRef<unsigned> Indices = KeyToIndex.at(V);
for (auto [K, Idx] : enumerate(Indices)) {
TE.ReorderIndices[Cnt + K] = Idx;
TE.Scalars[Cnt + K] = V;
}
Sz += Indices.size();
Cnt += Indices.size();
}
if (Sz > 1 && isa<Instruction>(P.second.front())) {
const unsigned SubVF = getFloorFullVectorNumberOfElements(
*TTI, TE.Scalars.front()->getType(), Sz, Opts.slp_revec);
SubVectors.emplace_back(Cnt - Sz, SubVF);
for (unsigned I : seq<unsigned>(Cnt - Sz, Cnt - Sz + SubVF))
DemandedElts.clearBit(I);
} else if (!P.second.empty() && isConstant(P.second.front())) {
for (unsigned I : seq<unsigned>(Cnt - Sz, Cnt))
DemandedElts.clearBit(I);
}
}
}
}
// Reuses always require shuffles, so consider it as profitable.
if (!TE.ReuseShuffleIndices.empty() || TE.ReorderIndices.empty())
return;
// Do simple cost estimation.
InstructionCost Cost = 0;
auto *ScalarTy = TE.Scalars.front()->getType();
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, TE.Scalars.size()));
for (auto [Idx, Sz] : SubVectors) {
Cost += getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, CostKind, {},
Idx, cast<VectorType>(getWidenedType(ScalarTy, Sz)));
}
Cost += getScalarizationOverhead(*TTI, Opts.slp_revec, ScalarTy, VecTy,
DemandedElts,
/*Insert=*/true,
/*Extract=*/false, CostKind);
int Sz = TE.Scalars.size();
SmallVector<int> ReorderMask(TE.ReorderIndices.begin(),
TE.ReorderIndices.end());
for (unsigned I : seq<unsigned>(Sz)) {
Value *V = TE.getOrdered(I);
if (isa<PoisonValue>(V)) {
ReorderMask[I] = PoisonMaskElem;
} else if (isConstant(V) || DemandedElts[I]) {
ReorderMask[I] = I + TE.ReorderIndices.size();
}
}
Cost += getShuffleCost(*TTI,
any_of(ReorderMask, [&](int I) { return I >= Sz; })
? TTI::SK_PermuteTwoSrc
: TTI::SK_PermuteSingleSrc,
VecTy, CostKind, ReorderMask);
DemandedElts = APInt::getAllOnes(TE.Scalars.size());
ReorderMask.assign(Sz, PoisonMaskElem);
for (unsigned I : seq<unsigned>(Sz)) {
Value *V = TE.getOrdered(I);
if (isConstant(V)) {
DemandedElts.clearBit(I);
if (!isa<PoisonValue>(V))
ReorderMask[I] = I;
} else {
ReorderMask[I] = I + Sz;
}
}
InstructionCost BVCost = getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, VecTy, DemandedElts,
/*Insert=*/true, /*Extract=*/false, CostKind);
if (!DemandedElts.isAllOnes())
BVCost += getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, CostKind,
ReorderMask);
if (Cost >= BVCost) {
SmallVector<int> Mask(TE.ReorderIndices.begin(), TE.ReorderIndices.end());
reorderScalars(TE.Scalars, Mask);
TE.ReorderIndices.clear();
}
}
/// Check if we can convert fadd/fsub sequence to FMAD.
/// \returns Cost of the FMAD, if conversion is possible, invalid cost
/// otherwise.
static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
const InstructionsState &S,
DominatorTree &DT, const DataLayout &DL,
TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
const BoUpSLP &R) {
assert(all_of(VL,
[](Value *V) {
return V->getType()->getScalarType()->isFloatingPointTy();
}) &&
"Can only convert to FMA for floating point types");
assert(S.isAddSubOrFNegLikeOp() &&
"Can only convert to FMA for add/sub or fneg chain links");
const TTI::TargetCostKind CostKind = R.getCostKind();
auto CheckForContractable = [](ArrayRef<Value *> VL,
const InstructionsState &S) {
FastMathFlags FMF;
FMF.set();
for (Instruction *I : make_isa_range<Instruction>(VL)) {
if (S.isCopyableElement(I))
continue;
Instruction *MatchingI = S.getMatchingMainOpOrAltOp(I);
if (S.getMainOp() != MatchingI && S.getAltOp() != MatchingI)
continue;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
}
return FMF.allowContract();
};
if (!CheckForContractable(VL, S))
return InstructionCost::getInvalid();
// fmul also should be contractable
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL, R);
// The operands follow the majority pattern of the lanes, while the backend
// fuses the fmul from either operand (c - a*b included); move it to the
// first column.
if (Operands.size() == 2) {
for (auto [Idx, V] : enumerate(VL)) {
auto *I = dyn_cast<Instruction>(V);
if (!I || S.isCopyableElement(I))
continue;
if (Instruction *FMul = getFusableFMulOperand(I, TTI);
FMul && Operands[1][Idx] == FMul)
std::swap(Operands[0][Idx], Operands[1][Idx]);
}
}
// The lanes without the fmul, e.g. the constant lanes of the copyable nodes,
// are the copyable lanes of the fmul column.
InstructionsState OpS = Analysis.buildInstructionsState(Operands.front(), R);
if (!OpS.valid())
return InstructionCost::getInvalid();
if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
return InstructionCost::getInvalid();
if (!CheckForContractable(Operands.front(), OpS))
return InstructionCost::getInvalid();
// Compare the costs.
InstructionCost FMulPlusFAddCost = 0;
InstructionCost FMACost = 0;
FastMathFlags FMF;
FMF.set();
const bool IsArithmeticState = S.isAddSubLikeOp() || S.isMulDivLikeOp() ||
S.isShiftOp() || S.isBitwiseLogicOp();
// A dropped fneg link stands in for the sign-flipped combine its operand
// joins and is costed as an fadd plus the fneg itself (both are removed).
auto GetLinkCost = [&](Instruction *I, bool IsCopyable = false) {
if (S.getOpcode() != Instruction::FNeg) {
if (IsCopyable || !IsArithmeticState ||
(I->getOpcode() != S.getOpcode() &&
I->getOpcode() != S.getAltOpcode()))
return TTI.getInstructionCost(I, CostKind);
return getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
return TTI.getArithmeticInstrCost(Instruction::FAdd, I->getType(), CostKind,
{},
TTI::getOperandInfo(I->getOperand(0))) +
TTI.getArithmeticInstrCost(Instruction::FNeg, I->getType(), CostKind,
TTI::getOperandInfo(I->getOperand(0)));
};
for (Instruction *I : make_isa_range<Instruction>(VL)) {
const bool IsCopyable = S.isCopyableElement(I);
if (!IsCopyable)
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
FMulPlusFAddCost += GetLinkCost(I, IsCopyable);
}
unsigned NumOps = 0;
for (auto [V, Op] :
make_filter_range(zip(VL, Operands.front()), [&](const auto &P) {
return !S.isCopyableElement(std::get<0>(P));
})) {
auto *I = dyn_cast<Instruction>(Op);
if (!I || OpS.isCopyableElement(I) ||
!isFusableFMulOperand(I, cast<Instruction>(V), TTI)) {
if (auto *OpI = dyn_cast<Instruction>(V))
FMACost += GetLinkCost(OpI);
if (I)
FMACost += TTI.getInstructionCost(I, CostKind);
continue;
}
++NumOps;
if (auto *FPCI = dyn_cast<FPMathOperator>(I))
FMF &= FPCI->getFastMathFlags();
FMulPlusFAddCost += getUnfusedBinOpCost(I, TTI, TLI, CostKind);
}
Type *Ty = VL.front()->getType();
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
FMACost += NumOps * TTI.getIntrinsicInstrCost(ICA, CostKind);
return FMACost < FMulPlusFAddCost ? FMACost : InstructionCost::getInvalid();
}
bool BoUpSLP::matchesShlZExt(const TreeEntry &TE, OrdersType &Order,
bool &IsBSwap, bool &ForLoads) const {
assert(TE.hasState() && TE.getOpcode() == Instruction::Shl &&
"Expected Shl node.");
IsBSwap = false;
ForLoads = false;
if (TE.State != TreeEntry::Vectorize || TE.isAltShuffle() ||
!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty() ||
MinBWs.contains(&TE) ||
any_of(TE.Scalars, [](Value *V) { return !V->hasOneUse(); }))
return false;
Type *ScalarTy = TE.getMainOp()->getType();
// TODO: Check if same can be done for the vector types.
if (!ScalarTy->isIntegerTy())
return false;
if (ScalarTy->isVectorTy())
return false;
const unsigned Sz = DL->getTypeSizeInBits(ScalarTy);
const TreeEntry *LhsTE = getOperandEntry(&TE, /*Idx=*/0);
const TreeEntry *RhsTE = getOperandEntry(&TE, /*Idx=*/1);
// Lhs should be zext i<stride> to I<sz>.
if (!(LhsTE->State == TreeEntry::Vectorize &&
LhsTE->getOpcode() == Instruction::ZExt && !LhsTE->isAltShuffle() &&
LhsTE->ReorderIndices.empty() && LhsTE->ReuseShuffleIndices.empty() &&
!MinBWs.contains(LhsTE) &&
all_of(LhsTE->Scalars, [](Value *V) { return V->hasOneUse(); })))
return false;
Type *SrcScalarTy = cast<ZExtInst>(LhsTE->getMainOp())->getSrcTy();
unsigned Stride = DL->getTypeSizeInBits(SrcScalarTy);
if (!isPowerOf2_64(Stride) || Stride >= Sz || Sz % Stride != 0 ||
!isPowerOf2_64(LhsTE->getVectorFactor()))
return false;
if (!(RhsTE->isGather() && RhsTE->ReorderIndices.empty() &&
RhsTE->ReuseShuffleIndices.empty() && !MinBWs.contains(RhsTE)))
return false;
Order.clear();
unsigned CurrentValue = 0;
// Rhs should be (0, Stride, 2 * Stride, ..., N-Stride), where N <= Sz.
if (all_of(RhsTE->Scalars,
[&](Value *V) {
CurrentValue += Stride;
if (isa<UndefValue>(V))
return true;
auto *C = dyn_cast<Constant>(V);
if (!C)
return false;
return C->getUniqueInteger() == CurrentValue - Stride;
}) &&
CurrentValue <= Sz) {
Order.clear();
} else {
const unsigned VF = RhsTE->getVectorFactor();
Order.assign(VF, VF);
// Track which logical positions we've seen; reject duplicate shift amounts.
SmallBitVector SeenPositions(VF);
// Check if need to reorder Rhs to make it in form (0, Stride, 2 * Stride,
// ..., N-Stride), where N <= Sz.
if (VF * Stride > Sz)
return false;
for (const auto [Idx, V] :
make_filter_range(enumerate(RhsTE->Scalars), [](const auto &P) {
return !isa<UndefValue>(P.value());
})) {
auto *C = dyn_cast<Constant>(V);
if (!C)
return false;
const APInt &Val = C->getUniqueInteger();
if (Val.isNegative() || Val.uge(Sz) || Val.getZExtValue() % Stride != 0)
return false;
unsigned Pos = Val.getZExtValue() / Stride;
// TODO: Support Pos >= VF, in this case need to shift the final value.
if (Order[Idx] != VF || Pos >= VF)
return false;
if (SeenPositions.test(Pos))
return false;
SeenPositions.set(Pos);
Order[Idx] = Pos;
}
// One of the indices not set - exit.
if (is_contained(Order, VF))
return false;
}
auto *SrcType = IntegerType::getIntNTy(ScalarTy->getContext(),
Stride * LhsTE->getVectorFactor());
FastMathFlags FMF;
SmallPtrSet<Value *, 4> CheckedExtracts;
auto *VecTy =
cast<VectorType>(getWidenedType(ScalarTy, TE.getVectorFactor()));
auto *SrcVecTy =
cast<VectorType>(getWidenedType(SrcScalarTy, LhsTE->getVectorFactor()));
TTI::CastContextHint CastCtx =
getCastContextHint(*getOperandEntry(LhsTE, /*Idx=*/0));
InstructionCost VecCost =
TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
TTI::getOperandInfo(LhsTE->Scalars),
TTI::getOperandInfo(RhsTE->Scalars), {},
TE.getMainOp()) +
TTI->getCastInstrCost(
Instruction::ZExt, VecTy,
getWidenedType(SrcScalarTy, LhsTE->getVectorFactor()), CastCtx,
CostKind, LhsTE->getMainOp());
InstructionCost BitcastCost = TTI->getCastInstrCost(
Instruction::BitCast, SrcType, SrcVecTy, CastCtx, CostKind);
if (!Order.empty()) {
fixupOrderingIndices(Order);
SmallVector<int> Mask;
inversePermutation(Order, Mask);
BitcastCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, SrcVecTy,
CostKind, Mask);
}
// Check if the combination can be modeled as a bitcast+byteswap operation.
constexpr unsigned ByteSize = 8;
if (!Order.empty() && isReverseOrder(Order) &&
DL->getTypeSizeInBits(SrcScalarTy) == ByteSize) {
IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, SrcType, {SrcType});
InstructionCost BSwapCost =
TTI->getCastInstrCost(Instruction::BitCast, SrcType, SrcVecTy, CastCtx,
CostKind) +
TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
if (BSwapCost <= BitcastCost) {
BitcastCost = BSwapCost;
IsBSwap = true;
Order.clear();
// Check for loads in the ZExt node.
const TreeEntry *SrcTE = getOperandEntry(LhsTE, /*Idx=*/0);
if (SrcTE->State == TreeEntry::Vectorize &&
SrcTE->ReorderIndices.empty() && SrcTE->ReuseShuffleIndices.empty() &&
SrcTE->getOpcode() == Instruction::Load && !SrcTE->isAltShuffle() &&
all_of(SrcTE->Scalars, [](Value *V) { return V->hasOneUse(); })) {
auto *LI = cast<LoadInst>(SrcTE->getMainOp());
IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, SrcType, {SrcType});
InstructionCost BSwapCost =
TTI->getMemoryOpCost(Instruction::Load, SrcType, LI->getAlign(),
LI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI->getPointerOperand())) +
TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
if (BSwapCost <= BitcastCost) {
VecCost += TTI->getMemoryOpCost(
Instruction::Load, SrcVecTy, LI->getAlign(),
LI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI->getPointerOperand()));
BitcastCost = BSwapCost;
ForLoads = true;
}
}
}
} else if (Order.empty() && DL->getTypeSizeInBits(SrcScalarTy) == ByteSize) {
// Check for loads in the ZExt node.
const TreeEntry *SrcTE = getOperandEntry(LhsTE, /*Idx=*/0);
if (SrcTE->State == TreeEntry::Vectorize && SrcTE->ReorderIndices.empty() &&
SrcTE->ReuseShuffleIndices.empty() &&
SrcTE->getOpcode() == Instruction::Load && !SrcTE->isAltShuffle() &&
all_of(SrcTE->Scalars, [](Value *V) { return V->hasOneUse(); })) {
auto *LI = cast<LoadInst>(SrcTE->getMainOp());
BitcastCost =
TTI->getMemoryOpCost(Instruction::Load, SrcType, LI->getAlign(),
LI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI->getPointerOperand()));
VecCost +=
TTI->getMemoryOpCost(Instruction::Load, SrcVecTy, LI->getAlign(),
LI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI->getPointerOperand()));
ForLoads = true;
}
}
if (SrcType != ScalarTy) {
BitcastCost += TTI->getCastInstrCost(Instruction::ZExt, ScalarTy, SrcType,
CastCtx, CostKind);
}
return BitcastCost < VecCost;
}
std::optional<BitPackInfo>
BoUpSLP::computeBitPackLayout(const TreeEntry &TE) const {
assert(TE.hasState() &&
(TE.getOpcode() == Instruction::And ||
TE.getOpcode() == Instruction::Shl) &&
"Expected And or Shl node.");
const unsigned BitWidth = DL->getTypeSizeInBits(TE.getMainOp()->getType());
const TreeEntry *ShlTE = &TE;
const TreeEntry *MaskTE = nullptr;
if (TE.getOpcode() == Instruction::And) {
MaskTE = getOperandEntry(&TE, 1);
ShlTE = getOperandEntry(&TE, 0);
}
const TreeEntry *ShiftTE = getOperandEntry(ShlTE, 1);
const TreeEntry *LhsTE = getOperandEntry(ShlTE, 0);
const unsigned VF = TE.getVectorFactor();
SmallVector<APInt> PossibleBits(VF), Masks(VF);
SmallVector<uint64_t> ShlAmts(VF);
auto IsCopyable = [](const TreeEntry *TE, unsigned Idx) {
return TE->hasCopyableElements() && TE->isCopyableElement(TE->Scalars[Idx]);
};
for (unsigned Idx : seq(VF)) {
uint64_t ShlAmt = 0;
const APInt *C;
if (!IsCopyable(ShlTE, Idx)) {
if (!match(ShiftTE->Scalars[Idx], m_APInt(C)) || C->uge(BitWidth))
return std::nullopt;
ShlAmt = C->getZExtValue();
}
APInt Mask = APInt::getAllOnes(BitWidth);
if (MaskTE && !IsCopyable(&TE, Idx)) {
if (!match(MaskTE->Scalars[Idx], m_APInt(C)))
return std::nullopt;
Mask = *C;
}
Value *X = LhsTE->Scalars[Idx];
if (isa<PoisonValue>(X)) {
PossibleBits[Idx] = APInt(BitWidth, 0);
} else {
PossibleBits[Idx] =
APInt::getLowBitsSet(BitWidth, getScalarMaxValue(X).getActiveBits()) &
~computeKnownBits(X, SimplifyQuery(*DL)).Zero;
}
ShlAmts[Idx] = ShlAmt;
Masks[Idx] = Mask;
}
return computeBitPackInfo(BitWidth, PossibleBits, ShlAmts, Masks);
}
const BoUpSLP::TreeEntry *BoUpSLP::getBitPackSource(const TreeEntry &LhsTE,
BitPackInfo &Info) const {
// The chain nodes are checked both before and after being combined into the
// root.
auto IsChainNode = [](const TreeEntry &TE) {
return (TE.State == TreeEntry::Vectorize ||
TE.CombinedOp == TreeEntry::ReducedBitPack) &&
TE.ReorderIndices.empty() && TE.ReuseShuffleIndices.empty() &&
!TE.hasCopyableElements() &&
all_of(TE.Scalars, [](Value *V) { return V->hasOneUse(); });
};
const TreeEntry *AndTE = &LhsTE;
if (AndTE->hasState() && AndTE->getOpcode() == Instruction::ZExt &&
IsChainNode(*AndTE))
AndTE = getOperandEntry(AndTE, /*Idx=*/0);
if (!AndTE->hasState() || AndTE->getOpcode() != Instruction::And ||
!IsChainNode(*AndTE))
return &LhsTE;
const TreeEntry *ShiftTE = getOperandEntry(AndTE, /*Idx=*/0);
if (!ShiftTE->hasState() ||
(ShiftTE->getOpcode() != Instruction::LShr &&
ShiftTE->getOpcode() != Instruction::Shl) ||
!IsChainNode(*ShiftTE))
return &LhsTE;
const TreeEntry *SrcTE = getOperandEntry(ShiftTE, /*Idx=*/0);
if (SrcTE->State != TreeEntry::Vectorize || !SrcTE->ReorderIndices.empty() ||
!SrcTE->ReuseShuffleIndices.empty())
return &LhsTE;
const unsigned SrcBW = SrcTE->getMainOp()->getType()->getScalarSizeInBits();
const unsigned FieldWidth = Info.FieldWidth;
std::optional<uint64_t> UniformOff;
for (auto [Idx, FieldOff] : enumerate(Info.LShrAmts)) {
// The and-mask keeps the field bits, which are shifted from the same bits
// of every source lane.
Value *Sh;
const APInt *Mask, *Amt;
if (FieldOff + FieldWidth > SrcBW ||
!match(AndTE->Scalars[Idx], m_c_And(m_Value(Sh), m_APInt(Mask))) ||
!Mask->extractBits(FieldWidth, FieldOff).isAllOnes())
return &LhsTE;
uint64_t Off = FieldOff;
if (match(Sh, m_LShr(m_Specific(SrcTE->Scalars[Idx]), m_APInt(Amt))))
Off += Amt->getZExtValue();
else if (match(Sh, m_Shl(m_Specific(SrcTE->Scalars[Idx]), m_APInt(Amt))) &&
Amt->ule(FieldOff))
Off -= Amt->getZExtValue();
else
return &LhsTE;
if (Off + FieldWidth > SrcBW || (UniformOff && *UniformOff != Off))
return &LhsTE;
UniformOff = Off;
}
fill(Info.LShrAmts, *UniformOff);
return SrcTE;
}
unsigned BoUpSLP::getZExtSrcWidth(const TreeEntry &TE) const {
if (!TE.hasState() || TE.hasCopyableElements() ||
TE.getOpcode() != Instruction::ZExt)
return 0;
return cast<ZExtInst>(TE.getMainOp())->getSrcTy()->getScalarSizeInBits();
}
bool BoUpSLP::matchesBitPack(const TreeEntry &TE) const {
assert(TE.hasState() &&
(TE.getOpcode() == Instruction::And ||
TE.getOpcode() == Instruction::Shl) &&
"Expected And or Shl node.");
auto IsVectorizedOneUseNode = [&](const TreeEntry *N) {
return N->State == TreeEntry::Vectorize && !N->isAltShuffle() &&
N->ReorderIndices.empty() && N->ReuseShuffleIndices.empty() &&
!MinBWs.contains(N) && all_of(N->Scalars, [&](Value *V) {
return (N->hasCopyableElements() && N->isCopyableElement(V)) ||
V->hasOneUse();
});
};
if (!IsVectorizedOneUseNode(&TE))
return false;
Type *ScalarTy = TE.getMainOp()->getType();
if (!ScalarTy->isIntegerTy())
return false;
const unsigned BitWidth = DL->getTypeSizeInBits(ScalarTy);
if (!isPowerOf2_64(BitWidth))
return false;
const TreeEntry *ShlTE = &TE;
const TreeEntry *MaskTE = nullptr;
if (TE.getOpcode() == Instruction::And) {
MaskTE = getOperandEntry(&TE, 1);
ShlTE = getOperandEntry(&TE, 0);
if (!ShlTE->hasState() || ShlTE->getOpcode() != Instruction::Shl)
return false;
}
if (!IsVectorizedOneUseNode(ShlTE))
return false;
const TreeEntry *ShiftTE = getOperandEntry(ShlTE, 1);
const TreeEntry *LhsTE = getOperandEntry(ShlTE, 0);
if (!ShiftTE->isGather() || (MaskTE && !MaskTE->isGather()))
return false;
// The operand must be emitted as a plain vector in lane order: a vectorized
// node or a simple gather.
if ((!LhsTE->isGather() && LhsTE->State != TreeEntry::Vectorize) ||
(LhsTE->hasState() && LhsTE->isAltShuffle()) ||
!LhsTE->ReorderIndices.empty() || !LhsTE->ReuseShuffleIndices.empty() ||
MinBWs.contains(LhsTE))
return false;
std::optional<BitPackInfo> Info = computeBitPackLayout(TE);
if (!Info)
return false;
auto *VecTy =
cast<FixedVectorType>(getWidenedType(ScalarTy, TE.getVectorFactor()));
FastMathFlags FMF;
InstructionCost VecCost =
TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind) +
TTI->getArithmeticInstrCost(Instruction::Shl, VecTy, CostKind,
TTI::getOperandInfo(LhsTE->Scalars),
TTI::getOperandInfo(ShiftTE->Scalars),
/*Args=*/{}, ShlTE->getMainOp(), TLI);
if (TE.getOpcode() == Instruction::And)
VecCost += TTI->getArithmeticInstrCost(
Instruction::And, VecTy, CostKind, TTI::getOperandInfo(ShlTE->Scalars),
TTI::getOperandInfo(MaskTE->Scalars), /*Args=*/{}, TE.getMainOp(), TLI);
const TreeEntry *SrcTE = getBitPackSource(*LhsTE, *Info);
auto *SrcTy = cast<FixedVectorType>(
getWidenedType(SrcTE->Scalars.front()->getType(), TE.getVectorFactor()));
unsigned ShiftWidth;
InstructionCost PackCost = getBitPackCost(
*TTI, SrcTy, ScalarTy, *Info, getZExtSrcWidth(*SrcTE),
getCastContextHint(*SrcTE), CostKind, TLI, TE.getMainOp(), ShiftWidth);
return PackCost.isValid() && PackCost <= VecCost;
}
bool BoUpSLP::matchesInversedZExtSelect(
const TreeEntry &SelectTE,
SmallVectorImpl<unsigned> &InversedCmpsIndices) const {
assert(SelectTE.hasState() && SelectTE.getOpcode() == Instruction::Select &&
"Expected select node.");
SmallVector<std::pair<Instruction *, unsigned>> ZExts;
for (auto [Idx, V] : enumerate(SelectTE.Scalars)) {
auto *Inst = dyn_cast<Instruction>(V);
if (!Inst || Inst->getOpcode() != Instruction::ZExt)
continue;
ZExts.emplace_back(Inst, Idx);
}
if (ZExts.empty())
return false;
const auto *CmpTE = getOperandEntry(&SelectTE, 0);
const auto *Op1TE = getOperandEntry(&SelectTE, 1);
const auto *Op2TE = getOperandEntry(&SelectTE, 2);
// Compares must be alternate vectorized, and other operands must be gathers
// or copyables.
// TODO: investigate opportunity for reordered/reused nodes.
if (CmpTE->State != TreeEntry::Vectorize || !CmpTE->isAltShuffle() ||
(CmpTE->getOpcode() != Instruction::ICmp &&
CmpTE->getOpcode() != Instruction::FCmp) ||
!CmpTE->ReorderIndices.empty() || !CmpTE->ReuseShuffleIndices.empty() ||
!Op1TE->ReorderIndices.empty() || !Op1TE->ReuseShuffleIndices.empty() ||
!Op2TE->ReorderIndices.empty() || !Op2TE->ReuseShuffleIndices.empty())
return false;
// The operands must be buildvectors/copyables.
if (!Op1TE->isGather() || !Op2TE->isGather())
return false;
// TODO: investigate opportunity for the vector nodes with copyables.
auto *Cmp = CmpTE->getMainOp();
CmpPredicate Pred;
auto MatchCmp = m_Cmp(Pred, m_Value(), m_Value());
if (!match(Cmp, MatchCmp))
return false;
CmpPredicate MainPred = Pred;
CmpPredicate InversedPred(CmpInst::getInversePredicate(Pred),
Pred.hasSameSign());
for (const auto [Idx, V] :
make_filter_range(enumerate(CmpTE->Scalars), [&](const auto &P) {
return match(P.value(), MatchCmp);
})) {
if (CmpPredicate::getMatching(MainPred, Pred))
continue;
if (!CmpPredicate::getMatching(InversedPred, Pred))
return false;
if (!V->hasOneUse())
return false;
InversedCmpsIndices.push_back(Idx);
}
if (InversedCmpsIndices.empty())
return false;
Type *VecTy =
getWidenedType(Cmp->getOperand(0)->getType(), CmpTE->getVectorFactor());
Type *CmpTy = CmpInst::makeCmpResultType(VecTy);
InstructionCost VecCost = TTI->getCmpSelInstrCost(
CmpTE->getOpcode(), VecTy, CmpTy, MainPred, CostKind,
TTI::getOperandInfo(CmpTE->getOperand(0)),
TTI::getOperandInfo(CmpTE->getOperand(1)), CmpTE->getMainOp());
InstructionCost BVCost = getScalarizationOverhead(
*TTI, Opts.slp_revec, Cmp->getType(), cast<VectorType>(CmpTy),
APInt::getAllOnes(CmpTE->getVectorFactor()),
/*Insert=*/true, /*Extract=*/false, CostKind);
for (Instruction *I : make_isa_range<Instruction>(CmpTE->Scalars))
BVCost += TTI->getInstructionCost(I, CostKind);
return VecCost < BVCost;
}
bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
assert(SelectTE.hasState() && SelectTE.getOpcode() == Instruction::Select &&
"Expected select node.");
if (DL->isBigEndian())
return false;
if (!SelectTE.ReorderIndices.empty() || !SelectTE.ReuseShuffleIndices.empty())
return false;
if (!UserIgnoreList || SelectTE.Idx != 0)
return false;
if (any_of(SelectTE.Scalars, [](Value *V) { return !V->hasOneUse(); }))
return false;
// Check that all reduction operands are or instructions.
if (any_of(*UserIgnoreList,
[](Value *V) { return !match(V, m_Or(m_Value(), m_Value())); }))
return false;
const TreeEntry *Op1TE = getOperandEntry(&SelectTE, 1);
const TreeEntry *Op2TE = getOperandEntry(&SelectTE, 2);
if (!Op1TE->isGather() || !Op2TE->isGather())
return false;
// No need to check for zeroes reordering.
if (!Op1TE->ReorderIndices.empty() || !Op1TE->ReuseShuffleIndices.empty() ||
!Op2TE->ReuseShuffleIndices.empty())
return false;
Type *ScalarTy = Op1TE->Scalars.front()->getType();
if (!ScalarTy->isIntegerTy())
return false;
// Check that second operand is all zeroes.
if (any_of(Op2TE->Scalars, [](Value *V) { return !match(V, m_ZeroInt()); }))
return false;
// Check that first operand is 1,2,4,...
if (any_of(enumerate(Op1TE->Scalars), [](const auto &P) {
uint64_t V;
return !(match(P.value(), m_ConstantInt(V)) && isPowerOf2_64(V) &&
Log2_64(V) == P.index());
}))
return false;
// Check if bitcast is cheaper than select.
auto *DstTy = IntegerType::getIntNTy(ScalarTy->getContext(),
SelectTE.getVectorFactor());
Type *OpTy = getWidenedType(DstTy, SelectTE.getVectorFactor());
Type *CmpTy = CmpInst::makeCmpResultType(OpTy);
auto *VecTy =
cast<VectorType>(getWidenedType(ScalarTy, SelectTE.getVectorFactor()));
auto It = MinBWs.find(&SelectTE);
if (It != MinBWs.end()) {
auto *EffectiveScalarTy =
IntegerType::get(F->getContext(), It->second.first);
VecTy = cast<VectorType>(
getWidenedType(EffectiveScalarTy, SelectTE.getVectorFactor()));
}
InstructionCost BitcastCost = TTI->getCastInstrCost(
Instruction::BitCast, DstTy, CmpTy, TTI::CastContextHint::None, CostKind);
if (DstTy != ScalarTy) {
BitcastCost += TTI->getCastInstrCost(Instruction::ZExt, ScalarTy, DstTy,
TTI::CastContextHint::None, CostKind);
}
// Use the condition predicate shared by all lanes, if there is one.
CmpPredicate SelPred = CmpInst::BAD_ICMP_PREDICATE;
CmpPredicate P;
if (match(SelectTE.getMainOp(),
m_Select(m_ICmp(P, m_Value(), m_Value()), m_Value(), m_Value())) &&
all_of(SelectTE.Scalars, [&](Value *V) {
CmpPredicate LaneP;
return match(V, m_Select(m_ICmp(LaneP, m_Value(), m_Value()), m_Value(),
m_Value())) &&
LaneP == P.dropSameSign();
}))
SelPred = P;
FastMathFlags FMF;
InstructionCost SelectCost =
TTI->getCmpSelInstrCost(Instruction::Select, VecTy, CmpTy, SelPred,
CostKind, TTI::getOperandInfo(Op1TE->Scalars),
TTI::getOperandInfo(Op2TE->Scalars),
SelectTE.getMainOp()) +
TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind);
return BitcastCost <= SelectCost;
}
void BoUpSLP::transformNodes() {
BaseGraphSize = VectorizableTree.size();
// Turn graph transforming mode on and off, when done.
class GraphTransformModeRAAI {
bool &SavedIsGraphTransformMode;
public:
GraphTransformModeRAAI(bool &IsGraphTransformMode)
: SavedIsGraphTransformMode(IsGraphTransformMode) {
IsGraphTransformMode = true;
}
~GraphTransformModeRAAI() { SavedIsGraphTransformMode = false; }
} TransformContext(IsGraphTransformMode);
// Operands are profitable if they are:
// 1. At least one constant
// or
// 2. Splats
// or
// 3. Results in good vectorization opportunity, i.e. may generate vector
// nodes and reduce cost of the graph.
auto CheckOperandsProfitability = [this](Instruction *I1, Instruction *I2,
const InstructionsState &S) {
SmallVector<SmallVector<std::pair<Value *, Value *>>> Candidates;
for (unsigned Op : seq<unsigned>(S.getMainOp()->getNumOperands()))
Candidates.emplace_back().emplace_back(I1->getOperand(Op),
I2->getOperand(Op));
return all_of(Candidates, [this](
ArrayRef<std::pair<Value *, Value *>> Cand) {
return all_of(Cand,
[](const std::pair<Value *, Value *> &P) {
return isa<Constant>(P.first) ||
isa<Constant>(P.second) || P.first == P.second;
}) ||
findBestRootPair(Cand, LookAheadHeuristics::ScoreSplatLoads).first;
});
};
// The reduction root gather feeds only the commutative reduction, which
// maps the reduced values to the lanes by the root scalars. Put the distinct
// sub-fields of the same wider scalar in the natural order to emit them
// without a permutation.
if (UserIgnoreList && getRootNode().isGather() &&
getRootNode().ReuseShuffleIndices.empty())
if (std::optional<std::tuple<Value *, unsigned, SmallVector<int>>> Fields =
matchGatheredExtractedFields(getRootNode().Scalars, *DL)) {
auto &[Src, FieldWidth, Mask] = *Fields;
SmallVector<int> SortedMask(Mask);
sort(SortedMask);
if (equal(SortedMask,
seq<int>(Src->getType()->getIntegerBitWidth() / FieldWidth)))
reorderScalars(getRootNode().Scalars, Mask);
}
// Try to reorder gather nodes for better vectorization opportunities.
for (unsigned Idx : seq<unsigned>(BaseGraphSize)) {
TreeEntry &E = *VectorizableTree[Idx];
if (E.isGather())
reorderGatherNode(E);
}
// Better to use full gathered loads analysis, if there are only 2 loads
// gathered nodes each having less than 16 elements.
constexpr unsigned VFLimit = 16;
bool ForceLoadGather =
count_if(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && TE->hasState() &&
TE->getOpcode() == Instruction::Load &&
TE->getVectorFactor() < VFLimit;
}) == 2;
// Checks if the scalars are used in other node.
auto AreReusedScalars = [&](const TreeEntry *TE, ArrayRef<Value *> VL,
function_ref<bool(Value *)> CheckContainer) {
return TE->isSame(VL) || all_of(VL, [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
auto *I = dyn_cast<Instruction>(V);
if (!I)
return false;
return is_contained(TE->Scalars, I) || CheckContainer(I);
});
};
auto CheckForSameVectorNodes = [&](const TreeEntry &E) {
if (E.hasState()) {
if (ArrayRef<TreeEntry *> TEs = getTreeEntries(E.getMainOp());
!TEs.empty() && any_of(TEs, [&](const TreeEntry *TE) {
return AreReusedScalars(TE, E.Scalars, [&](Value *V) {
ArrayRef<TreeEntry *> VTEs = getTreeEntries(V);
return !VTEs.empty() && any_of(VTEs, [&](const TreeEntry *TE) {
return is_contained(TEs, TE);
});
});
}))
return true;
;
if (ArrayRef<TreeEntry *> TEs = getSplitTreeEntries(E.getMainOp());
!TEs.empty() && any_of(TEs, [&](const TreeEntry *TE) {
return AreReusedScalars(TE, E.Scalars, [&](Value *V) {
ArrayRef<TreeEntry *> VTEs = getSplitTreeEntries(V);
return !VTEs.empty() && any_of(VTEs, [&](const TreeEntry *TE) {
return is_contained(TEs, TE);
});
});
}))
return true;
} else {
// Check if the gather node full copy of split node.
auto *It = find_if(E.Scalars, IsaPred<Instruction>);
if (It != E.Scalars.end()) {
if (ArrayRef<TreeEntry *> TEs = getSplitTreeEntries(*It);
!TEs.empty() && any_of(TEs, [&](const TreeEntry *TE) {
return AreReusedScalars(TE, E.Scalars, [&](Value *V) {
ArrayRef<TreeEntry *> VTEs = getSplitTreeEntries(V);
return !VTEs.empty() && any_of(VTEs, [&](const TreeEntry *TE) {
return is_contained(TEs, TE);
});
});
}))
return true;
}
}
return false;
};
// Marks the fadd/fsub node \p E as fmuladd, if its fmul operand is fused
// into it.
auto CombineFMulAdd = [&](TreeEntry &E) {
// Check if possible to convert (a*b)+c to fma; fma takes exactly two
// operands, so bail out on reassociated (>2-operand) nodes.
if (E.State != TreeEntry::Vectorize ||
!E.getOperations().isAddSubLikeOp() ||
E.getOperations().isAltShuffle() || E.getNumOperands() != 2)
return;
const TreeEntry *LHS = getOperandEntry(&E, 0);
const TreeEntry *RHS = getOperandEntry(&E, 1);
auto IsOneUseVectorFMulOperand = [](const TreeEntry *TE) {
return TE->State == TreeEntry::Vectorize && TE->ReorderIndices.empty() &&
TE->ReuseShuffleIndices.empty() &&
TE->getOpcode() == Instruction::FMul && !TE->isAltShuffle() &&
all_of(TE->Scalars, [&](Value *V) {
return (TE->hasCopyableElements() && TE->isCopyableElement(V)) ||
V->hasOneUse();
});
};
// Both a*b - c and c - a*b are fused.
const bool IsLHSFMul = IsOneUseVectorFMulOperand(LHS);
if (!IsLHSFMul && !IsOneUseVectorFMulOperand(RHS))
return;
if (!canConvertToFMA(E.Scalars, E.getOperations(), *DT, *DL, *TTI, *TLI,
*this)
.isValid())
return;
// This node is a fmuladd node.
E.CombinedOp = TreeEntry::FMulAdd;
TreeEntry *FMulEntry = getOperandEntry(&E, IsLHSFMul ? 0 : 1);
if (FMulEntry->UserTreeIndex && FMulEntry->State == TreeEntry::Vectorize) {
// The FMul node is part of the combined fmuladd node.
FMulEntry->State = TreeEntry::CombinedVectorize;
}
};
// The tree may grow here, so iterate over nodes, built before.
for (unsigned Idx : seq<unsigned>(BaseGraphSize)) {
TreeEntry &E = *VectorizableTree[Idx];
if (E.isGather()) {
ArrayRef<Value *> VL = E.Scalars;
const unsigned Sz = getVectorElementSize(VL.front());
unsigned MinVF = getMinVF(2 * Sz);
// Do not try partial vectorization for small nodes (<= 2), nodes with the
// same opcode and same parent block or all constants.
if (VL.size() <= 2 || LoadEntriesToVectorize.contains(Idx) ||
!(!E.hasState() || E.getOpcode() == Instruction::Load ||
// We use allSameOpcode instead of isAltShuffle because we don't
// want to use interchangeable instruction here.
!allSameOpcode(VL) || !allSameBlock(VL)) ||
allConstant(VL) || isSplat(VL) ||
(E.ReuseShuffleIndices.empty() &&
matchGatheredExtractedFields(VL, *DL)))
continue;
if (ForceLoadGather && E.hasState() && E.getOpcode() == Instruction::Load)
continue;
// Check if the node is a copy of other vector nodes.
if (CheckForSameVectorNodes(E))
continue;
// Try to find vectorizable sequences and transform them into a series of
// insertvector instructions.
unsigned StartIdx = 0;
unsigned End = VL.size();
SmallBitVector Processed(End);
for (unsigned VF = getFloorFullVectorNumberOfElements(
*TTI, VL.front()->getType(), VL.size() - 1, Opts.slp_revec);
VF >= MinVF;
VF = getFloorFullVectorNumberOfElements(*TTI, VL.front()->getType(),
VF - 1, Opts.slp_revec)) {
if (StartIdx + VF > End)
continue;
SmallVector<std::pair<unsigned, unsigned>> Slices;
bool AllStrided = true;
// Walk the range in steps of VF, but allow the trailing slice to be
// shorter (SliceVF < VF) so non-power-of-2 tails can be vectorized.
// Processed only records slice starts (Cnt), and downstream consumers
// (test(Cnt), AddCombinedNode's range set/StartIdx update) operate on
// start positions, so partial coverage is consistent.
for (unsigned Cnt = StartIdx; Cnt < End; Cnt += VF) {
const unsigned SliceVF = std::min(VF, End - Cnt);
if (SliceVF <= 1)
continue;
ArrayRef<Value *> Slice = VL.slice(Cnt, SliceVF);
// If any instruction is vectorized already - do not try again.
// Reuse the existing node, if it fully matches the slice.
if ((Processed.test(Cnt) || isVectorized(Slice.front())) &&
!getSameValuesTreeEntry(Slice.front(), Slice, /*SameVF=*/true))
continue;
// Constant already handled effectively - skip.
if (allConstant(Slice))
continue;
// Do not try to vectorize small splats (less than vector register and
// only with the single non-undef element).
bool IsSplat = isSplat(Slice);
bool IsTwoRegisterSplat = true;
if (IsSplat && VF == 2) {
unsigned NumRegs2VF = getNumberOfParts(
getWidenedType(getValueType(Slice.front(), Opts.slp_revec),
2 * VF),
getValueType(Slice.front(), Opts.slp_revec));
IsTwoRegisterSplat = NumRegs2VF == 2;
}
if (Slices.empty() || !IsSplat || !IsTwoRegisterSplat ||
count(Slice, Slice.front()) ==
static_cast<long>(isa<UndefValue>(Slice.front()) ? SliceVF - 1
: 1)) {
if (IsSplat)
continue;
InstructionsState S = getSameOpcode(Slice, *TLI);
if (!S || !allSameOpcode(Slice) || !allSameBlock(Slice) ||
(S.getOpcode() == Instruction::Load &&
areKnownNonVectorizableLoads(Slice)))
continue;
if (VF == 2) {
// Cache the cost check lazily - both branches below may need it.
std::optional<bool> MainOpIsCheap;
auto IsMainOpCheap = [&, TTI = TTI] {
if (!MainOpIsCheap)
MainOpIsCheap =
TTI->getInstructionCost(S.getMainOp(), CostKind) <
TTI::TCC_Expensive;
return *MainOpIsCheap;
};
// Try to vectorize reduced values or if all users are vectorized.
// For expensive instructions extra extracts might be profitable.
if ((!UserIgnoreList || E.Idx != 0) && IsMainOpCheap() &&
!all_of(make_isa_range<Instruction>(Slice),
[&](Instruction *I) {
return areAllUsersVectorized(I, UserIgnoreList);
}))
continue;
if (S.getOpcode() == Instruction::Load) {
OrdersType Order;
SmallVector<Value *> PointerOps;
StridedPtrInfo SPtrInfo;
LoadsState Res = canVectorizeLoads(Slice, Slice.front(), Order,
PointerOps, SPtrInfo);
AllStrided &= Res == LoadsState::StridedVectorize ||
Res == LoadsState::ScatterVectorize ||
Res == LoadsState::Gather;
// Do not vectorize gathers.
if (Res == LoadsState::ScatterVectorize ||
Res == LoadsState::Gather) {
if (Res == LoadsState::Gather) {
registerNonVectorizableLoads(Slice);
// If reductions and the scalars from the root node are
// analyzed - mark as non-vectorizable reduction.
if (UserIgnoreList && E.Idx == 0)
analyzedReductionVals(Slice);
}
continue;
}
} else if (S.getOpcode() == Instruction::ExtractElement ||
(IsMainOpCheap() &&
!CheckOperandsProfitability(
S.getMainOp(),
cast<Instruction>(*find_if(reverse(Slice),
IsaPred<Instruction>)),
S))) {
// Do not vectorize extractelements (handled effectively
// alread). Do not vectorize non-profitable instructions (with
// low cost and non-vectorizable operands.)
continue;
}
}
}
Slices.emplace_back(Cnt, Slice.size());
}
// Do not try to vectorize if all slides are strided or gathered with
// vector factor 2 and there are more than 2 slices. Better to handle
// them in gathered loads analysis, may result in better vectorization.
if (VF == 2 && AllStrided && Slices.size() > 2)
continue;
auto AddCombinedNode = [&](unsigned Idx, unsigned Cnt, unsigned Sz) {
E.CombinedEntriesWithIndices.emplace_back(Idx, Cnt);
Processed.set(Cnt, Cnt + Sz);
if (StartIdx == Cnt)
StartIdx = Cnt + Sz;
if (End == Cnt + Sz)
End = Cnt;
};
for (auto [Cnt, Sz] : Slices) {
ArrayRef<Value *> Slice = VL.slice(Cnt, Sz);
const TreeEntry *SameTE = nullptr;
if (const auto *It = find_if(Slice, IsaPred<Instruction>);
It != Slice.end()) {
// If any instruction is vectorized already - do not try again.
SameTE = getSameValuesTreeEntry(*It, Slice);
}
unsigned PrevSize = VectorizableTree.size();
[[maybe_unused]] unsigned PrevEntriesSize =
LoadEntriesToVectorize.size();
buildTreeRec(Slice, 0, EdgeInfo(&E, UINT_MAX));
if (PrevSize + 1 == VectorizableTree.size() && !SameTE &&
VectorizableTree[PrevSize]->isGather() &&
VectorizableTree[PrevSize]->hasState() &&
VectorizableTree[PrevSize]->getOpcode() !=
Instruction::ExtractElement &&
!isSplat(Slice)) {
if (UserIgnoreList && E.Idx == 0 && VF == 2)
analyzedReductionVals(Slice);
VectorizableTree.pop_back();
assert(PrevEntriesSize == LoadEntriesToVectorize.size() &&
"LoadEntriesToVectorize expected to remain the same");
continue;
}
AddCombinedNode(PrevSize, Cnt, Sz);
}
}
// Restore ordering, if no extra vectorization happened.
if (E.CombinedEntriesWithIndices.empty() && !E.ReorderIndices.empty()) {
SmallVector<int> Mask(E.ReorderIndices.begin(), E.ReorderIndices.end());
reorderScalars(E.Scalars, Mask);
E.ReorderIndices.clear();
}
}
if (!E.hasState())
continue;
switch (E.getOpcode()) {
case Instruction::Load: {
// No need to reorder masked gather loads, just reorder the scalar
// operands.
// Lanes without a load would break the common alignment below.
if (E.State != TreeEntry::Vectorize ||
any_of(E.Scalars, IsaPred<UndefValue>))
break;
Type *ScalarTy = E.getMainOp()->getType();
auto *VecTy =
cast<FixedVectorType>(getWidenedType(ScalarTy, E.Scalars.size()));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E.Scalars);
// Check if profitable to represent consecutive load + reverse as strided
// load with stride -1.
if (E.isReverse() &&
TTI->isLegalStridedLoadStore(VecTy, CommonAlignment)) {
SmallVector<int> Mask;
inversePermutation(E.ReorderIndices, Mask);
auto *BaseLI = cast<LoadInst>(E.Scalars.back());
InstructionCost OriginalVecCost =
TTI->getMemoryOpCost(
Instruction::Load, VecTy, BaseLI->getAlign(),
BaseLI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(BaseLI->getPointerOperand())) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
Type *StrideTy = DL->getIndexType(BaseLI->getPointerOperandType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::experimental_vp_strided_load, VecTy,
BaseLI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, BaseLI,
ConstantInt::getSigned(
StrideTy,
-static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost || Opts.slp_force_strided_loads) {
// Strided load is more profitable than consecutive load + reverse -
// transform the node to strided load.
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::get(StrideTy, 1);
SPtrInfo.Ty = VecTy;
TreeEntryToStridedPtrInfoMap[&E] = SPtrInfo;
E.State = TreeEntry::StridedVectorize;
}
}
break;
}
case Instruction::Store: {
if (E.State == TreeEntry::ExpandVectorize)
break;
Type *ScalarTy =
cast<StoreInst>(E.getMainOp())->getValueOperand()->getType();
auto *VecTy =
cast<FixedVectorType>(getWidenedType(ScalarTy, E.Scalars.size()));
Align CommonAlignment = computeCommonAlignment<StoreInst>(E.Scalars);
// Check if profitable to represent consecutive load + reverse as strided
// load with stride -1.
if (E.isReverse() &&
TTI->isLegalStridedLoadStore(VecTy, CommonAlignment)) {
SmallVector<int> Mask;
inversePermutation(E.ReorderIndices, Mask);
auto *BaseSI = cast<StoreInst>(E.Scalars.back());
InstructionCost OriginalVecCost =
TTI->getMemoryOpCost(Instruction::Store, VecTy, BaseSI->getAlign(),
BaseSI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(E.getOperand(0))) +
getShuffleCost(*TTI, TTI::SK_Reverse, VecTy, CostKind, Mask);
Type *StrideTy = DL->getIndexType(BaseSI->getPointerOperandType());
InstructionCost StridedCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(
Intrinsic::experimental_vp_strided_store, VecTy,
BaseSI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment, BaseSI,
ConstantInt::getSigned(
StrideTy,
-static_cast<int64_t>(DL->getTypeAllocSize(ScalarTy)))),
CostKind);
if (StridedCost < OriginalVecCost) {
// Strided store is more profitable than reverse + consecutive store -
// transform the node to strided store.
E.State = TreeEntry::StridedVectorize;
StridedPtrInfo SPtrInfo;
SPtrInfo.StrideVal = ConstantInt::getSigned(StrideTy, -1);
SPtrInfo.Ty = VecTy;
TreeEntryToStridedPtrInfoMap[&E] = SPtrInfo;
}
} else if (!E.ReorderIndices.empty()) {
// Check for interleaved stores.
auto IsInterleaveMask = [&, &TTI = *TTI](ArrayRef<int> Mask) {
auto *BaseSI = cast<StoreInst>(E.Scalars.front());
assert(Mask.size() > 1 && "Expected mask greater than 1 element.");
if (Mask.size() < 4)
return 0u;
for (unsigned Factor : seq<unsigned>(2, Mask.size() / 2 + 1)) {
if (ShuffleVectorInst::isInterleaveMask(
Mask, Factor, VecTy->getElementCount().getFixedValue()) &&
TTI.isLegalInterleavedAccessType(
VecTy, Factor, BaseSI->getAlign(),
BaseSI->getPointerAddressSpace()))
return Factor;
}
return 0u;
};
SmallVector<int> Mask(E.ReorderIndices.begin(), E.ReorderIndices.end());
unsigned InterleaveFactor = IsInterleaveMask(Mask);
if (InterleaveFactor != 0)
E.setInterleave(InterleaveFactor);
}
break;
}
case Instruction::Select: {
if (E.State != TreeEntry::Vectorize)
break;
auto [MinMaxID, SelectOnly] = canConvertToMinOrMaxIntrinsic(E.Scalars);
if (MinMaxID != Intrinsic::not_intrinsic) {
// This node is a minmax node.
E.CombinedOp = TreeEntry::MinMax;
TreeEntry *CondEntry = getOperandEntry(&E, 0);
if (SelectOnly && CondEntry->UserTreeIndex &&
CondEntry->State == TreeEntry::Vectorize) {
// The condition node is part of the combined minmax node.
CondEntry->State = TreeEntry::CombinedVectorize;
}
break;
}
// Check for zext + selects, which can be reordered.
SmallVector<unsigned> InversedCmpsIndices;
if (matchesInversedZExtSelect(E, InversedCmpsIndices)) {
auto *CmpTE = getOperandEntry(&E, 0);
auto *Op1TE = getOperandEntry(&E, 1);
auto *Op2TE = getOperandEntry(&E, 2);
// State now is uniform, not alternate opcode.
CmpTE->setOperations(
InstructionsState(CmpTE->getMainOp(), CmpTE->getMainOp()));
// Update mapping between the swapped values and their internal matching
// nodes.
auto UpdateGatherEntry = [&](TreeEntry *OldTE, TreeEntry *NewTE,
Value *V) {
if (isConstant(V))
return;
auto It = ValueToGatherNodes.find(V);
assert(It != ValueToGatherNodes.end() &&
"Expected to find the value in the map.");
auto &C = It->getSecond();
if (!is_contained(OldTE->Scalars, V))
C.remove(OldTE);
C.insert(NewTE);
};
ValueList &Op1 = E.getOperand(1);
ValueList &Op2 = E.getOperand(2);
for (const unsigned Idx : InversedCmpsIndices) {
Value *V1 = Op1TE->Scalars[Idx];
Value *V2 = Op2TE->Scalars[Idx];
std::swap(Op1TE->Scalars[Idx], Op2TE->Scalars[Idx]);
std::swap(Op1[Idx], Op2[Idx]);
UpdateGatherEntry(Op1TE, Op2TE, V1);
UpdateGatherEntry(Op2TE, Op1TE, V2);
}
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 1), Op1TE);
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 2), Op2TE);
// NB: Fallback to check if select can be converted to cmp bitcast.
}
if (matchesSelectOfBits(E)) {
// This node is a (reduced or) cmp bitcast node.
const TreeEntry::CombinedOpcode Code = TreeEntry::ReducedCmpBitcast;
E.CombinedOp = Code;
auto *Op1TE = getOperandEntry(&E, 1);
auto *Op2TE = getOperandEntry(&E, 2);
Op1TE->State = TreeEntry::CombinedVectorize;
Op1TE->CombinedOp = Code;
Op2TE->State = TreeEntry::CombinedVectorize;
Op2TE->CombinedOp = Code;
break;
}
break;
}
case Instruction::FSub:
case Instruction::FAdd:
CombineFMulAdd(E);
break;
case Instruction::Shl:
case Instruction::And: {
// Shl/And are not reassociated; guard since this case indexes operands
// 0/1.
if (E.getNumOperands() != 2)
break;
if (E.Idx != 0 || DL->isBigEndian())
break;
if (!UserIgnoreList)
break;
// Check that all reduction operands are disjoint or instructions.
if (E.getOpcode() == Instruction::Shl &&
all_of(*UserIgnoreList, [](Value *V) {
return match(V, m_DisjointOr(m_Value(), m_Value()));
})) {
OrdersType Order;
bool IsBSwap;
bool ForLoads;
if (matchesShlZExt(E, Order, IsBSwap, ForLoads)) {
// This node is a (reduced disjoint or) bitcast node.
TreeEntry::CombinedOpcode Code =
IsBSwap ? (ForLoads ? TreeEntry::ReducedBitcastBSwapLoads
: TreeEntry::ReducedBitcastBSwap)
: (ForLoads ? TreeEntry::ReducedBitcastLoads
: TreeEntry::ReducedBitcast);
E.CombinedOp = Code;
E.ReorderIndices = std::move(Order);
TreeEntry *ZExtEntry = getOperandEntry(&E, 0);
assert(ZExtEntry->UserTreeIndex &&
ZExtEntry->State == TreeEntry::Vectorize &&
ZExtEntry->getOpcode() == Instruction::ZExt &&
"Expected ZExt node.");
// The ZExt node is part of the combined node.
ZExtEntry->State = TreeEntry::CombinedVectorize;
ZExtEntry->CombinedOp = Code;
if (ForLoads) {
TreeEntry *LoadsEntry = getOperandEntry(ZExtEntry, 0);
assert(LoadsEntry->UserTreeIndex &&
LoadsEntry->State == TreeEntry::Vectorize &&
LoadsEntry->getOpcode() == Instruction::Load &&
"Expected Load node.");
// The Load node is part of the combined node.
LoadsEntry->State = TreeEntry::CombinedVectorize;
LoadsEntry->CombinedOp = Code;
}
TreeEntry *ConstEntry = getOperandEntry(&E, 1);
assert(ConstEntry->UserTreeIndex && ConstEntry->isGather() &&
"Expected constants only.");
// The ConstNode node is part of the combined node.
ConstEntry->State = TreeEntry::CombinedVectorize;
ConstEntry->CombinedOp = Code;
break;
}
}
// Check that all reduction operands are or instructions.
if (any_of(*UserIgnoreList, [](Value *V) {
return !match(V, m_Or(m_Value(), m_Value()));
}))
break;
if (!matchesBitPack(E))
break;
// This node is a (reduced or) bit pack node.
E.CombinedOp = TreeEntry::ReducedBitPack;
TreeEntry *ShlEntry = &E;
if (E.getOpcode() == Instruction::And) {
ShlEntry = getOperandEntry(&E, 0);
assert(ShlEntry->UserTreeIndex &&
ShlEntry->State == TreeEntry::Vectorize &&
ShlEntry->getOpcode() == Instruction::Shl &&
"Expected Shl node.");
// The Shl node is part of the combined node.
ShlEntry->State = TreeEntry::CombinedVectorize;
ShlEntry->CombinedOp = TreeEntry::ReducedBitPack;
TreeEntry *MaskEntry = getOperandEntry(&E, 1);
assert(MaskEntry->UserTreeIndex && MaskEntry->isGather() &&
"Expected constants only.");
MaskEntry->State = TreeEntry::CombinedVectorize;
MaskEntry->CombinedOp = TreeEntry::ReducedBitPack;
}
TreeEntry *ShiftEntry = getOperandEntry(ShlEntry, 1);
assert(ShiftEntry->UserTreeIndex && ShiftEntry->isGather() &&
"Expected constants only.");
ShiftEntry->State = TreeEntry::CombinedVectorize;
ShiftEntry->CombinedOp = TreeEntry::ReducedBitPack;
// The chain of a uniform field extract and its constant operands are
// part of the combined node.
std::optional<BitPackInfo> Info = computeBitPackLayout(E);
TreeEntry *LhsTE = getOperandEntry(ShlEntry, /*Idx=*/0);
const TreeEntry *SrcTE = getBitPackSource(*LhsTE, *Info);
for (TreeEntry *ChainTE = LhsTE; ChainTE != SrcTE;
ChainTE = getOperandEntry(ChainTE, /*Idx=*/0)) {
ChainTE->State = TreeEntry::CombinedVectorize;
ChainTE->CombinedOp = TreeEntry::ReducedBitPack;
for (unsigned Idx : seq<unsigned>(1, ChainTE->getNumOperands())) {
TreeEntry *ConstTE = getOperandEntry(ChainTE, Idx);
ConstTE->State = TreeEntry::CombinedVectorize;
ConstTE->CombinedOp = TreeEntry::ReducedBitPack;
}
}
break;
}
default:
break;
}
}
// The nodes, built for the slices of the gathered nodes, are fused with their
// fmul operands in the vector code too.
for (std::unique_ptr<TreeEntry> &TE :
drop_begin(VectorizableTree, BaseGraphSize))
if (TE->hasState() &&
is_contained({Instruction::FAdd, Instruction::FSub}, TE->getOpcode()))
CombineFMulAdd(*TE);
if (LoadEntriesToVectorize.empty()) {
// Single load node - exit.
if (VectorizableTree.size() <= 1 && getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::Load)
return;
// Small graph with small VF - exit.
constexpr unsigned SmallTree = 3;
constexpr unsigned SmallVF = 2;
if ((VectorizableTree.size() <= SmallTree &&
getRootNodeScalars().size() == SmallVF) ||
(VectorizableTree.size() <= 2 && UserIgnoreList))
return;
if (getRootNode().isNonPowOf2Vec() &&
getCanonicalGraphSize() != getTreeSize() && UserIgnoreList &&
getCanonicalGraphSize() <= SmallTree &&
count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
[](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && TE->hasState() &&
TE->getOpcode() == Instruction::Load &&
!allSameBlock(TE->Scalars);
}) == 1)
return;
}
// A list of loads to be gathered during the vectorization process. We can
// try to vectorize them at the end, if profitable.
SmallMapVector<std::tuple<BasicBlock *, Value *, Type *>,
SmallVector<SmallVector<std::pair<LoadInst *, int64_t>>>, 8>
GatheredLoads;
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
TreeEntry &E = *TE;
if (E.isGather() &&
((E.hasState() && E.getOpcode() == Instruction::Load) ||
(!E.hasState() && any_of(make_isa_range<LoadInst>(E.Scalars),
[&](LoadInst *LI) {
return !isVectorized(LI) && !isDeleted(LI);
}))) &&
!isSplat(E.Scalars)) {
for (Value *V : E.Scalars) {
auto *LI = dyn_cast<LoadInst>(V);
if (!LI)
continue;
if (isDeleted(LI) || isVectorized(LI) || !LI->isSimple())
continue;
gatherPossiblyVectorizableLoads(
*this, V, *DL, *SE, *TTI,
GatheredLoads[std::make_tuple(
LI->getParent(),
getUnderlyingObject(LI->getPointerOperand(),
Opts.slp_recursion_max_depth),
LI->getType())]);
}
}
}
// Try to vectorize gathered loads if this is not just a gather of loads.
if (!GatheredLoads.empty())
tryToVectorizeGatheredLoads(GatheredLoads);
}
/// Merges shuffle masks and emits final shuffle instruction, if required. It
/// supports shuffling of 2 input vectors. It implements lazy shuffles emission,
/// when the actual shuffle instruction is generated only if this is actually
/// required. Otherwise, the shuffle instruction emission is delayed till the
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
bool IsFinalized = false;
SmallVector<int> CommonMask;
// Each input vector can be tagged with extra context information
struct InVector {
PointerUnion<Value *, const TreeEntry *> Vec = nullptr;
TargetTransformInfo::VectorInstrContext Ctx =
TargetTransformInfo::VectorInstrContext::None;
InVector(Value *Vec, TargetTransformInfo::VectorInstrContext Ctx =
TargetTransformInfo::VectorInstrContext::None)
: Vec(Vec), Ctx(Ctx) {}
InVector(const TreeEntry *Vec,
TargetTransformInfo::VectorInstrContext Ctx =
TargetTransformInfo::VectorInstrContext::None)
: Vec(Vec), Ctx(Ctx) {}
};
SmallVector<InVector, 2> InVectors;
/// Captures the original scalar VL of a single, "clean" gather() call so
/// the values can be forwarded as the Args operand to getShuffleCost() for
/// the final permutation in finalize(). This lets the target cost model
/// recognize patterns such as broadcast-of-load (e.g. on X86,
/// vbroadcast{ss,sd} folds the broadcast and the load into one instruction
/// under AVX/AVX2 and is reported as TCC_Free by getShuffleCost). The
/// state machine is:
/// * engaged + empty: tracking active, no qualifying gather seen yet.
/// * engaged + non-empty: exactly one qualifying gather observed and its
/// VL still corresponds to InVectors.front().
/// * disengaged: the cached VL is no longer trustworthy (multiple
/// gather() calls, or a state-mutating add() happened).
std::optional<SmallVector<Value *>> BVValues = SmallVector<Value *>();
const TargetTransformInfo &TTI;
InstructionCost Cost = 0;
SmallDenseSet<Value *> VectorizedVals;
BoUpSLP &R;
SmallPtrSetImpl<Value *> &CheckedExtracts;
const TTI::TargetCostKind CostKind;
/// While set, still trying to estimate the cost for the same nodes and we
/// can delay actual cost estimation (virtual shuffle instruction emission).
/// May help better estimate the cost if same nodes must be permuted + allows
/// to move most of the long shuffles cost estimation to TTI.
bool SameNodesEstimated = true;
static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) {
if (Ty->getScalarType()->isPointerTy()) {
Constant *Res = ConstantExpr::getIntToPtr(
ConstantInt::getAllOnesValue(
IntegerType::get(Ty->getContext(),
DL.getTypeStoreSizeInBits(Ty->getScalarType()))),
Ty->getScalarType());
if (auto *VTy = dyn_cast<VectorType>(Ty))
Res = ConstantVector::getSplat(VTy->getElementCount(), Res);
return Res;
}
return Constant::getAllOnesValue(Ty);
}
InstructionCost getBuildVectorCost(ArrayRef<Value *> VL, Value *Root,
TTI::VectorInstrContext ContextHint) {
if ((!Root && allConstant(VL)) || all_of(VL, IsaPred<UndefValue>))
return TTI::TCC_Free;
auto *VecTy = cast<VectorType>(getWidenedType(ScalarTy, VL.size()));
InstructionCost GatherCost = 0;
if (!Root && isSplat(VL)) {
// Found the broadcasting of the single scalar, calculate the cost as
// the broadcast.
const auto *It = find_if_not(VL, IsaPred<UndefValue>);
assert(It != VL.end() && "Expected at least one non-undef value.");
// Add broadcast for non-identity shuffle only.
bool NeedShuffle =
count(VL, *It) > 1 &&
(VL.front() != *It || !all_of(VL.drop_front(), IsaPred<UndefValue>));
if (!NeedShuffle) {
if (isa<FixedVectorType>(ScalarTy)) {
assert(R.Opts.slp_revec && "FixedVectorType is not expected.");
return TTI.getShuffleCost(
TTI::SK_InsertSubvector, VecTy, VecTy, CostKind, {},
std::distance(VL.begin(), It) * getNumElements(ScalarTy),
cast<FixedVectorType>(ScalarTy));
}
return TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
CostKind, std::distance(VL.begin(), It),
PoisonValue::get(VecTy), *It,
ContextHint);
}
SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
transform(VL, ShuffleMask.begin(), [](Value *V) {
return isa<PoisonValue>(V) ? PoisonMaskElem : 0;
});
InstructionCost InsertCost =
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind, 0,
PoisonValue::get(VecTy), *It, ContextHint);
return InsertCost + getShuffleCost(TTI, TargetTransformInfo::SK_Broadcast,
VecTy, CostKind, ShuffleMask,
/*Index=*/0, /*SubTp=*/nullptr,
/*Args=*/*It, ContextHint);
}
assert(ContextHint != TTI::VectorInstrContext::SplatOpFolded &&
"Expected SplatOpFolded to be a splat");
return GatherCost + (all_of(VL, IsaPred<UndefValue>)
? TTI::TCC_Free
: R.getGatherCost(VL, !Root, ScalarTy));
};
/// Compute the cost of creating a vector containing the extracted values from
/// \p VL.
InstructionCost
computeExtractCost(ArrayRef<Value *> VL, ArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
unsigned NumParts) {
assert(VL.size() > NumParts && "Unexpected scalarized shuffle.");
unsigned NumElts = accumulate(VL, 0, [](unsigned Sz, Value *V) {
auto *EE = dyn_cast<ExtractElementInst>(V);
if (!EE)
return Sz;
auto *VecTy = dyn_cast<FixedVectorType>(EE->getVectorOperandType());
if (!VecTy)
return Sz;
return std::max(Sz, VecTy->getNumElements());
});
// FIXME: this must be moved to TTI for better estimation.
unsigned EltsPerVector = getPartNumElems(VL.size(), NumParts);
auto CheckPerRegistersShuffle = [&](MutableArrayRef<int> Mask,
SmallVectorImpl<unsigned> &Indices,
SmallVectorImpl<unsigned> &SubVecSizes)
-> std::optional<TTI::ShuffleKind> {
if (NumElts <= EltsPerVector)
return std::nullopt;
int OffsetReg0 = alignDown(accumulate(Mask, INT_MAX,
[](int S, int I) {
if (I == PoisonMaskElem)
return S;
return std::min(S, I);
}),
EltsPerVector);
int OffsetReg1 = OffsetReg0;
DenseSet<int> RegIndices;
// Check that if trying to permute same single/2 input vectors.
TTI::ShuffleKind ShuffleKind = TTI::SK_PermuteSingleSrc;
int FirstRegId = -1;
Indices.assign(1, OffsetReg0);
auto HasLane = [](const auto &P) { return P.value() != PoisonMaskElem; };
for (auto [Pos, I] : make_filter_range(enumerate(Mask), HasLane)) {
int Idx = I - OffsetReg0;
int RegId =
(Idx / NumElts) * NumParts + (Idx % NumElts) / EltsPerVector;
if (FirstRegId < 0)
FirstRegId = RegId;
RegIndices.insert(RegId);
if (RegIndices.size() > 2)
return std::nullopt;
if (RegIndices.size() == 2) {
ShuffleKind = TTI::SK_PermuteTwoSrc;
if (Indices.size() == 1) {
OffsetReg1 = alignDown(
std::accumulate(
std::next(Mask.begin(), Pos), Mask.end(), INT_MAX,
[&](int S, int I) {
if (I == PoisonMaskElem)
return S;
int RegId = ((I - OffsetReg0) / NumElts) * NumParts +
((I - OffsetReg0) % NumElts) / EltsPerVector;
if (RegId == FirstRegId)
return S;
return std::min(S, I);
}),
EltsPerVector);
unsigned Index = OffsetReg1 % NumElts;
Indices.push_back(Index);
SubVecSizes.push_back(std::min(NumElts - Index, EltsPerVector));
}
Idx = I - OffsetReg1;
}
I = (Idx % NumElts) % EltsPerVector +
(RegId == FirstRegId ? 0 : EltsPerVector);
}
return ShuffleKind;
};
InstructionCost Cost = 0;
// Process extracts in blocks of EltsPerVector to check if the source vector
// operand can be re-used directly. If not, add the cost of creating a
// shuffle to extract the values into a vector register.
for (unsigned Part : seq<unsigned>(NumParts)) {
if (!ShuffleKinds[Part])
continue;
ArrayRef<int> MaskSlice = Mask.slice(
Part * EltsPerVector, getNumElems(Mask.size(), EltsPerVector, Part));
SmallVector<int> SubMask(EltsPerVector, PoisonMaskElem);
copy(MaskSlice, SubMask.begin());
SmallVector<unsigned, 2> Indices;
SmallVector<unsigned, 2> SubVecSizes;
std::optional<TTI::ShuffleKind> RegShuffleKind =
CheckPerRegistersShuffle(SubMask, Indices, SubVecSizes);
if (!RegShuffleKind) {
if (*ShuffleKinds[Part] != TTI::SK_PermuteSingleSrc ||
!ShuffleVectorInst::isIdentityMask(
MaskSlice, std::max<unsigned>(NumElts, MaskSlice.size())))
Cost += getShuffleCost(
TTI, *ShuffleKinds[Part],
cast<VectorType>(getWidenedType(ScalarTy, NumElts)), CostKind,
MaskSlice);
continue;
}
if (*RegShuffleKind != TTI::SK_PermuteSingleSrc ||
!ShuffleVectorInst::isIdentityMask(SubMask, EltsPerVector)) {
Cost += getShuffleCost(
TTI, *RegShuffleKind,
cast<VectorType>(getWidenedType(ScalarTy, EltsPerVector)), CostKind,
SubMask);
}
const unsigned BaseVF = getFullVectorNumberOfElements(
*R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector),
R.Opts.slp_revec);
for (const auto [Idx, SubVecSize] : zip(Indices, SubVecSizes)) {
assert((Idx + SubVecSize) <= BaseVF &&
"SK_ExtractSubvector index out of range");
Cost += getShuffleCost(
TTI, TTI::SK_ExtractSubvector,
cast<VectorType>(getWidenedType(ScalarTy, BaseVF)), CostKind, {},
Idx, cast<VectorType>(getWidenedType(ScalarTy, SubVecSize)));
}
// Second attempt to check, if just a permute is better estimated than
// subvector extract.
SubMask.assign(NumElts, PoisonMaskElem);
copy(MaskSlice, SubMask.begin());
InstructionCost OriginalCost =
getShuffleCost(TTI, *ShuffleKinds[Part],
cast<VectorType>(getWidenedType(ScalarTy, NumElts)),
CostKind, SubMask);
if (OriginalCost < Cost)
Cost = OriginalCost;
}
return Cost;
}
/// Adds the cost of reshuffling \p E1 and \p E2 (if present), using given
/// mask \p Mask, register number \p Part, that includes \p SliceSize
/// elements.
void estimateNodesPermuteCost(const TreeEntry &E1, const TreeEntry *E2,
ArrayRef<int> Mask, unsigned Part,
unsigned SliceSize) {
if (SameNodesEstimated) {
// Delay the cost estimation if the same nodes are reshuffling.
// If we already requested the cost of reshuffling of E1 and E2 before, no
// need to estimate another cost with the sub-Mask, instead include this
// sub-Mask into the CommonMask to estimate it later and avoid double cost
// estimation.
if ((InVectors.size() == 2 &&
cast<const TreeEntry *>(InVectors.front().Vec) == &E1 &&
cast<const TreeEntry *>(InVectors.back().Vec) == E2) ||
(!E2 && cast<const TreeEntry *>(InVectors.front().Vec) == &E1)) {
unsigned Limit = getNumElems(Mask.size(), SliceSize, Part);
assert(all_of(ArrayRef(CommonMask).slice(Part * SliceSize, Limit),
[](int Idx) { return Idx == PoisonMaskElem; }) &&
"Expected all poisoned elements.");
ArrayRef<int> SubMask = ArrayRef(Mask).slice(Part * SliceSize, Limit);
copy(SubMask, std::next(CommonMask.begin(), SliceSize * Part));
return;
}
// Found non-matching nodes - need to estimate the cost for the matched
// and transform mask.
if (InVectors.size() == 1)
Cost += createShuffle(InVectors.front().Vec, nullptr, CommonMask, {},
InVectors.front().Ctx);
else
Cost += createShuffle(InVectors.front().Vec, InVectors.back().Vec,
CommonMask, {}, InVectors.front().Ctx,
InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else if (InVectors.size() == 2) {
Cost +=
createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
{}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
SameNodesEstimated = false;
if (!E2 && InVectors.size() == 1) {
unsigned VF = E1.getVectorFactor();
if (Value *V1 = dyn_cast<Value *>(InVectors.front().Vec)) {
VF = std::max(VF, getVF(V1));
} else {
const auto *E = cast<const TreeEntry *>(InVectors.front().Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
Cost += createShuffle(InVectors.front().Vec, &E1, CommonMask, {},
InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else {
auto P = InVectors.front();
Cost += createShuffle(&E1, E2, Mask, {}, TTI::VectorInstrContext::None);
unsigned VF = Mask.size();
if (Value *V1 = dyn_cast<Value *>(P.Vec)) {
VF = std::max(VF,
getNumElements(V1->getType()));
} else {
const auto *E = cast<const TreeEntry *>(P.Vec);
VF = std::max(VF, E->getVectorFactor());
}
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx + (InVectors.empty() ? 0 : VF);
Cost += createShuffle(P.Vec, InVectors.front().Vec, CommonMask, {}, P.Ctx,
InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
}
class ShuffleCostBuilder {
const TargetTransformInfo &TTI;
const TTI::TargetCostKind CostKind;
static bool isEmptyOrIdentity(ArrayRef<int> Mask, unsigned VF) {
int Index = -1;
return Mask.empty() ||
(VF == Mask.size() &&
ShuffleVectorInst::isIdentityMask(Mask, VF)) ||
(ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) &&
Index == 0);
}
public:
ShuffleCostBuilder(const TargetTransformInfo &TTI,
const TTI::TargetCostKind CostKind)
: TTI(TTI), CostKind(CostKind) {}
~ShuffleCostBuilder() = default;
InstructionCost createShuffleVector(Value *V1, Value *,
ArrayRef<int> Mask) const {
// Empty mask or identity mask are free.
unsigned VF =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
return getShuffleCost(TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(V1->getType()), CostKind, Mask);
}
InstructionCost
createShuffleVector(Value *V1, ArrayRef<int> Mask, ArrayRef<Value *> VL,
TTI::VectorInstrContext ContextHint =
TTI::VectorInstrContext::None) const {
// Empty mask or identity mask are free.
unsigned VF =
cast<VectorType>(V1->getType())->getElementCount().getKnownMinValue();
if (isEmptyOrIdentity(Mask, VF))
return TTI::TCC_Free;
return getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(V1->getType()), CostKind, Mask,
/*Index=*/0, /*SubTp=*/nullptr, VL, ContextHint);
}
InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; }
InstructionCost createPoison(Type *Ty, unsigned VF) const {
return TTI::TCC_Free;
}
void resizeToMatch(Value *&, Value *&) const {}
};
/// Smart shuffle instruction emission, walks through shuffles trees and
/// tries to find the best matching vector for the actual shuffle
/// instruction.
InstructionCost
createShuffle(const PointerUnion<Value *, const TreeEntry *> &P1,
const PointerUnion<Value *, const TreeEntry *> &P2,
ArrayRef<int> Mask, ArrayRef<Value *> VL = {},
TTI::VectorInstrContext Ctx1 = TTI::VectorInstrContext::None,
TTI::VectorInstrContext Ctx2 = TTI::VectorInstrContext::None) {
ShuffleCostBuilder Builder(TTI, CostKind);
SmallVector<int> CommonMask(Mask);
Value *V1 = dyn_cast<Value *>(P1), *V2 = dyn_cast_if_present<Value *>(P2);
unsigned CommonVF = Mask.size();
InstructionCost ExtraCost = 0;
auto GetNodeMinBWAffectedCost = [&](const TreeEntry &E,
unsigned VF) -> InstructionCost {
if (E.isGather() && allConstant(E.Scalars))
return TTI::TCC_Free;
Type *EScalarTy = E.Scalars.front()->getType();
bool IsSigned = true;
if (auto It = R.MinBWs.find(&E); It != R.MinBWs.end()) {
EScalarTy = IntegerType::get(EScalarTy->getContext(), It->second.first);
IsSigned = It->second.second;
}
if (EScalarTy != ScalarTy) {
unsigned CastOpcode = Instruction::Trunc;
unsigned DstSz = R.DL->getTypeSizeInBits(ScalarTy);
unsigned SrcSz = R.DL->getTypeSizeInBits(EScalarTy);
if (DstSz > SrcSz)
CastOpcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
return TTI.getCastInstrCost(CastOpcode, getWidenedType(ScalarTy, VF),
getWidenedType(EScalarTy, VF),
R.getCastContextHint(E), CostKind);
}
return TTI::TCC_Free;
};
auto GetValueMinBWAffectedCost = [&](const Value *V) -> InstructionCost {
if (isa<Constant>(V))
return TTI::TCC_Free;
auto *VecTy = cast<VectorType>(V->getType());
Type *EScalarTy = VecTy->getElementType();
if (EScalarTy != ScalarTy) {
bool IsSigned = !isKnownNonNegative(V, SimplifyQuery(*R.DL));
unsigned CastOpcode = Instruction::Trunc;
unsigned DstSz = R.DL->getTypeSizeInBits(ScalarTy);
unsigned SrcSz = R.DL->getTypeSizeInBits(EScalarTy);
if (DstSz > SrcSz)
CastOpcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
TTI::CastContextHint CCH = TTI::CastContextHint::None;
if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(V); TEs.size() == 1)
CCH = R.getCastContextHint(*TEs.front());
return TTI.getCastInstrCost(
CastOpcode, VectorType::get(ScalarTy, VecTy->getElementCount()),
VecTy, CCH, CostKind);
}
return TTI::TCC_Free;
};
if (!V1 && !V2 && !P2.isNull()) {
// Shuffle 2 entry nodes.
const TreeEntry *E = cast<const TreeEntry *>(P1);
unsigned VF = E->getVectorFactor();
const TreeEntry *E2 = cast<const TreeEntry *>(P2);
CommonVF = std::max(VF, E2->getVectorFactor());
assert(all_of(Mask,
[=](int Idx) {
return Idx < 2 * static_cast<int>(CommonVF);
}) &&
"All elements in mask must be less than 2 * CommonVF.");
if (E->Scalars.size() == E2->Scalars.size()) {
SmallVector<int> EMask = E->getCommonMask();
SmallVector<int> E2Mask = E2->getCommonMask();
if (!EMask.empty() || !E2Mask.empty()) {
for (int &Idx : make_filter_range(
CommonMask, [](int Idx) { return Idx != PoisonMaskElem; })) {
if (Idx < static_cast<int>(CommonVF) && !EMask.empty())
Idx = EMask[Idx];
else if (Idx >= static_cast<int>(CommonVF))
Idx = (E2Mask.empty() ? Idx - CommonVF : E2Mask[Idx - CommonVF]) +
E->Scalars.size();
}
}
CommonVF = E->Scalars.size();
ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF) +
GetNodeMinBWAffectedCost(*E2, CommonVF);
} else {
ExtraCost += GetNodeMinBWAffectedCost(*E, E->getVectorFactor()) +
GetNodeMinBWAffectedCost(*E2, E2->getVectorFactor());
}
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else if (!V1 && P2.isNull()) {
// Shuffle single entry node.
const TreeEntry *E = cast<const TreeEntry *>(P1);
unsigned VF = E->getVectorFactor();
CommonVF = VF;
assert(
all_of(Mask,
[=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
"All elements in mask must be less than CommonVF.");
if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
SmallVector<int> EMask = E->getCommonMask();
assert(!EMask.empty() && "Expected non-empty common mask.");
for (int &Idx : CommonMask) {
if (Idx != PoisonMaskElem)
Idx = EMask[Idx];
}
CommonVF = E->Scalars.size();
} else if (unsigned Factor = E->getInterleaveFactor();
Factor > 0 && E->Scalars.size() != Mask.size() &&
ShuffleVectorInst::isDeInterleaveMaskOfFactor(CommonMask,
Factor)) {
// Deinterleaved nodes are free.
std::iota(CommonMask.begin(), CommonMask.end(), 0);
}
ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF);
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
// Not identity/broadcast? Try to see if the original vector is better.
if (!E->ReorderIndices.empty() && CommonVF == E->ReorderIndices.size() &&
CommonVF == CommonMask.size() &&
any_of(enumerate(CommonMask),
[](const auto &&P) {
return P.value() != PoisonMaskElem &&
static_cast<unsigned>(P.value()) != P.index();
}) &&
any_of(CommonMask,
[](int Idx) { return Idx != PoisonMaskElem && Idx != 0; })) {
SmallVector<int> ReorderMask;
inversePermutation(E->ReorderIndices, ReorderMask);
addMask(CommonMask, ReorderMask);
}
} else if (V1 && P2.isNull()) {
// Shuffle single vector.
ExtraCost += GetValueMinBWAffectedCost(V1);
CommonVF = getVF(V1);
assert(
all_of(Mask,
[=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
"All elements in mask must be less than CommonVF.");
} else if (V1 && !V2) {
// Shuffle vector and tree node.
unsigned VF = getVF(V1);
const TreeEntry *E2 = cast<const TreeEntry *>(P2);
CommonVF = std::max(VF, E2->getVectorFactor());
assert(all_of(Mask,
[=](int Idx) {
return Idx < 2 * static_cast<int>(CommonVF);
}) &&
"All elements in mask must be less than 2 * CommonVF.");
if (E2->Scalars.size() == VF && VF != CommonVF) {
SmallVector<int> E2Mask = E2->getCommonMask();
assert(!E2Mask.empty() && "Expected non-empty common mask.");
for (int &Idx : make_filter_range(
CommonMask, [](int Idx) { return Idx != PoisonMaskElem; })) {
if (Idx >= static_cast<int>(CommonVF))
Idx = E2Mask[Idx - CommonVF] + VF;
}
CommonVF = VF;
}
ExtraCost += GetValueMinBWAffectedCost(V1);
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
ExtraCost += GetNodeMinBWAffectedCost(
*E2, std::min(CommonVF, E2->getVectorFactor()));
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else if (!V1 && V2) {
// Shuffle vector and tree node.
unsigned VF = getVF(V2);
const TreeEntry *E1 = cast<const TreeEntry *>(P1);
CommonVF = std::max(VF, E1->getVectorFactor());
assert(all_of(Mask,
[=](int Idx) {
return Idx < 2 * static_cast<int>(CommonVF);
}) &&
"All elements in mask must be less than 2 * CommonVF.");
if (E1->Scalars.size() == VF && VF != CommonVF) {
SmallVector<int> E1Mask = E1->getCommonMask();
assert(!E1Mask.empty() && "Expected non-empty common mask.");
for (int &Idx : make_filter_range(
CommonMask, [](int Idx) { return Idx != PoisonMaskElem; })) {
if (Idx >= static_cast<int>(CommonVF))
Idx = E1Mask[Idx - CommonVF] + VF;
else
Idx = E1Mask[Idx];
}
CommonVF = VF;
}
ExtraCost += GetNodeMinBWAffectedCost(
*E1, std::min(CommonVF, E1->getVectorFactor()));
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
ExtraCost += GetValueMinBWAffectedCost(V2);
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else {
assert(V1 && V2 && "Expected both vectors.");
unsigned VF = getVF(V1);
CommonVF = std::max(VF, getVF(V2));
assert(all_of(Mask,
[=](int Idx) {
return Idx < 2 * static_cast<int>(CommonVF);
}) &&
"All elements in mask must be less than 2 * CommonVF.");
ExtraCost +=
GetValueMinBWAffectedCost(V1) + GetValueMinBWAffectedCost(V2);
if (V1->getType() != V2->getType()) {
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else {
if (cast<VectorType>(V1->getType())->getElementType() != ScalarTy)
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
if (cast<VectorType>(V2->getType())->getElementType() != ScalarTy)
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
}
}
InVectors.front() =
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
if (InVectors.size() == 2)
InVectors.pop_back();
TTI::VectorInstrContext Ctx =
P2.isNull() ? Ctx1 : TTI::combineVectorInstrContexts(Ctx1, Ctx2);
return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
V1, V2, CommonMask, Builder, ScalarTy,
R.Opts.slp_revec, VL, Ctx);
}
public:
ShuffleCostEstimator(Type *ScalarTy, TargetTransformInfo &TTI,
ArrayRef<Value *> VectorizedVals, BoUpSLP &R,
SmallPtrSetImpl<Value *> &CheckedExtracts)
: BaseShuffleAnalysis(ScalarTy), TTI(TTI),
VectorizedVals(VectorizedVals.begin(), VectorizedVals.end()), R(R),
CheckedExtracts(CheckedExtracts), CostKind(R.getCostKind()) {}
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
unsigned NumParts, bool &UseVecBaseAsInput) {
UseVecBaseAsInput = false;
if (Mask.empty())
return nullptr;
Value *VecBase = nullptr;
SmallVector<Value *> VL(E->Scalars.begin(), E->Scalars.end());
if (!E->ReorderIndices.empty()) {
SmallVector<int> ReorderMask(E->ReorderIndices.begin(),
E->ReorderIndices.end());
reorderScalars(VL, ReorderMask);
}
// Check if it can be considered reused if same extractelements were
// vectorized already.
bool PrevNodeFound = any_of(
ArrayRef(R.VectorizableTree).take_front(E->Idx),
[&](const std::unique_ptr<TreeEntry> &TE) {
return ((TE->hasState() && !TE->isAltShuffle() &&
TE->getOpcode() == Instruction::ExtractElement) ||
TE->isGather()) &&
all_of(enumerate(TE->Scalars), [&](auto &&Data) {
return VL.size() > Data.index() &&
(Mask[Data.index()] == PoisonMaskElem ||
isa<UndefValue>(VL[Data.index()]) ||
Data.value() == VL[Data.index()]);
});
});
SmallPtrSet<Value *, 4> UniqueBases;
unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
SmallDenseMap<Value *, APInt, 4> VectorOpsToExtracts;
for (unsigned Part : seq<unsigned>(NumParts)) {
unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
ArrayRef<int> SubMask = Mask.slice(Part * SliceSize, Limit);
for (auto [I, V] :
enumerate(ArrayRef(VL).slice(Part * SliceSize, Limit))) {
// Ignore non-extractelement scalars.
if (isa<UndefValue>(V) ||
(!SubMask.empty() && SubMask[I] == PoisonMaskElem))
continue;
// If all users of instruction are going to be vectorized and this
// instruction itself is not going to be vectorized, consider this
// instruction as dead and remove its cost from the final cost of the
// vectorized tree.
// Also, avoid adjusting the cost for extractelements with multiple uses
// in different graph entries.
auto *EE = cast<ExtractElementInst>(V);
VecBase = EE->getVectorOperand();
UniqueBases.insert(VecBase);
ArrayRef<TreeEntry *> VEs = R.getTreeEntries(V);
if (!CheckedExtracts.insert(V).second ||
!R.areAllUsersVectorized(cast<Instruction>(V), &VectorizedVals) ||
any_of(VEs,
[&](const TreeEntry *TE) {
return R.DeletedNodes.contains(TE) ||
R.TransformedToGatherNodes.contains(TE);
}) ||
(E->UserTreeIndex && E->UserTreeIndex.EdgeIdx == UINT_MAX &&
!R.isVectorized(EE) &&
count_if(E->Scalars, [&](Value *V) { return V == EE; }) !=
count_if(E->UserTreeIndex.UserTE->Scalars,
[&](Value *V) { return V == EE; })) ||
any_of(EE->users(),
[&](User *U) {
return isa<GetElementPtrInst>(U) &&
!R.areAllUsersVectorized(cast<Instruction>(U),
&VectorizedVals);
}) ||
(!VEs.empty() && !is_contained(VEs, E)))
continue;
std::optional<unsigned> EEIdx = getExtractIndex(EE);
if (!EEIdx)
continue;
unsigned Idx = *EEIdx;
// Take credit for instruction that will become dead.
if (EE->hasOneUse() || !PrevNodeFound) {
Instruction *Ext = EE->user_back();
if (isa<SExtInst, ZExtInst>(Ext) &&
all_of(Ext->users(), IsaPred<GetElementPtrInst>)) {
// Use getExtractWithExtendCost() to calculate the cost of
// extractelement/ext pair.
Cost -= TTI.getExtractWithExtendCost(
Ext->getOpcode(), Ext->getType(), EE->getVectorOperandType(),
Idx, CostKind);
// Add back the cost of s|zext which is subtracted separately.
Cost += TTI.getCastInstrCost(
Ext->getOpcode(), Ext->getType(), EE->getType(),
TTI::getCastContextHint(Ext), CostKind, Ext);
continue;
}
}
APInt &DemandedElts =
VectorOpsToExtracts
.try_emplace(VecBase,
APInt::getZero(getNumElements(VecBase->getType())))
.first->getSecond();
DemandedElts.setBit(Idx);
}
}
for (const auto &[Vec, DemandedElts] : VectorOpsToExtracts)
Cost -= TTI.getScalarizationOverhead(cast<VectorType>(Vec->getType()),
DemandedElts, /*Insert=*/false,
/*Extract=*/true, CostKind);
// Check that gather of extractelements can be represented as just a
// shuffle of a single/two vectors the scalars are extracted from.
// Found the bunch of extractelement instructions that must be gathered
// into a vector and can be represented as a permutation elements in a
// single input vector or of 2 input vectors.
// Done for reused if same extractelements were vectorized already.
if (!PrevNodeFound)
Cost += computeExtractCost(VL, Mask, ShuffleKinds, NumParts);
InVectors.assign(1, E);
CommonMask.assign(Mask.begin(), Mask.end());
transformMaskAfterShuffle(CommonMask, CommonMask);
SameNodesEstimated = false;
if (NumParts != 1 && UniqueBases.size() != 1) {
UseVecBaseAsInput = true;
VecBase =
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
}
return VecBase;
}
/// Checks if the specified entry \p E needs to be delayed because of its
/// dependency nodes.
std::optional<InstructionCost>
needToDelay(const TreeEntry *,
ArrayRef<SmallVector<const TreeEntry *>>) const {
// No need to delay the cost estimation during analysis.
return std::nullopt;
}
/// Reset the builder to handle perfect diamond match.
void resetForSameNode() {
IsFinalized = false;
CommonMask.clear();
InVectors.clear();
Cost = 0;
VectorizedVals.clear();
SameNodesEstimated = true;
}
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
BVValues.reset();
if (&E1 == &E2) {
assert(all_of(Mask,
[&](int Idx) {
return Idx < static_cast<int>(E1.getVectorFactor());
}) &&
"Expected single vector shuffle mask.");
add(E1, Mask);
return;
}
if (InVectors.empty()) {
CommonMask.assign(Mask.begin(), Mask.end());
InVectors.assign({&E1, &E2});
return;
}
assert(!CommonMask.empty() && "Expected non-empty common mask.");
auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
unsigned NumParts = R.getNumberOfParts(MaskVecTy, ScalarTy, Mask.size());
unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
const auto *It = find_if(Mask, not_equal_to(PoisonMaskElem));
unsigned Part = std::distance(Mask.begin(), It) / SliceSize;
estimateNodesPermuteCost(E1, &E2, Mask, Part, SliceSize);
}
void add(const TreeEntry &E1, ArrayRef<int> Mask) {
BVValues.reset();
if (InVectors.empty()) {
CommonMask.assign(Mask.begin(), Mask.end());
InVectors.assign(1, &E1);
return;
}
assert(!CommonMask.empty() && "Expected non-empty common mask.");
auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
unsigned NumParts = R.getNumberOfParts(MaskVecTy, ScalarTy, Mask.size());
unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
const auto *It = find_if(Mask, not_equal_to(PoisonMaskElem));
unsigned Part = std::distance(Mask.begin(), It) / SliceSize;
estimateNodesPermuteCost(E1, nullptr, Mask, Part, SliceSize);
if (!SameNodesEstimated && InVectors.size() == 1)
InVectors.emplace_back(&E1);
}
/// Adds 2 input vectors and the mask for their shuffling.
void add(Value *V1, Value *V2, ArrayRef<int> Mask) {
// May come only for shuffling of 2 vectors with extractelements, already
// handled in adjustExtracts.
assert(InVectors.size() == 1 &&
all_of(enumerate(CommonMask),
[&](auto P) {
if (P.value() == PoisonMaskElem)
return Mask[P.index()] == PoisonMaskElem;
auto *EI = cast<ExtractElementInst>(
cast<const TreeEntry *>(InVectors.front().Vec)
->getOrdered(P.index()));
return EI->getVectorOperand() == V1 ||
EI->getVectorOperand() == V2;
}) &&
"Expected extractelement vectors.");
}
/// Adds another one input vector and the mask for the shuffling.
void
add(Value *V1, ArrayRef<int> Mask, bool ForExtracts = false,
TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
if (BVValues && !isa<Constant>(V1))
BVValues.reset();
if (InVectors.empty()) {
assert(CommonMask.empty() && !ForExtracts &&
"Expected empty input mask/vectors.");
CommonMask.assign(Mask.begin(), Mask.end());
InVectors.assign(1, InVector(V1, ContextHint));
return;
}
if (ForExtracts) {
// No need to add vectors here, already handled them in adjustExtracts.
assert(InVectors.size() == 1 &&
isa<const TreeEntry *>(InVectors[0].Vec) && !CommonMask.empty() &&
all_of(enumerate(CommonMask),
[&](auto P) {
Value *Scalar = cast<const TreeEntry *>(InVectors[0].Vec)
->getOrdered(P.index());
if (P.value() == PoisonMaskElem)
return P.value() == Mask[P.index()] ||
isa<UndefValue>(Scalar);
if (isa<Constant>(V1))
return true;
auto *EI = cast<ExtractElementInst>(Scalar);
return EI->getVectorOperand() == V1;
}) &&
"Expected only tree entry for extractelement vectors.");
return;
}
assert(!InVectors.empty() && !CommonMask.empty() &&
"Expected only tree entries from extracts/reused buildvectors.");
unsigned VF = getVF(V1);
if (InVectors.size() == 2) {
Cost +=
createShuffle(InVectors.front().Vec, InVectors.back().Vec, CommonMask,
{}, InVectors.front().Ctx, InVectors.back().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
VF = std::max<unsigned>(VF, CommonMask.size());
} else if (const auto *InTE =
dyn_cast<const TreeEntry *>(InVectors.front().Vec)) {
VF = std::max(VF, InTE->getVectorFactor());
} else {
VF = std::max(VF, cast<FixedVectorType>(
cast<Value *>(InVectors.front().Vec)->getType())
->getNumElements());
}
InVectors.emplace_back(V1, ContextHint);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + VF;
}
Value *
gather(ArrayRef<Value *> VL, unsigned MaskVF = 0, Value *Root = nullptr,
TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None) {
Cost += getBuildVectorCost(VL, Root, ContextHint);
if (BVValues) {
if (BVValues->empty() && InVectors.empty())
BVValues->assign(VL.begin(), VL.end());
else
BVValues.reset();
}
if (!Root) {
// FIXME: Need to find a way to avoid use of getNullValue here.
SmallVector<Constant *> Vals;
unsigned VF = VL.size();
if (MaskVF != 0)
VF = std::min(VF, MaskVF);
Type *VLScalarTy = VL.front()->getType();
for (Value *V : VL.take_front(VF)) {
Type *ScalarTy = VLScalarTy->getScalarType();
if (isa<PoisonValue>(V)) {
Vals.push_back(PoisonValue::get(ScalarTy));
continue;
}
if (isa<UndefValue>(V)) {
Vals.push_back(UndefValue::get(ScalarTy));
continue;
}
Vals.push_back(Constant::getNullValue(ScalarTy));
}
if (auto *VecTy = dyn_cast<FixedVectorType>(VLScalarTy)) {
assert(R.Opts.slp_revec && "FixedVectorType is not expected.");
// When REVEC is enabled, we need to expand vector types into scalar
// types.
Vals = replicateMask(Vals, VecTy->getNumElements());
}
return ConstantVector::get(Vals);
}
return ConstantVector::getSplat(
ElementCount::getFixed(
cast<FixedVectorType>(Root->getType())->getNumElements()),
getAllOnesValue(*R.DL, ScalarTy->getScalarType()));
}
/// Estimates the cost of the gather of zero-extended sub-fields of width
/// \p FieldWidth of the same wider integer scalar \p Src, permuted by
/// \p Mask, as a bitcast to the field vector plus a permutation and an
/// extension to the lane type.
InstructionCost createExtractedFieldsVector(Value *Src, unsigned FieldWidth,
ArrayRef<int> Mask,
const TreeEntry &E) {
auto *FieldTy = IntegerType::get(Src->getContext(), FieldWidth);
unsigned SrcWidth = Src->getType()->getIntegerBitWidth();
assert(SrcWidth % FieldWidth == 0 &&
"Expected the field width to divide the source width.");
unsigned NumFields = SrcWidth / FieldWidth;
auto *FieldVecTy = cast<VectorType>(getWidenedType(FieldTy, NumFields));
TTI::CastContextHint CCH = R.getCastContextHint(E);
InstructionCost Cost = TTI.getCastInstrCost(
Instruction::BitCast, FieldVecTy, Src->getType(), CCH, CostKind);
// The permutation is elided only for the full-length identity mask.
if (Mask.size() != NumFields ||
!ShuffleVectorInst::isIdentityMask(Mask, NumFields))
Cost += getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, FieldVecTy,
CostKind, Mask);
if (!ScalarTy->isIntegerTy(FieldWidth))
Cost += TTI.getCastInstrCost(
Instruction::ZExt, getWidenedType(ScalarTy, Mask.size()),
getWidenedType(FieldTy, Mask.size()), CCH, CostKind);
// The extraction instructions die once the fields are emitted from the
// source scalar; their scalar cost is credited back for the gathers
// directly feeding the reduction. Instructions vectorized elsewhere are
// skipped: their scalar cost is already a part of the scalar baseline.
if (R.UserIgnoreList &&
(!E.UserTreeIndex || E.UserTreeIndex.UserTE->Idx == 0))
for (Instruction *I : make_isa_range<Instruction>(E.Scalars))
if (CheckedExtracts.insert(I).second && !R.isVectorized(I) &&
R.areAllUsersVectorized(I, &VectorizedVals))
Cost -= isa<CastInst>(I)
? TTI.getCastInstrCost(I->getOpcode(), I->getType(),
I->getOperand(0)->getType(),
TTI::getCastContextHint(I),
CostKind, I)
: TTI.getArithmeticInstrCost(
I->getOpcode(), I->getType(), CostKind,
TTI::getOperandInfo(I->getOperand(0)),
TTI::getOperandInfo(I->getOperand(1)),
{I->getOperand(0), I->getOperand(1)}, I);
return Cost;
}
InstructionCost createFreeze(InstructionCost Cost) { return Cost; }
/// Finalize emission of the shuffles.
InstructionCost finalize(
ArrayRef<int> ExtMask,
ArrayRef<std::pair<const TreeEntry *, unsigned>> SubVectors,
ArrayRef<int> SubVectorsMask, unsigned VF = 0,
function_ref<void(Value *&, SmallVectorImpl<int> &,
function_ref<Value *(Value *, Value *, ArrayRef<int>)>)>
Action = {}) {
IsFinalized = true;
if (Action) {
const PointerUnion<Value *, const TreeEntry *> &Vec =
InVectors.front().Vec;
if (InVectors.size() == 2)
Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
InVectors.front().Ctx, InVectors.back().Ctx);
else
Cost +=
createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
assert(VF > 0 &&
"Expected vector length for the final value before action.");
Value *V = cast<Value *>(Vec);
Action(V, CommonMask, [this](Value *V1, Value *V2, ArrayRef<int> Mask) {
Cost += createShuffle(V1, V2, Mask);
return V1;
});
InVectors.front() = V;
}
if (!SubVectors.empty()) {
const PointerUnion<Value *, const TreeEntry *> &Vec =
InVectors.front().Vec;
if (InVectors.size() == 2)
Cost += createShuffle(Vec, InVectors.back().Vec, CommonMask, {},
InVectors.front().Ctx, InVectors.back().Ctx);
else
Cost +=
createShuffle(Vec, nullptr, CommonMask, {}, InVectors.front().Ctx);
transformMaskAfterShuffle(CommonMask, CommonMask);
// Add subvectors permutation cost.
if (!SubVectorsMask.empty()) {
assert(SubVectorsMask.size() <= CommonMask.size() &&
"Expected same size of masks for subvectors and common mask.");
SmallVector<int> SVMask(CommonMask.size(), PoisonMaskElem);
copy(SubVectorsMask, SVMask.begin());
for (auto [I1, I2] : zip(SVMask, CommonMask)) {
if (I2 != PoisonMaskElem) {
assert(I1 == PoisonMaskElem && "Expected unused subvectors mask");
I1 = I2 + CommonMask.size();
}
}
Cost += getShuffleCost(
TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(getWidenedType(ScalarTy, CommonMask.size())),
CostKind, SVMask);
}
for (auto [E, Idx] : SubVectors) {
Type *EScalarTy = E->Scalars.front()->getType();
bool IsSigned = true;
if (auto It = R.MinBWs.find(E); It != R.MinBWs.end()) {
EScalarTy =
IntegerType::get(EScalarTy->getContext(), It->second.first);
IsSigned = It->second.second;
}
if (ScalarTy != EScalarTy) {
unsigned CastOpcode = Instruction::Trunc;
unsigned DstSz = R.DL->getTypeSizeInBits(ScalarTy);
unsigned SrcSz = R.DL->getTypeSizeInBits(EScalarTy);
if (DstSz > SrcSz)
CastOpcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
Cost += TTI.getCastInstrCost(
CastOpcode, getWidenedType(ScalarTy, E->getVectorFactor()),
getWidenedType(EScalarTy, E->getVectorFactor()),
TTI::CastContextHint::Normal, CostKind);
}
Cost += getShuffleCost(
TTI, TTI::SK_InsertSubvector,
cast<VectorType>(getWidenedType(ScalarTy, CommonMask.size())),
CostKind, {}, Idx,
cast<VectorType>(getWidenedType(ScalarTy, E->getVectorFactor())));
if (!CommonMask.empty()) {
std::iota(std::next(CommonMask.begin(), Idx),
std::next(CommonMask.begin(), Idx + E->getVectorFactor()),
Idx);
}
}
}
if (!ExtMask.empty()) {
if (CommonMask.empty()) {
CommonMask.assign(ExtMask.begin(), ExtMask.end());
} else {
SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
if (ExtMask[I] == PoisonMaskElem)
continue;
NewMask[I] = CommonMask[ExtMask[I]];
}
CommonMask.swap(NewMask);
}
}
if (CommonMask.empty()) {
assert(InVectors.size() == 1 && "Expected only one vector with no mask");
return Cost;
}
ArrayRef<Value *> VL;
if (BVValues)
VL = *BVValues;
if (InVectors.size() == 2)
return Cost + createShuffle(InVectors.front().Vec, InVectors.back().Vec,
CommonMask, VL, InVectors.front().Ctx,
InVectors.back().Ctx);
return Cost + createShuffle(InVectors.front().Vec, nullptr, CommonMask, VL,
InVectors.front().Ctx);
}
~ShuffleCostEstimator() {
assert((IsFinalized || CommonMask.empty()) &&
"Shuffle construction must be finalized.");
}
};
const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(const TreeEntry *E,
unsigned Idx) const {
TreeEntry *Op = OperandsToTreeEntry.at({E, Idx});
assert(Op->isSame(E->getOperand(Idx)) && "Operands mismatch!");
return Op;
}
TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
if (TE.State == TreeEntry::ScatterVectorize ||
TE.State == TreeEntry::StridedVectorize)
return TTI::CastContextHint::GatherScatter;
if (TE.State == TreeEntry::CompressVectorize ||
TE.State == TreeEntry::BlendedLoadVectorize)
return TTI::CastContextHint::Masked;
if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load &&
!TE.isAltShuffle()) {
if (TE.ReorderIndices.empty())
return TTI::CastContextHint::Normal;
SmallVector<int> Mask;
inversePermutation(TE.ReorderIndices, Mask);
if (ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
return TTI::CastContextHint::Reversed;
}
// A gather of extracted sub-fields inherits the context of the entry
// vectorizing the common source scalar, or of the source scalar load. The
// scalars erased by the vectorization have no operands to match.
if (TE.isGather() && none_of(make_isa_range<Instruction>(TE.Scalars),
[&](Instruction *I) { return isDeleted(I); }))
if (std::optional<std::tuple<Value *, unsigned, SmallVector<int>>> Fields =
matchGatheredExtractedFields(TE.Scalars, *DL)) {
Value *Src = std::get<0>(*Fields);
if (ArrayRef<TreeEntry *> TEs = getTreeEntries(Src); !TEs.empty())
return getCastContextHint(*TEs.front());
if (isa<LoadInst>(Src))
return TTI::CastContextHint::Normal;
}
return TTI::CastContextHint::None;
}
uint64_t BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
Instruction *U) {
BasicBlock *Parent = nullptr;
if (U) {
// The extractelement for a PHI-node user is created in the incoming
// block that feeds the matching operand, not in the PHI block itself
// When the PHI is inside a loop that incoming block can belong to a deeper
// loop than the PHI block. Scaling by the PHI block would use
// the outer trip count instead of inner*outer, and because
// ExtractCostCalculated deduplicates by scalar (only the first external
// user fixes the scale) it would also make the cost depend on external-user
// ordering. A PHI outside all loops is a plain loop-exit phi: its live-out
// lanes are normally rebuilt as a vector LCSSA phi in the exit block, which
// hoists the extract out of the loop, so scale = 1 (via U->getParent()
// below) is kept and the adjustment is restricted to in-loop PHIs.
if (auto *PHI = dyn_cast<PHINode>(U); PHI && Scalar) {
if (LI->getLoopFor(PHI->getParent())) {
// Use the deepest incoming block among all slots where Scalar
// appears, to be conservative when the same value appears in
// multiple predecessors.
for (unsigned I : seq<unsigned>(PHI->getNumIncomingValues())) {
if (PHI->getIncomingValue(I) != Scalar)
continue;
BasicBlock *InBB = PHI->getIncomingBlock(I);
if (!Parent || LI->getLoopDepth(InBB) > LI->getLoopDepth(Parent))
Parent = InBB;
}
}
}
if (!Parent)
Parent = U->getParent();
} else if (TE.isGather() || TE.State == TreeEntry::SplitVectorize) {
EdgeInfo EI = TE.UserTreeIndex;
while (EI.UserTE) {
if (EI.UserTE->isGather() ||
EI.UserTE->State == TreeEntry::SplitVectorize) {
EI = EI.UserTE->UserTreeIndex;
continue;
}
if (EI.UserTE->State == TreeEntry::Vectorize &&
EI.UserTE->getOpcode() == Instruction::PHI) {
auto *PH = cast<PHINode>(EI.UserTE->getMainOp());
Parent = PH->getIncomingBlock(EI.EdgeIdx);
const Loop *PhiL = LI->getLoopFor(PH->getParent());
const Loop *InL = LI->getLoopFor(Parent);
if (PhiL && InL && !PhiL->contains(InL) && !InL->contains(PhiL)) {
const SCEV *PhiBTC = SE->getBackedgeTakenCount(PhiL);
const SCEV *InBTC = SE->getBackedgeTakenCount(InL);
if (isa<SCEVCouldNotCompute>(PhiBTC) || PhiBTC != InBTC) {
// If all gathered elements are invariant in the phi's nest below
// the common ancestor of the two nests, the gather is a pure
// nest-crossing transfer, paid once per crossing: scale it by
// the common parent nest.
const Loop *FirstVariantL =
findInnermostNonInvariantLoop(PhiL, TE.Scalars);
if (FirstVariantL && FirstVariantL->contains(InL))
Parent = FirstVariantL->getHeader();
}
}
} else {
Parent = EI.UserTE->getMainOp()->getParent();
}
break;
}
if (!Parent)
return 1;
} else {
Parent = TE.getMainOp()->getParent();
}
const Loop *L = LI->getLoopFor(Parent);
if (!L)
return 1;
// The entry's cost is paid once per execution of the innermost loop in
// which some of its operands are variant. Operands that are invariant in
// all enclosing loops are executed once (LICM will hoist them out).
return getLoopNestScale(findInnermostNonInvariantLoop(
L, Scalar ? ArrayRef(Scalar) : ArrayRef(TE.Scalars)));
}
uint64_t BoUpSLP::getLoopNestScale(const Loop *L) {
if (!L || Opts.slp_cost_loop_trip_count == 0)
return 1;
if (auto It = LoopNestScaleCache.find(L); It != LoopNestScaleCache.end())
return It->second;
// Collect loops from L outward up to (but not including) the first cached
// ancestor or the function top, then walk back inward multiplying trip
// counts. Use uint64_t to avoid silent overflow on deep/large nests.
SmallVector<const Loop *> Chain;
for (const Loop *Cur = L; Cur; Cur = Cur->getParentLoop()) {
if (LoopNestScaleCache.contains(Cur))
break;
Chain.push_back(Cur);
}
assert(!Chain.empty() && "Early-return above should have handled cache hit.");
uint64_t Scale = 1;
if (const Loop *Parent = Chain.back()->getParentLoop())
Scale = LoopNestScaleCache.lookup(Parent);
// Walk from the outermost uncached loop inward, accumulating trip counts.
// Use SaturatingMultiply to clamp at uint64_t max on deep/large nests
// rather than wrapping around.
for (const Loop *Cur : reverse(Chain)) {
uint64_t TC = std::max<uint64_t>(
1, getLoopTripCount(Cur, *SE, Opts.slp_cost_loop_trip_count));
Scale = SaturatingMultiply(Scale, TC);
LoopNestScaleCache.try_emplace(Cur, std::max<uint64_t>(1, Scale));
}
return std::max<uint64_t>(1, Scale);
}
uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE,
Instruction *U) {
// Only meaningful for gather/buildvector-like entries; the per-lane
// insertelements that make up such an entry are LICM-hoistable by
// optimizeGatherSequence() when their operand is loop-invariant.
assert((TE.isGather() || TE.State == TreeEntry::SplitVectorize) &&
"Expected gather/split tree entry.");
uint64_t BaseScale = getScaleToLoopIterations(TE, nullptr, U);
if (!Opts.slp_per_lane_gather_scale || Opts.slp_cost_loop_trip_count == 0 ||
BaseScale <= 1)
return BaseScale;
// Average the per-lane execution scales: for each lane, reuse the same
// scale helper the rest of the cost model uses, but ask it about that
// one lane's value. Lanes that are loop-invariant in the current nest
// collapse to their outer-loop scale (or 1 for fully invariant/constant
// lanes), which matches the LICM hoisting performed by
// optimizeGatherSequence(). Cap per-lane contributions by BaseScale so a
// refinement can never raise the cost above the whole-entry scale.
// Each lane contributes at most BaseScale, so Sum is bounded above by
// N * BaseScale. If BaseScale is near uint64_t max (saturated by
// getLoopNestScale on a deep nest) Sum can still overflow uint64_t,
// which would silently wrap and produce a wrong average. Use
// SaturatingAdd and bail out to BaseScale on overflow: the true average
// is bounded above by BaseScale anyway, so this preserves the
// refinement's invariant that it can never raise cost.
uint64_t Sum = 0;
unsigned N = 0;
bool Overflow = false;
for (Value *V :
make_filter_range(TE.Scalars, [](Value *V) { return !isConstant(V); })) {
++N;
uint64_t LaneScale =
std::min(getScaleToLoopIterations(TE, V, U), BaseScale);
Sum = SaturatingAdd(Sum, LaneScale, &Overflow);
if (Overflow)
return BaseScale;
}
if (N == 0)
return BaseScale;
// Ceil-divide so we never round the effective scale down below 1.
uint64_t Numerator = SaturatingAdd(Sum, uint64_t(N - 1), &Overflow);
if (Overflow)
return BaseScale;
uint64_t Avg = Numerator / N;
return std::clamp<uint64_t>(Avg, 1, BaseScale);
}
uint64_t BoUpSLP::getEntryEffectiveScale(const TreeEntry &TE, Instruction *U) {
if (TE.isGather() || TE.State == TreeEntry::SplitVectorize)
return getGatherNodeEffectiveScale(TE, U);
return getScaleToLoopIterations(TE);
}
InstructionCost
BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
Type *VecTy, Type *FinalVecTy,
const TTI::TargetCostKind CostKind) const {
InstructionCost SpillsReloads = 0;
// Estimate vector register pressure per target register class: operand
// vectors plus the result. The same vector operand is counted once via
// CountedOpEntries deduplication. PHIs take the max operand pressure across
// incoming slots (only one predecessor is live at a time) plus the result.
// All-constant operand bundles are skipped.
if (!E->hasState() || E->getOpcode() == Instruction::Store ||
E->getOpcode() == Instruction::ExtractElement ||
E->getOpcode() == Instruction::ExtractValue ||
E->getOpcode() == Instruction::Freeze ||
(E->getOpcode() == Instruction::Load &&
E->State != TreeEntry::ScatterVectorize))
return SpillsReloads;
const bool IsPHI =
E->State == TreeEntry::Vectorize && E->getOpcode() == Instruction::PHI;
SmallPtrSet<const TreeEntry *, 8> CountedOpEntries;
SmallDenseMap<unsigned, unsigned> PressureByClass;
auto AddPartsToClass = [&](unsigned RegClass, unsigned Parts) {
assert(Parts != 0 && "Expected non-zero number of parts (registers).");
PressureByClass[RegClass] += Parts;
};
auto GetEntryVecTy = [&](const TreeEntry *TE) -> std::pair<Type *, Type *> {
Type *ScalarTy = getValueType(TE->Scalars.front(), Opts.slp_revec);
auto BWIt = MinBWs.find(TE);
if (BWIt != MinBWs.end()) {
auto *VTy = dyn_cast<FixedVectorType>(ScalarTy);
ScalarTy = IntegerType::get(F->getContext(), BWIt->second.first);
if (VTy)
ScalarTy = getWidenedType(ScalarTy, VTy->getNumElements());
}
return std::make_pair(ScalarTy,
getWidenedType(ScalarTy, TE->getVectorFactor()));
};
if (E->State == TreeEntry::SplitVectorize) {
for (const auto &[Idx, _] : E->CombinedEntriesWithIndices) {
const TreeEntry *OpTE = VectorizableTree[Idx].get();
if (!CountedOpEntries.insert(OpTE).second)
continue;
auto [ScalarTy, OpVecTy] = GetEntryVecTy(OpTE);
const unsigned Parts = getNumberOfParts(OpVecTy, ScalarTy);
if (Parts == 0)
continue;
const unsigned RC =
TTI->getRegisterClassForType(/*Vector=*/true, OpVecTy);
AddPartsToClass(RC, Parts);
}
} else if (IsPHI) {
// Only one predecessor is live at a time - take the max operand pressure
// across incoming slots.
SmallDenseMap<unsigned, unsigned> MaxOpPressureByClass;
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
const TreeEntry *OpTE = getOperandEntry(E, Idx);
auto [ScalarTy, OpVecTy] = GetEntryVecTy(OpTE);
const unsigned Parts = getNumberOfParts(OpVecTy, ScalarTy);
if (Parts == 0)
continue;
const unsigned RC =
TTI->getRegisterClassForType(/*Vector=*/true, OpVecTy);
MaxOpPressureByClass[RC] = std::max(MaxOpPressureByClass[RC], Parts);
}
for (auto [RC, Parts] : MaxOpPressureByClass)
AddPartsToClass(RC, Parts);
} else {
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
// InsertElement operand 0 is the vector being inserted into, which is
// built incrementally and does not occupy an extra register.
if ((E->getOpcode() == Instruction::InsertElement ||
E->getOpcode() == Instruction::InsertValue) &&
Idx == 0)
continue;
ArrayRef<Value *> Ops = E->getOperand(Idx);
if (Ops.empty() || allConstant(Ops) || isSplat(Ops))
continue;
Value *Op = Ops.front();
if (!Op)
continue;
const TreeEntry *OpTE = getOperandEntry(E, Idx);
if (!CountedOpEntries.insert(OpTE).second)
continue;
auto *OpVecTy = getWidenedType(Op->getType(), Ops.size());
const unsigned Parts = getNumberOfParts(OpVecTy, Op->getType());
if (Parts == 0)
continue;
const unsigned RC =
TTI->getRegisterClassForType(/*Vector=*/true, OpVecTy);
AddPartsToClass(RC, Parts);
}
}
if (E->getOpcode() != Instruction::Load) {
const unsigned ResParts = getNumberOfParts(VecTy, ScalarTy);
if (ResParts != 0) {
const unsigned RC = TTI->getRegisterClassForType(/*Vector=*/true, VecTy);
AddPartsToClass(RC, ResParts);
}
if (VecTy != FinalVecTy) {
const unsigned FinalResParts = getNumberOfParts(FinalVecTy, ScalarTy);
if (FinalResParts != 0) {
const unsigned RC =
TTI->getRegisterClassForType(/*Vector=*/true, FinalVecTy);
AddPartsToClass(RC, FinalResParts);
}
}
}
for (auto [RegClass, UsedRegs] : PressureByClass) {
const unsigned NumAvailRegs = TTI->getNumberOfRegisters(RegClass);
if (NumAvailRegs == 0 || UsedRegs <= NumAvailRegs)
continue;
const unsigned SpillCount = UsedRegs - NumAvailRegs;
InstructionCost SingleRegSpillReload =
TTI->getRegisterClassReloadCost(RegClass, CostKind);
// No need to spill cost only for the root entry (Idx == 0), for reduction
// and non-returning instructions, like void calls.
if (E->Idx > 0 || !UserIgnoreList || !E->Scalars[0]->getType()->isVoidTy())
SingleRegSpillReload +=
TTI->getRegisterClassSpillCost(RegClass, CostKind);
SpillsReloads += SingleRegSpillReload * SpillCount;
}
return SpillsReloads;
}
/// Calculates a VectorInstrContext from the values in \p VL at indices in
/// \p DemandedElts.
static TTI::VectorInstrContext
getVectorInstrContextHint(ArrayRef<Value *> VL, const APInt &DemandedElts) {
TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
for (unsigned I : seq(VL.size())) {
if (!DemandedElts[I])
continue;
Value *V = VL[I];
if (isa<UndefValue>(V))
continue;
auto *Inst = dyn_cast<Instruction>(V);
if (!Inst)
return TTI::VectorInstrContext::None;
TTI::VectorInstrContext IVIC = TTI::getVectorInstrContextHint(Inst);
if (VIC == TTI::VectorInstrContext::None) {
VIC = IVIC;
continue;
}
if (VIC != IVIC)
return TTI::VectorInstrContext::None;
}
return VIC;
}
InstructionCost
BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
SmallPtrSetImpl<Value *> &CheckedExtracts) {
ArrayRef<Value *> VL = E->Scalars;
Type *ScalarTy = getValueType(VL[0], Opts.slp_revec);
if (Opts.slp_revec && E->State == TreeEntry::Vectorize &&
E->getOpcode() == Instruction::InsertElement &&
!E->getOperand(1).back()->getType()->isVectorTy())
ScalarTy = ScalarTy->getScalarType();
if (!isValidElementType(ScalarTy, Opts.slp_revec))
return InstructionCost::getInvalid();
// If we have computed a smaller type for the expression, update VecTy so
// that the costs will be accurate.
auto It = MinBWs.find(E);
Type *OrigScalarTy = ScalarTy;
if (It != MinBWs.end()) {
auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy);
ScalarTy = IntegerType::get(F->getContext(), It->second.first);
if (VecTy)
ScalarTy = getWidenedType(ScalarTy, VecTy->getNumElements());
} else if (E->Idx == 0 && isReducedBitcastRoot() &&
getRootNode().CombinedOp != TreeEntry::ReducedBitPack) {
const TreeEntry *ZExt = getOperandEntry(E, /*Idx=*/0);
ScalarTy = cast<CastInst>(ZExt->getMainOp())->getSrcTy();
}
auto *VecTy = getWidenedType(ScalarTy, VL.size());
unsigned EntryVF = E->getVectorFactor();
auto *FinalVecTy = getWidenedType(ScalarTy, EntryVF);
const InstructionCost SpillsReloads =
getVectorSpillReloadCost(E, ScalarTy, VecTy, FinalVecTy, CostKind);
if (E->isGather() || TransformedToGatherNodes.contains(E)) {
if (allConstant(VL))
return 0;
if (isa<InsertElementInst, InsertValueInst>(VL[0]))
return InstructionCost::getInvalid();
// The fmuls of the node turned into a gather stay scalar and lose the
// fusion with their vectorized fadd/fsub users.
return SpillsReloads +
processBuildVector<ShuffleCostEstimator, InstructionCost>(
E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts) +
getUnfusedFMulsPenalty(*E);
}
if (E->State == TreeEntry::SplitVectorize) {
assert(E->CombinedEntriesWithIndices.size() == 2 &&
"Expected exactly 2 combined entries.");
assert(E->ReuseShuffleIndices.empty() && "Expected empty reuses mask.");
InstructionCost VectorCost = 0;
if (E->ReorderIndices.empty()) {
VectorCost = getShuffleCost(
*TTI, TTI::SK_InsertSubvector, cast<VectorType>(FinalVecTy), CostKind,
{}, E->CombinedEntriesWithIndices.back().second,
cast<VectorType>(getWidenedType(
ScalarTy,
VectorizableTree[E->CombinedEntriesWithIndices.back().first]
->getVectorFactor())));
} else {
unsigned CommonVF =
std::max(VectorizableTree[E->CombinedEntriesWithIndices.front().first]
->getVectorFactor(),
VectorizableTree[E->CombinedEntriesWithIndices.back().first]
->getVectorFactor());
VectorCost =
getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(getWidenedType(ScalarTy, CommonVF)),
CostKind, E->getSplitMask());
}
VectorCost += SpillsReloads;
LLVM_DEBUG(dumpTreeCosts(E, 0, VectorCost, 0, "Calculated costs for Tree"));
return VectorCost;
}
InstructionCost CommonCost = 0;
SmallVector<int> Mask;
if (!E->ReorderIndices.empty() && E->State != TreeEntry::CompressVectorize &&
(E->State != TreeEntry::StridedVectorize || !E->isReverse() ||
isWidenedStridedLoad(E))) {
SmallVector<int> NewMask;
if (E->getOpcode() == Instruction::Store) {
// For stores the order is actually a mask.
NewMask.resize(E->ReorderIndices.size());
copy(E->ReorderIndices, NewMask.begin());
} else {
inversePermutation(E->ReorderIndices, NewMask);
}
addMask(Mask, NewMask);
}
if (!E->ReuseShuffleIndices.empty())
addMask(Mask, E->ReuseShuffleIndices);
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
assert(!isa<StructType>(FinalVecTy) &&
"Expected non-struct vector type for shuffle cost calculation.");
CommonCost = getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(FinalVecTy), CostKind, Mask,
/*Index=*/0, cast<VectorType>(VecTy));
}
assert((E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::ScatterVectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize ||
E->State == TreeEntry::BlendedLoadVectorize) &&
"Unhandled state");
assert(E->getOpcode() &&
((allSameType(VL) && allSameBlock(VL)) ||
(E->getOpcode() == Instruction::GetElementPtr &&
E->getMainOp()->getType()->isPointerTy()) ||
E->hasCopyableElements()) &&
"Invalid VL");
Instruction *VL0 = E->getMainOp();
unsigned ShuffleOrOp =
E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
if (E->CombinedOp != TreeEntry::NotCombinedOp)
ShuffleOrOp = E->CombinedOp;
SmallSetVector<Value *, 16> UniqueValues;
SmallVector<unsigned, 16> UniqueIndexes;
for (auto [Idx, V] : enumerate(VL))
if (UniqueValues.insert(V))
UniqueIndexes.push_back(Idx);
const unsigned Sz = UniqueValues.size();
SmallBitVector UsedScalars(Sz, false);
for (unsigned I = 0; I < Sz; ++I) {
if (isa<Instruction>(UniqueValues[I]) &&
!E->isCopyableElement(UniqueValues[I]) &&
getTreeEntries(UniqueValues[I]).front() == E)
continue;
UsedScalars.set(I);
}
auto GetCastContextHint = [&](Value *V) {
if (ArrayRef<TreeEntry *> OpTEs = getTreeEntries(V); OpTEs.size() == 1)
return getCastContextHint(*OpTEs.front());
InstructionsState SrcState = getSameOpcode(E->getOperand(0), *TLI);
if (SrcState && SrcState.getOpcode() == Instruction::Load &&
!SrcState.isAltShuffle())
return TTI::CastContextHint::GatherScatter;
return TTI::CastContextHint::None;
};
auto GetCostDiff =
[=](function_ref<InstructionCost(unsigned)> ScalarEltCost,
function_ref<InstructionCost(InstructionCost)> VectorCost) {
// Calculate the cost of this instruction.
InstructionCost ScalarCost = 0;
if (isa<CastInst, CallInst>(VL0)) {
// For some of the instructions no need to calculate cost for each
// particular instruction, we can use the cost of the single
// instruction x total number of scalar instructions.
ScalarCost = (Sz - UsedScalars.count()) * ScalarEltCost(0);
} else {
for (unsigned I = 0; I < Sz; ++I) {
if (UsedScalars.test(I))
continue;
ScalarCost += ScalarEltCost(I);
}
}
InstructionCost VecCost = VectorCost(CommonCost);
// Check if the current node must be resized, if the parent node is not
// resized. Nodes without a parent (roots of the splat gather
// subtrees) are resized back to their original type when the reusing
// gather nodes are emitted.
if (It != MinBWs.end() && !UnaryInstruction::isCast(E->getOpcode()) &&
E->Idx != 0 &&
(E->getOpcode() != Instruction::Load || E->UserTreeIndex)) {
const EdgeInfo &EI = E->UserTreeIndex;
if (!EI.UserTE || !EI.UserTE->hasState() ||
EI.UserTE->getOpcode() != Instruction::Select ||
EI.EdgeIdx != 0) {
auto UserBWIt = MinBWs.find(EI.UserTE);
Type *UserScalarTy =
!EI.UserTE ? OrigScalarTy
: (EI.UserTE->isGather() ||
EI.UserTE->State == TreeEntry::SplitVectorize)
? EI.UserTE->Scalars.front()->getType()
: EI.UserTE->getOperand(EI.EdgeIdx).front()->getType();
if (UserBWIt != MinBWs.end())
UserScalarTy = IntegerType::get(ScalarTy->getContext(),
UserBWIt->second.first);
if (ScalarTy != UserScalarTy) {
unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
unsigned SrcBWSz = DL->getTypeSizeInBits(UserScalarTy);
unsigned VecOpcode;
auto *UserVecTy = getWidenedType(UserScalarTy, E->Scalars.size());
if (BWSz > SrcBWSz)
VecOpcode = Instruction::Trunc;
else
VecOpcode =
It->second.second ? Instruction::SExt : Instruction::ZExt;
TTI::CastContextHint CCH = GetCastContextHint(VL0);
VecCost += TTI->getCastInstrCost(VecOpcode, UserVecTy, VecTy, CCH,
CostKind);
}
}
}
VecCost += SpillsReloads;
LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost - CommonCost,
ScalarCost, "Calculated costs for Tree"));
return VecCost - ScalarCost;
};
// Price peeled intermediate instructions on the scalar side: they are
// erased when the node vectorizes. The peeled cost is folded into the
// first scalar-cost query so the cost dump reports the full scalar cost.
// Peeled chain links are always 2-operand associative binops, priced per
// instruction so the operand properties (constants, uniformity) apply.
auto GetCostDiffWithPeeled =
[&](function_ref<InstructionCost(unsigned)> ScalarEltCost,
function_ref<InstructionCost(InstructionCost)> VectorCost) {
InstructionCost PeeledScalarCost = 0;
for (Value *V : E->getReassocScalars()) {
auto *I = cast<Instruction>(V);
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(I->getOperand(0));
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(I->getOperand(1));
PeeledScalarCost += TTI->getArithmeticInstrCost(
I->getOpcode(), OrigScalarTy, CostKind, Op1Info, Op2Info, {}, I);
}
bool PeeledCostAdded = false;
InstructionCost CostDiff = GetCostDiff(
[&](unsigned Idx) {
InstructionCost Cost = ScalarEltCost(Idx);
if (!PeeledCostAdded) {
PeeledCostAdded = true;
Cost += PeeledScalarCost;
}
return Cost;
},
VectorCost);
// Every scalar may be marked as used elsewhere, leaving the
// scalar-cost query uncalled and the peeled cost unapplied.
if (!PeeledCostAdded)
CostDiff -= PeeledScalarCost;
return CostDiff;
};
// Calculate cost difference from vectorizing set of GEPs.
// Negative value means vectorizing is profitable.
auto GetGEPCostDiff = [=](ArrayRef<Value *> Ptrs, Value *BasePtr) {
assert((E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize) &&
"Entry state expected to be Vectorize, StridedVectorize, "
"ExpandVectorize or CompressVectorize here.");
InstructionCost ScalarCost = 0;
InstructionCost VecCost = 0;
std::tie(ScalarCost, VecCost) =
getGEPCosts(*TTI, Ptrs, BasePtr, E->getOpcode(), CostKind, OrigScalarTy,
cast<VectorType>(VecTy));
LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost,
"Calculated GEPs cost for Tree"));
return VecCost - ScalarCost + SpillsReloads;
};
auto GetMinMaxCost = [&](Type *Ty, Instruction *VI = nullptr) {
auto [MinMaxID, SelectOnly] = canConvertToMinOrMaxIntrinsic(VI ? VI : VL);
if (MinMaxID == Intrinsic::not_intrinsic)
return InstructionCost::getInvalid();
Type *CanonicalType = Ty;
if (CanonicalType->isPtrOrPtrVectorTy())
CanonicalType = CanonicalType->getWithNewType(IntegerType::get(
CanonicalType->getContext(),
DL->getTypeSizeInBits(CanonicalType->getScalarType())));
IntrinsicCostAttributes CostAttrs(MinMaxID, CanonicalType,
{CanonicalType, CanonicalType});
InstructionCost IntrinsicCost =
TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
// If the selects are the only uses of the compares, they will be
// dead and we can adjust the cost by removing their cost.
if (VI && SelectOnly) {
assert((!Ty->isVectorTy() || Opts.slp_revec) &&
"Expected only for scalar type.");
auto *CI = cast<CmpInst>(VI->getOperand(0));
IntrinsicCost -= TTI->getCmpSelInstrCost(
CI->getOpcode(), Ty, Builder.getInt1Ty(), CI->getPredicate(),
CostKind, TTI::getOperandInfo(CI->getOperand(0)),
TTI::getOperandInfo(CI->getOperand(1)), CI);
}
return IntrinsicCost;
};
// The fadd/fsub, fused with its fmul operand into an fmuladd in the scalar
// code. If the fmul is not vectorized, the vector code still emits it, so
// only the fusion delta is attributed to the fadd/fsub.
auto GetFMulAddCost = [&, &TTI = *TTI](const InstructionsState &S,
Instruction *VI) {
InstructionCost Cost = canConvertToFMA(VI, S, *DT, *DL, TTI, *TLI, *this);
Instruction *FMul = getFusableFMulOperand(VI, TTI);
if (!Cost.isValid() || !FMul || isVectorized(FMul))
return Cost;
InstructionCost FMulCost = getUnfusedBinOpCost(FMul, TTI, *TLI, CostKind);
return Cost > FMulCost ? Cost - FMulCost : InstructionCost(TTI::TCC_Free);
};
// The fmul, fused into its fadd/fsub user, is free if the user lane is
// priced as fmuladd, otherwise the user stays scalar and only the fusion
// delta is attributed to the fmul.
auto GetFusedFMulCost = [&, &TTI = *TTI](Instruction *VI) {
auto [U, FMACost] = getFMulFusingUser(VI);
if (!U)
return InstructionCost::getInvalid();
if (isFusedVectorizedFAddSub(U))
return InstructionCost(TTI::TCC_Free);
InstructionCost FAddCost = getUnfusedBinOpCost(U, TTI, *TLI, CostKind);
return FMACost > FAddCost ? FMACost - FAddCost
: InstructionCost(TTI::TCC_Free);
};
switch (ShuffleOrOp) {
case Instruction::PHI: {
// Count reused scalars.
InstructionCost ScalarCost = 0;
SmallPtrSet<const TreeEntry *, 4> CountedOps;
for (PHINode *PHI : make_isa_range<PHINode>(UniqueValues)) {
ValueList Operands(PHI->getNumIncomingValues(), nullptr);
for (unsigned I = 0, N = PHI->getNumIncomingValues(); I < N; ++I) {
Value *Op = PHI->getIncomingValue(I);
Operands[I] = Op;
}
if (const TreeEntry *OpTE =
getSameValuesTreeEntry(Operands.front(), Operands))
if (CountedOps.insert(OpTE).second &&
!OpTE->ReuseShuffleIndices.empty())
ScalarCost += TTI::TCC_Basic * (OpTE->ReuseShuffleIndices.size() -
OpTE->Scalars.size());
}
return CommonCost - ScalarCost + SpillsReloads;
}
case Instruction::ExtractValue:
case Instruction::ExtractElement: {
// For ExtractValue entries vectorized via the struct-call path the scalar
// extractvalue instructions are free (they become extractvalue from a
// struct-of-vectors, which is also free). The cost is fully captured by
// the underlying Call entry and the external-use extraction costs.
if (ShuffleOrOp == Instruction::ExtractValue && !E->StructEVIndices.empty())
return CommonCost;
APInt DemandedElts;
APInt CopyableInsertElts;
VectorType *SrcVecTy = nullptr;
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
// Copyable lanes are not extracts; they are inserted into the reused
// source vector, so charge the insert to the vector side only.
if (E->isCopyableElement(UniqueValues[Idx])) {
if (CopyableInsertElts.isZero())
CopyableInsertElts = APInt::getZero(E->getVectorFactor());
CopyableInsertElts.setBit(Idx);
return InstructionCost(TTI::TCC_Free);
}
auto *I = cast<Instruction>(UniqueValues[Idx]);
// Undef/poison extract indices form holes in the reused source vector,
// such lanes are free.
std::optional<unsigned> ExtIdx = getExtractIndex(I);
if (!ExtIdx)
return InstructionCost(TTI::TCC_Free);
if (!SrcVecTy) {
if (ShuffleOrOp == Instruction::ExtractElement) {
auto *EE = cast<ExtractElementInst>(I);
SrcVecTy = EE->getVectorOperandType();
} else {
auto *EV = cast<ExtractValueInst>(I);
Type *AggregateTy = EV->getAggregateOperand()->getType();
unsigned NumElts;
if (auto *ATy = dyn_cast<ArrayType>(AggregateTy))
NumElts = ATy->getNumElements();
else
NumElts = AggregateTy->getStructNumElements();
SrcVecTy = cast<VectorType>(getWidenedType(OrigScalarTy, NumElts));
}
}
if (I->hasOneUse()) {
Instruction *Ext = I->user_back();
if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
all_of(Ext->users(), IsaPred<GetElementPtrInst>)) {
// Use getExtractWithExtendCost() to calculate the cost of
// extractelement/ext pair.
InstructionCost Cost = TTI->getExtractWithExtendCost(
Ext->getOpcode(), Ext->getType(), SrcVecTy, *ExtIdx, CostKind);
// Subtract the cost of s|zext which is subtracted separately.
Cost -= TTI->getCastInstrCost(
Ext->getOpcode(), Ext->getType(), I->getType(),
TTI::getCastContextHint(Ext), CostKind, Ext);
return Cost;
}
}
// The extract, which the target folds into its users (e.g. the lane
// operand of a by-element fmul on AArch64), costs nothing in the scalar
// code; its removal saves nothing.
if (ShuffleOrOp == Instruction::ExtractElement &&
TTI->getVectorInstrCost(*I, SrcVecTy, CostKind, *ExtIdx,
TTI::getVectorInstrContextHint(I)) == 0)
return InstructionCost(TTI::TCC_Free);
if (DemandedElts.isZero())
DemandedElts = APInt::getZero(getNumElements(SrcVecTy));
DemandedElts.setBit(*ExtIdx);
return InstructionCost(TTI::TCC_Free);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
return CommonCost +
(CopyableInsertElts.isZero()
? TTI::TCC_Free
: TTI.getScalarizationOverhead(
cast<VectorType>(
getWidenedType(OrigScalarTy, E->getVectorFactor())),
CopyableInsertElts, /*Insert=*/true,
/*Extract=*/false, CostKind)) -
(DemandedElts.isZero()
? TTI::TCC_Free
: TTI.getScalarizationOverhead(SrcVecTy, DemandedElts,
/*Insert=*/false,
/*Extract=*/true, CostKind));
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(E->ReuseShuffleIndices.empty() &&
"Unique insertelements only are expected.");
FixedVectorType *SrcVecTy = getInsertBuildVectorSrcTy(E);
unsigned const NumElts = getNumElements(SrcVecTy);
unsigned const NumScalars = VL.size();
unsigned NumOfParts =
getNumberOfParts(SrcVecTy, VL0->getOperand(1)->getType());
SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
unsigned OffsetBeg = *getElementIndex(VL.front());
unsigned OffsetEnd = OffsetBeg;
InsertMask[OffsetBeg] = 0;
for (auto [I, V] : enumerate(VL.drop_front())) {
unsigned Idx = *getElementIndex(V);
if (OffsetBeg > Idx)
OffsetBeg = Idx;
else if (OffsetEnd < Idx)
OffsetEnd = Idx;
InsertMask[Idx] = I + 1;
}
unsigned VecScalarsSz = PowerOf2Ceil(NumElts);
if (NumOfParts > 0 && NumOfParts < NumElts)
VecScalarsSz = PowerOf2Ceil((NumElts + NumOfParts - 1) / NumOfParts);
unsigned VecSz = (1 + OffsetEnd / VecScalarsSz - OffsetBeg / VecScalarsSz) *
VecScalarsSz;
unsigned Offset = VecScalarsSz * (OffsetBeg / VecScalarsSz);
unsigned InsertVecSz = std::min<unsigned>(
PowerOf2Ceil(OffsetEnd - OffsetBeg + 1),
((OffsetEnd - OffsetBeg + VecScalarsSz) / VecScalarsSz) * VecScalarsSz);
bool IsWholeSubvector =
OffsetBeg == Offset && ((OffsetEnd + 1) % VecScalarsSz == 0);
// Check if we can safely insert a subvector. If it is not possible, just
// generate a whole-sized vector and shuffle the source vector and the new
// subvector.
if (OffsetBeg + InsertVecSz > VecSz) {
// Align OffsetBeg to generate correct mask.
OffsetBeg = alignDown(OffsetBeg, VecSz, Offset);
InsertVecSz = VecSz;
}
APInt DemandedElts = APInt::getZero(NumElts);
// TODO: Add support for Instruction::InsertValue.
SmallVector<int> Mask;
if (!E->ReorderIndices.empty()) {
inversePermutation(E->ReorderIndices, Mask);
Mask.append(InsertVecSz - Mask.size(), PoisonMaskElem);
} else {
Mask.assign(VecSz, PoisonMaskElem);
std::iota(Mask.begin(), std::next(Mask.begin(), InsertVecSz), 0);
}
bool IsIdentity = true;
SmallVector<int> PrevMask(InsertVecSz, PoisonMaskElem);
SmallVector<Value *> AdjustedVL(NumElts, PoisonValue::get(ScalarTy));
Mask.swap(PrevMask);
for (unsigned I = 0; I < NumScalars; ++I) {
unsigned InsertIdx = *getElementIndex(VL[PrevMask[I]]);
DemandedElts.setBit(InsertIdx);
AdjustedVL[InsertIdx] = VL[PrevMask[I]];
IsIdentity &= InsertIdx - OffsetBeg == I;
Mask[InsertIdx - OffsetBeg] = I;
}
assert(Offset < NumElts && "Failed to find vector index offset");
InstructionCost Cost = 0;
// First cost - resize to actual vector size if not identity shuffle or
// need to shift the vector.
// Do not calculate the cost if the actual size is the register size and
// we can merge this shuffle with the following SK_Select.
auto *InsertVecTy = cast<VectorType>(getWidenedType(ScalarTy, InsertVecSz));
if (!IsIdentity)
Cost += getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
InsertVecTy, CostKind, Mask);
auto *FirstInsert = cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
}));
// Second cost - permutation with subvector, if some elements are from the
// initial vector or inserting a subvector.
// TODO: Implement the analysis of the FirstInsert->getOperand(0)
// subvector of ActualVecTy.
SmallBitVector InMask =
isUndefVector(FirstInsert->getOperand(0),
buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask));
Cost -= getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, SrcVecTy, DemandedElts,
/*Insert=*/true, /*Extract=*/false, CostKind, InMask.all(), AdjustedVL,
getVectorInstrContextHint(AdjustedVL, DemandedElts));
if (!InMask.all() && NumScalars != NumElts && !IsWholeSubvector) {
if (InsertVecSz != VecSz) {
auto *ActualVecTy = cast<VectorType>(getWidenedType(ScalarTy, VecSz));
Cost += getShuffleCost(*TTI, TTI::SK_InsertSubvector, ActualVecTy,
CostKind, {}, OffsetBeg - Offset, InsertVecTy);
} else {
for (unsigned I = 0, End = OffsetBeg - Offset; I < End; ++I)
Mask[I] = InMask.test(I) ? PoisonMaskElem : I;
for (unsigned I = OffsetBeg - Offset, End = OffsetEnd - Offset;
I <= End; ++I)
if (Mask[I] != PoisonMaskElem)
Mask[I] = I + VecSz;
for (unsigned I = OffsetEnd + 1 - Offset; I < VecSz; ++I)
Mask[I] =
((I >= InMask.size()) || InMask.test(I)) ? PoisonMaskElem : I;
Cost += getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, InsertVecTy,
CostKind, Mask);
}
}
if (ShuffleOrOp == Instruction::InsertValue &&
!canVectorStoreInsertValue(E)) {
// Match the stack slot alignment used during codegen (see the store +
// load roundtrip in vectorizeTree()): the slot holds both the source
// vector and the original aggregate.
Align VecAlign = std::max(DL->getPrefTypeAlign(SrcVecTy),
DL->getPrefTypeAlign(VL0->getType()));
// The stack slot roundtrip is synthesized by codegen; there is no IR
// value or pointer to take operand info from.
Cost += TTI->getMemoryOpCost(Instruction::Store, SrcVecTy, VecAlign,
/*AddressSpace=*/0, CostKind) +
TTI->getMemoryOpCost(Instruction::Load, VL0->getType(), VecAlign,
/*AddressSpace=*/0, CostKind);
}
return Cost + SpillsReloads;
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::PtrToAddr:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
Type *SrcScalarTy = VL0->getOperand(0)->getType();
auto *SrcVecTy = getWidenedType(SrcScalarTy, VL.size());
unsigned Opcode = ShuffleOrOp;
unsigned VecOpcode = Opcode;
if (!ScalarTy->isFPOrFPVectorTy() && !SrcScalarTy->isFPOrFPVectorTy() &&
(SrcIt != MinBWs.end() || It != MinBWs.end())) {
// Check if the values are candidates to demote.
unsigned SrcBWSz = DL->getTypeSizeInBits(SrcScalarTy->getScalarType());
if (SrcIt != MinBWs.end()) {
SrcBWSz = SrcIt->second.first;
unsigned SrcScalarTyNumElements = getNumElements(SrcScalarTy);
SrcScalarTy = IntegerType::get(F->getContext(), SrcBWSz);
SrcVecTy =
getWidenedType(SrcScalarTy, VL.size() * SrcScalarTyNumElements);
}
unsigned BWSz = DL->getTypeSizeInBits(ScalarTy->getScalarType());
if (BWSz == SrcBWSz) {
VecOpcode = Instruction::BitCast;
} else if (BWSz < SrcBWSz) {
VecOpcode = Instruction::Trunc;
} else if (It != MinBWs.end()) {
assert(BWSz > SrcBWSz && "Invalid cast!");
VecOpcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
} else if (SrcIt != MinBWs.end()) {
assert(BWSz > SrcBWSz && "Invalid cast!");
VecOpcode =
SrcIt->second.second ? Instruction::SExt : Instruction::ZExt;
}
} else if (VecOpcode == Instruction::SIToFP && SrcIt != MinBWs.end() &&
!SrcIt->second.second) {
VecOpcode = Instruction::UIToFP;
}
const bool IsSExtBack = VecOpcode == Instruction::UIToFP &&
SrcIt != MinBWs.end() && SrcIt->second.second;
auto GetScalarCost = [&](unsigned Idx) -> InstructionCost {
assert(Idx == 0 && "Expected 0 index only");
return TTI->getCastInstrCost(Opcode, VL0->getType(),
VL0->getOperand(0)->getType(),
TTI::getCastContextHint(VL0), CostKind, VL0);
};
auto GetVectorCost = [=](InstructionCost CommonCost) {
// Do not count cost here if minimum bitwidth is in effect and it is just
// a bitcast (here it is just a noop).
if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast)
return CommonCost;
auto *VI = VL0->getOpcode() == Opcode ? VL0 : nullptr;
TTI::CastContextHint CCH = GetCastContextHint(VL0->getOperand(0));
bool IsArithmeticExtendedReduction =
E->Idx == 0 && UserIgnoreList &&
all_of(*UserIgnoreList, [](Value *V) {
auto *I = cast<Instruction>(V);
return is_contained({Instruction::Add, Instruction::FAdd,
Instruction::Mul, Instruction::FMul,
Instruction::And, Instruction::Or,
Instruction::Xor},
I->getOpcode());
});
if (IsArithmeticExtendedReduction &&
(VecOpcode == Instruction::ZExt || VecOpcode == Instruction::SExt))
return CommonCost;
if (IsSExtBack) {
auto *DemotedVecTy = getWidenedType(
IntegerType::get(F->getContext(), SrcIt->second.first), VL.size());
CommonCost += TTI->getCastInstrCost(Instruction::SExt, SrcVecTy,
DemotedVecTy, CCH, CostKind);
}
return CommonCost +
TTI->getCastInstrCost(VecOpcode, VecTy, SrcVecTy, CCH, CostKind,
VecOpcode == Opcode ? VI : nullptr);
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case Instruction::FCmp:
case Instruction::ICmp:
// Override ScalarTy/VecTy with the compared operand type (not i1). The
// cost of a compare instruction is determined by the operand width, and
// getCmpSelInstrCost expects the compared type as its first type arg.
OrigScalarTy = ScalarTy =
getValueType(VL0, Opts.slp_revec, /*LookThroughCmp=*/true);
VecTy = getWidenedType(ScalarTy, VL.size());
[[fallthrough]];
case Instruction::Select: {
CmpPredicate VecPred, SwappedVecPred;
auto MatchCmp = m_Cmp(VecPred, m_Value(), m_Value());
if (match(VL0, m_Select(MatchCmp, m_Value(), m_Value())) ||
match(VL0, MatchCmp))
SwappedVecPred = CmpInst::getSwappedPredicate(VecPred);
else
SwappedVecPred = VecPred = ScalarTy->isFloatingPointTy()
? CmpInst::BAD_FCMP_PREDICATE
: CmpInst::BAD_ICMP_PREDICATE;
// The bit tests of trunc X to i1 lanes compare and A, B with a constant.
const bool IsBitTest = any_of(E->Scalars, IsaPred<TruncInst>);
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
if (!isa<SelectInst>(UniqueValues[Idx])) {
auto *I = cast<Instruction>(UniqueValues[Idx]);
InstructionCost Cost = TTI->getInstructionCost(I, CostKind);
// The and of the bit test is erased with the compare.
if (IsBitTest && isa<ICmpInst>(I))
Cost += TTI->getInstructionCost(cast<Instruction>(I->getOperand(0)),
CostKind);
return Cost;
}
auto *VI = cast<Instruction>(UniqueValues[Idx]);
CmpPredicate CurrentPred = ScalarTy->isFloatingPointTy()
? CmpInst::BAD_FCMP_PREDICATE
: CmpInst::BAD_ICMP_PREDICATE;
Value *LHS = nullptr, *RHS = nullptr;
auto MatchCmp = m_Cmp(CurrentPred, m_Value(), m_Value());
bool IsSelect =
ShuffleOrOp == Instruction::Select &&
(match(VI, m_Select(MatchCmp, m_Value(LHS), m_Value(RHS))) ||
match(VI, m_Select(m_Value(), m_Value(LHS), m_Value(RHS))));
if ((!IsSelect && !match(VI, MatchCmp)) ||
(CurrentPred != static_cast<CmpInst::Predicate>(VecPred) &&
CurrentPred != static_cast<CmpInst::Predicate>(SwappedVecPred)))
VecPred = SwappedVecPred = ScalarTy->isFloatingPointTy()
? CmpInst::BAD_FCMP_PREDICATE
: CmpInst::BAD_ICMP_PREDICATE;
// Check if operands are of i1 types, like a condition expression.
// TODO: consider implementing this in TTI.
InstructionCost ScalarCost = InstructionCost::getInvalid();
if (IsSelect && LHS->getType() == VI->getOperand(0)->getType()) {
assert(LHS->getType() == RHS->getType() &&
"Expected same type for LHS/RHS");
// select i1 v, i1 true, i1 b -> or i1 v, i1 b
if (match(LHS, m_AllOnes())) {
ScalarCost = TTI->getArithmeticInstrCost(
Instruction::Or, LHS->getType(), CostKind,
TTI::getOperandInfo(VI->getOperand(0)), TTI::getOperandInfo(RHS));
} else if (match(RHS, m_Zero())) {
// select i1 v, i1 b, i1 false -> and i1 v, i1 b
ScalarCost = TTI->getArithmeticInstrCost(
Instruction::And, LHS->getType(), CostKind,
TTI::getOperandInfo(VI->getOperand(0)), TTI::getOperandInfo(LHS));
}
}
if (!ScalarCost.isValid()) {
// For selects, the "condition type" arg is the condition operand's
// type; for standalone compares, it is the result type (i1).
ScalarCost = TTI->getCmpSelInstrCost(
E->getOpcode(), OrigScalarTy,
ShuffleOrOp == Instruction::Select ? VL0->getOperand(0)->getType()
: VL0->getType(),
CurrentPred, CostKind,
TTI::getOperandInfo(
VI->getOperand(ShuffleOrOp == Instruction::Select ? 1 : 0)),
TTI::getOperandInfo(
VI->getOperand(ShuffleOrOp == Instruction::Select ? 2 : 1)),
VI);
}
InstructionCost IntrinsicCost = GetMinMaxCost(OrigScalarTy, VI);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
return ScalarCost;
};
auto GetVectorCost = [&](InstructionCost CommonCost) {
// An absorbed boolean zero-test leaf emits no conversion; only the
// reorder shuffle (in CommonCost) may remain.
if (isBooleanizedNode(E))
return CommonCost;
// For selects, the condition type may differ from the result type
// (e.g. condition is <N x i1> while result is <N x i32>). For
// compares, the result type IS the mask (i1/vNi1). Construct the
// right type so getCmpSelInstrCost sees the actual mask/result width.
auto *MaskTy = cast<VectorType>(getWidenedType(
ShuffleOrOp == Instruction::Select ? VL0->getOperand(0)->getType()
: VL0->getType(),
VL.size()));
InstructionCost VecCost = InstructionCost::getInvalid();
if (ShuffleOrOp == Instruction::Select) {
ArrayRef<Value *> Cond = E->getOperand(0);
ArrayRef<Value *> LHS = E->getOperand(1);
ArrayRef<Value *> RHS = E->getOperand(2);
// select <VF x i1>, <VF x i1>, <VF x i1>?
// TODO: consider implementing this in TTI.
if (Cond.front()->getType() == LHS.front()->getType()) {
// select <VF x i1> v, <VF x i1> true, <VF x i1> b -> or <VF x i1> v,
// <VF x i1> b
if (all_of(LHS, [&](Value *V) { return match(V, m_AllOnes()); })) {
VecCost = TTI->getArithmeticInstrCost(
Instruction::Or, VecTy, CostKind, TTI::getOperandInfo(Cond),
TTI::getOperandInfo(RHS));
} else if (all_of(RHS,
[&](Value *V) { return match(V, m_Zero()); })) {
// select <VF x i1> v, <VF x i1> b, <VF x i1> false -> and <VF x i1>
// v, <VF x i1> b
VecCost = TTI->getArithmeticInstrCost(
Instruction::And, VecTy, CostKind, TTI::getOperandInfo(Cond),
TTI::getOperandInfo(LHS));
}
}
}
if (!VecCost.isValid()) {
VecCost = TTI->getCmpSelInstrCost(
E->getOpcode(), VecTy, MaskTy, VecPred, CostKind,
TTI::getOperandInfo(
E->getOperand(ShuffleOrOp == Instruction::Select ? 1 : 0)),
TTI::getOperandInfo(
E->getOperand(ShuffleOrOp == Instruction::Select ? 2 : 1)),
VL0);
if (IsBitTest)
VecCost +=
TTI->getArithmeticInstrCost(Instruction::And, VecTy, CostKind);
if (isa<SelectInst>(VL0)) {
unsigned CondNumElements = getNumElements(MaskTy);
unsigned VecTyNumElements = getNumElements(VecTy);
assert(VecTyNumElements >= CondNumElements &&
VecTyNumElements % CondNumElements == 0 &&
"Cannot vectorize Instruction::Select");
if (CondNumElements != VecTyNumElements) {
// When the return type is i1 but the source is fixed vector type,
// we need to duplicate the condition value.
VecCost += getShuffleCost(
*TTI, TTI::SK_PermuteSingleSrc, MaskTy, CostKind,
createReplicatedMask(VecTyNumElements / CondNumElements,
CondNumElements));
}
}
}
return VecCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::MinMax: {
auto GetScalarCost = [&](unsigned Idx) {
return GetMinMaxCost(OrigScalarTy);
};
auto GetVectorCost = [&](InstructionCost CommonCost) {
InstructionCost VecCost = GetMinMaxCost(VecTy);
return VecCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::FMulAdd: {
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *I = cast<Instruction>(UniqueValues[Idx]);
InstructionCost Cost = GetFMulAddCost(E->getOperations(), I);
if (Cost.isValid())
return Cost;
// The lanes without the fmul, e.g. the constant lanes of the fmul node,
// stay plain fadd/fsub.
unsigned Lane = UniqueIndexes[Idx];
Value *Op1 = E->getOperand(0)[Lane];
Value *Op2 = E->getOperand(1)[Lane];
return TTI->getArithmeticInstrCost(
E->getOpcode(), OrigScalarTy, CostKind, TTI::getOperandInfo(Op1),
TTI::getOperandInfo(Op2), {Op1, Op2}, I, TLI);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
FastMathFlags FMF;
FMF.set();
for (FPMathOperator *FPCI : make_isa_range<FPMathOperator>(E->Scalars)) {
FMF &= FPCI->getFastMathFlags();
// The fused fmul may sit in either operand (c - a*b included).
if (Instruction *FMul =
getFusableFMulOperand(cast<Instruction>(FPCI), TTI))
FMF &= FMul->getFastMathFlags();
}
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, VecTy,
{VecTy, VecTy, VecTy}, FMF);
InstructionCost VecCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
return VecCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::ReducedBitcast:
case TreeEntry::ReducedBitcastBSwap: {
auto GetScalarCost = [&, &TTI = *TTI](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *Shl = dyn_cast<Instruction>(UniqueValues[Idx]);
if (!Shl)
return InstructionCost(TTI::TCC_Free);
InstructionCost ScalarCost = TTI.getInstructionCost(Shl, CostKind);
auto *ZExt = dyn_cast<Instruction>(Shl->getOperand(0));
if (!ZExt)
return ScalarCost;
ScalarCost += TTI.getInstructionCost(ZExt, CostKind);
return ScalarCost;
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
const TreeEntry *LhsTE = getOperandEntry(E, /*Idx=*/0);
TTI::CastContextHint CastCtx =
getCastContextHint(*getOperandEntry(LhsTE, /*Idx=*/0));
Type *SrcScalarTy = cast<ZExtInst>(LhsTE->getMainOp())->getSrcTy();
auto *SrcVecTy = getWidenedType(SrcScalarTy, LhsTE->getVectorFactor());
InstructionCost BitcastCost = TTI.getCastInstrCost(
Instruction::BitCast, ScalarTy, SrcVecTy, CastCtx, CostKind);
if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwap) {
auto *SrcType = IntegerType::getIntNTy(
ScalarTy->getContext(),
DL->getTypeSizeInBits(SrcScalarTy) * EntryVF);
IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, SrcType, {SrcType});
InstructionCost IntrinsicCost =
TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
BitcastCost += IntrinsicCost;
if (SrcType != ScalarTy) {
BitcastCost += TTI.getCastInstrCost(Instruction::ZExt, ScalarTy,
SrcType, CastCtx, CostKind);
}
}
return BitcastCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::ReducedBitcastLoads:
case TreeEntry::ReducedBitcastBSwapLoads: {
auto GetScalarCost = [&, &TTI = *TTI](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *Shl = dyn_cast<Instruction>(UniqueValues[Idx]);
if (!Shl)
return InstructionCost(TTI::TCC_Free);
InstructionCost ScalarCost = TTI.getInstructionCost(Shl, CostKind);
auto *ZExt = dyn_cast<Instruction>(Shl->getOperand(0));
if (!ZExt)
return ScalarCost;
ScalarCost += TTI.getInstructionCost(ZExt, CostKind);
auto *Load = dyn_cast<Instruction>(ZExt->getOperand(0));
if (!Load)
return ScalarCost;
ScalarCost += TTI.getInstructionCost(Load, CostKind);
return ScalarCost;
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
const TreeEntry *LhsTE = getOperandEntry(E, /*Idx=*/0);
const TreeEntry *LoadTE = getOperandEntry(LhsTE, /*Idx=*/0);
auto *LI0 = cast<LoadInst>(LoadTE->getMainOp());
auto *SrcType = IntegerType::getIntNTy(
ScalarTy->getContext(),
DL->getTypeSizeInBits(LI0->getType()) * EntryVF);
InstructionCost LoadCost =
TTI.getMemoryOpCost(Instruction::Load, SrcType, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind);
if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwapLoads) {
IntrinsicCostAttributes CostAttrs(Intrinsic::bswap, SrcType, {SrcType});
InstructionCost IntrinsicCost =
TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
LoadCost += IntrinsicCost;
if (SrcType != ScalarTy) {
LoadCost +=
TTI.getCastInstrCost(Instruction::ZExt, ScalarTy, SrcType,
getCastContextHint(*LoadTE), CostKind);
}
}
return LoadCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::ReducedBitPack: {
const TreeEntry *ShlTE =
E->getOpcode() == Instruction::And ? getOperandEntry(E, /*Idx=*/0) : E;
const TreeEntry *LhsTE = getOperandEntry(ShlTE, /*Idx=*/0);
std::optional<BitPackInfo> Info = computeBitPackLayout(*E);
const TreeEntry *SrcTE = Info ? getBitPackSource(*LhsTE, *Info) : LhsTE;
// The scalars of the root, the shl and the combined chain are removed.
SmallVector<const TreeEntry *> CombinedTEs = {E};
if (ShlTE != E)
CombinedTEs.push_back(ShlTE);
for (const TreeEntry *TE = LhsTE; TE != SrcTE;
TE = getOperandEntry(TE, /*Idx=*/0))
CombinedTEs.push_back(TE);
auto GetLaneCost = [&, &TTI = *TTI](unsigned Lane) {
InstructionCost ScalarCost = 0;
for (const TreeEntry *TE : CombinedTEs) {
auto *I = dyn_cast<Instruction>(TE->Scalars[Lane]);
// A copyable lane is the operand of the node, costed with its node.
if (!I || TE->isCopyableElement(I))
continue;
if (I->isCast()) {
ScalarCost += TTI.getCastInstrCost(
I->getOpcode(), I->getType(), I->getOperand(0)->getType(),
TTI::getCastContextHint(I), CostKind, I);
continue;
}
assert(is_contained(
{Instruction::And, Instruction::LShr, Instruction::Shl},
I->getOpcode()) &&
"Expected and/shift in the bit pack.");
ScalarCost += TTI.getArithmeticInstrCost(
I->getOpcode(), I->getType(), CostKind,
TTI::getOperandInfo(I->getOperand(0)),
TTI::getOperandInfo(I->getOperand(1)),
{I->getOperand(0), I->getOperand(1)}, I, TLI);
}
return ScalarCost;
};
auto GetScalarCost = [&](unsigned Idx) {
return GetLaneCost(UniqueIndexes[Idx]);
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
if (!Info)
return InstructionCost::getInvalid();
unsigned ShiftWidth;
// The pack works on the vectorized operand in its own (possibly
// demoted) type.
Type *SrcScalarTy = SrcTE->Scalars.front()->getType();
if (auto It = MinBWs.find(SrcTE); It != MinBWs.end())
SrcScalarTy = IntegerType::get(F->getContext(), It->second.first);
// The source lanes are packed into the node type.
Type *DstScalarTy = SrcTE == LhsTE ? SrcScalarTy : ScalarTy;
InstructionCost PackCost =
getBitPackCost(TTI,
cast<FixedVectorType>(getWidenedType(
SrcScalarTy, SrcTE->getVectorFactor())),
DstScalarTy, *Info, getZExtSrcWidth(*SrcTE),
getCastContextHint(*SrcTE), CostKind, TLI,
E->getMainOp(), ShiftWidth);
// A demoted result is re-extended to the reduction type.
if (DstScalarTy != OrigScalarTy) {
auto It = MinBWs.find(E);
PackCost += TTI.getCastInstrCost(
It != MinBWs.end() && It->second.second ? Instruction::SExt
: Instruction::ZExt,
OrigScalarTy, DstScalarTy, getCastContextHint(*LhsTE), CostKind);
}
// The copyable lanes are not the scalars of the node, but their scalars
// in the shl and the combined chain are removed too.
for (auto [Lane, V] : enumerate(E->Scalars))
if (E->isCopyableElement(V))
PackCost -= GetLaneCost(Lane);
return PackCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case TreeEntry::ReducedCmpBitcast: {
auto GetScalarCost = [&, &TTI = *TTI](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *Sel = dyn_cast<Instruction>(UniqueValues[Idx]);
if (!Sel)
return InstructionCost(TTI::TCC_Free);
InstructionCost ScalarCost = TTI.getInstructionCost(Sel, CostKind);
return ScalarCost;
};
auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
Type *CmpTy = CmpInst::makeCmpResultType(VecTy);
auto *DstTy =
IntegerType::getIntNTy(ScalarTy->getContext(), E->getVectorFactor());
InstructionCost BitcastCost =
TTI.getCastInstrCost(Instruction::BitCast, DstTy, CmpTy,
TTI::CastContextHint::None, CostKind);
if (DstTy != ScalarTy) {
BitcastCost +=
TTI.getCastInstrCost(Instruction::ZExt, ScalarTy, DstTy,
TTI::CastContextHint::None, CostKind);
}
return BitcastCost + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case Instruction::FNeg:
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
// We cannot retrieve the operand from UniqueValues[Idx] because an
// interchangeable instruction may be used. The order and the actual
// operand might differ from what is retrieved from UniqueValues[Idx].
unsigned Lane = UniqueIndexes[Idx];
Value *Op1 = E->getOperand(0)[Lane];
Value *Op2;
SmallVector<const Value *, 2> Operands(1, Op1);
if (isa<UnaryOperator>(UniqueValues[Idx])) {
Op2 = Op1;
} else {
Op2 = E->getOperand(1)[Lane];
Operands.push_back(Op2);
}
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(Op1);
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(Op2);
auto *I = dyn_cast<Instruction>(UniqueValues[Idx]);
InstructionCost ScalarCost = TTI->getArithmeticInstrCost(
ShuffleOrOp, OrigScalarTy, CostKind, Op1Info, Op2Info, Operands,
I && I->getOpcode() == ShuffleOrOp ? I : nullptr);
if (I && (ShuffleOrOp == Instruction::FAdd ||
ShuffleOrOp == Instruction::FSub)) {
InstructionCost IntrinsicCost = GetFMulAddCost(E->getOperations(), I);
if (IntrinsicCost.isValid())
ScalarCost = IntrinsicCost;
} else if (I && ShuffleOrOp == Instruction::FMul) {
InstructionCost FusedCost = GetFusedFMulCost(I);
if (FusedCost.isValid())
ScalarCost = FusedCost;
}
return ScalarCost;
};
auto GetVectorCost = [=](InstructionCost CommonCost) {
// And peephole only applies to plain 2-operand nodes.
if (ShuffleOrOp == Instruction::And && It != MinBWs.end() &&
!E->hasReassocScalars()) {
for (unsigned I : seq<unsigned>(0, E->getNumOperands())) {
ArrayRef<Value *> Ops = E->getOperand(I);
if (all_of(Ops, [&](Value *Op) {
auto *CI = dyn_cast<ConstantInt>(Op);
return CI && CI->getValue().countr_one() >= It->second.first;
}))
return CommonCost;
}
}
// Masked path ignores Op1Info/Op2Info like its codegen does; keep it
// out of the operand-aware cost comparison below.
if (InstructionCost MaskedCost = getMaskedDivRemCost(
*TTI, Opts.slp_revec, ShuffleOrOp, ScalarTy, VL.size(), CostKind);
MaskedCost.isValid())
return MaskedCost + CommonCost;
unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(E->getOperand(OpIdx));
InstructionCost Cost = TTI->getArithmeticInstrCost(
ShuffleOrOp, VecTy, CostKind, Op1Info, Op2Info, {},
VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
// N columns need N-1 vector combines; price extra columns
// conservatively, skipping identity-only columns (not combined by
// codegen).
if (E->hasReassocScalars()) {
const unsigned CombineOpcode = getReassocCombineOpcode(E->getOpcode());
for (unsigned Idx : seq<unsigned>(2, E->getNumOperands())) {
if (all_of(E->getOperand(Idx), [&](Value *V) {
return isBinOpIdentityConstant(V, CombineOpcode);
}))
continue;
// The first operand is the running fold, a computed value with no
// static properties to model.
Cost += TTI->getArithmeticInstrCost(
ShuffleOrOp, VecTy, CostKind, {},
TTI::getOperandInfo(E->getOperand(Idx)), {},
VL0->getOpcode() == ShuffleOrOp ? VL0 : nullptr, TLI);
}
}
return Cost + CommonCost;
};
return GetCostDiffWithPeeled(GetScalarCost, GetVectorCost);
}
case Instruction::GetElementPtr: {
return CommonCost + GetGEPCostDiff(VL, VL0);
}
case Instruction::Load: {
auto GetScalarCost = [&](unsigned Idx) {
auto *VI = cast<LoadInst>(UniqueValues[Idx]);
return TTI->getMemoryOpCost(
Instruction::Load, OrigScalarTy, VI->getAlign(),
VI->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(VI->getPointerOperand()), VI);
};
auto *LI0 = cast<LoadInst>(VL0);
auto GetVectorCost = [&](InstructionCost CommonCost) {
InstructionCost VecLdCost;
switch (E->State) {
case TreeEntry::Vectorize:
if (unsigned Factor = E->getInterleaveFactor()) {
VecLdCost = TTI->getInterleavedMemoryOpCost(
Instruction::Load, VecTy, Factor, {}, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind);
} else {
VecLdCost = TTI->getMemoryOpCost(
Instruction::Load, VecTy,
commonAlignment(LI0->getAlign(), getVectorLoadShift(E->Scalars)),
LI0->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI0->getPointerOperand()));
}
break;
case TreeEntry::StridedVectorize: {
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *StridedLoadTy = SPtrInfo.Ty;
assert(StridedLoadTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
bool IsReverse = E->isReverse() && !isWidenedStridedLoad(E);
Value *Stride = getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy,
*DL, IsReverse);
VecLdCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_load,
StridedLoadTy, LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment,
/*I=*/nullptr, Stride),
CostKind);
if (StridedLoadTy != VecTy)
VecLdCost +=
getWidenedStridedCastCost(*TTI, StridedLoadTy, VecTy, *DL,
getCastContextHint(*E), CostKind);
break;
}
case TreeEntry::CompressVectorize: {
bool IsMasked;
unsigned InterleaveFactor;
SmallVector<int> CompressMask;
VectorType *LoadVecTy;
SmallVector<Value *> Scalars(VL);
if (!E->ReorderIndices.empty()) {
SmallVector<int> Mask(E->ReorderIndices.begin(),
E->ReorderIndices.end());
reorderScalars(Scalars, Mask);
}
SmallVector<Value *> PointerOps(Scalars.size());
for (auto [I, V] : enumerate(Scalars))
PointerOps[I] = cast<LoadInst>(V)->getPointerOperand();
[[maybe_unused]] bool IsVectorized = isMaskedLoadCompress(
Scalars, PointerOps, E->ReorderIndices, *TTI, *DL, *SE, *AC, *DT,
*TLI, CostKind, [](Value *) { return true; }, Opts.slp_revec,
IsMasked, InterleaveFactor, CompressMask, LoadVecTy, Cache);
CompressEntryToData.try_emplace(E, CompressMask, LoadVecTy,
InterleaveFactor, IsMasked);
Align CommonAlignment = LI0->getAlign();
// For REVEC the emitted shuffle uses the compress mask expanded to the
// individual elements of the vector scalars.
SmallVector<int> CompressCostMask(CompressMask);
transformScalarShuffleIndiciesToVector(getNumElements(LI0->getType()),
CompressCostMask);
if (InterleaveFactor) {
VecLdCost = TTI->getInterleavedMemoryOpCost(
Instruction::Load, LoadVecTy, InterleaveFactor, {},
CommonAlignment, LI0->getPointerAddressSpace(), CostKind);
} else if (IsMasked) {
VecLdCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_load, LoadVecTy,
CommonAlignment,
LI0->getPointerAddressSpace()),
CostKind);
// TODO: include this cost into CommonCost.
VecLdCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, LoadVecTy,
CostKind, CompressCostMask);
} else {
VecLdCost = TTI->getMemoryOpCost(
Instruction::Load, LoadVecTy, CommonAlignment,
LI0->getPointerAddressSpace(), CostKind,
TTI::getOperandInfo(LI0->getPointerOperand()));
// TODO: include this cost into CommonCost.
VecLdCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, LoadVecTy,
CostKind, CompressCostMask);
}
break;
}
case TreeEntry::ScatterVectorize: {
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
VecLdCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_gather, VecTy,
LI0->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment),
CostKind);
break;
}
case TreeEntry::BlendedLoadVectorize: {
// Two masked loads (one per candidate base) blended by a select.
Align CommonAlignment =
computeBlendedLoadBaseAlignment(UniqueValues.getArrayRef(), *DL);
VecLdCost = getBlendedLoadCost(*TTI, VecTy, CommonAlignment,
LI0->getPointerAddressSpace(), CostKind);
break;
}
case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
case TreeEntry::NeedToGather:
llvm_unreachable("Unexpected vectorization state.");
}
return VecLdCost + CommonCost;
};
InstructionCost Cost = GetCostDiff(GetScalarCost, GetVectorCost);
// Masked gather and blended loads are not terminal nodes: their address
// cost is estimated separately (blended loads have no per-lane address).
if (E->State == TreeEntry::ScatterVectorize ||
E->State == TreeEntry::BlendedLoadVectorize)
return Cost;
// Estimate cost of GEPs since this tree node is a terminator.
SmallVector<Value *> PointerOps;
for (LoadInst *LI : make_isa_range<LoadInst>(VL))
PointerOps.push_back(LI->getPointerOperand());
return Cost + GetGEPCostDiff(PointerOps, LI0->getPointerOperand());
}
case Instruction::Store: {
bool IsReorder = !E->ReorderIndices.empty();
auto GetScalarCost = [=](unsigned Idx) {
auto *VI = cast<StoreInst>(VL[Idx]);
TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(VI->getValueOperand());
return TTI->getMemoryOpCost(Instruction::Store, OrigScalarTy,
VI->getAlign(), VI->getPointerAddressSpace(),
CostKind, OpInfo, VI);
};
auto *BaseSI =
cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
auto GetVectorCost = [=](InstructionCost CommonCost) {
// We know that we can merge the stores. Calculate the cost.
InstructionCost VecStCost;
if (E->State == TreeEntry::StridedVectorize) {
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
Value *Stride =
getStrideBytesIfConstant(SPtrInfo.StrideVal, ScalarTy, *DL);
VecStCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::experimental_vp_strided_store,
StridedStoreTy,
BaseSI->getPointerOperand(),
/*VariableMask=*/false, CommonAlignment,
/*I=*/nullptr, Stride),
CostKind);
if (StridedStoreTy != VecTy)
VecStCost +=
getWidenedStridedCastCost(*TTI, VecTy, StridedStoreTy, *DL,
getCastContextHint(*E), CostKind);
} else if (E->State == TreeEntry::ExpandVectorize) {
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *MaskedStoreTy = SPtrInfo.Ty;
assert(MaskedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment =
computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
// Masked store: the values are expanded into the widened vector and
// stored with a constant mask.
VecStCost = TTI->getMemIntrinsicInstrCost(
MemIntrinsicCostAttributes(Intrinsic::masked_store, MaskedStoreTy,
CommonAlignment,
BaseSI->getPointerAddressSpace()),
CostKind);
} else {
assert(E->State == TreeEntry::Vectorize &&
"Expected either strided, consecutive, or expanded stores.");
if (unsigned Factor = E->getInterleaveFactor()) {
assert(E->ReuseShuffleIndices.empty() && !E->ReorderIndices.empty() &&
"No reused shuffles expected");
CommonCost = 0;
VecStCost = TTI->getInterleavedMemoryOpCost(
Instruction::Store, VecTy, Factor, {}, BaseSI->getAlign(),
BaseSI->getPointerAddressSpace(), CostKind);
} else {
TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(E->getOperand(0));
VecStCost = TTI->getMemoryOpCost(
Instruction::Store, VecTy, BaseSI->getAlign(),
BaseSI->getPointerAddressSpace(), CostKind, OpInfo);
}
}
return VecStCost + CommonCost;
};
SmallVector<Value *> PointerOps(VL.size());
for (auto [I, V] : enumerate(VL)) {
unsigned Idx = IsReorder ? E->ReorderIndices[I] : I;
PointerOps[Idx] = cast<StoreInst>(V)->getPointerOperand();
}
return GetCostDiff(GetScalarCost, GetVectorCost) +
GetGEPCostDiff(PointerOps, BaseSI->getPointerOperand());
}
case Instruction::Call: {
auto GetScalarCost = [&](unsigned Idx) {
// A copyable lane has no scalar call of its own; its real cost is
// accounted for wherever it is otherwise computed.
if (isa<PoisonValue>(UniqueValues[Idx]) ||
E->isCopyableElement(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *CI = cast<CallInst>(UniqueValues[Idx]);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
if (ID != Intrinsic::not_intrinsic) {
IntrinsicCostAttributes CostAttrs(ID, *CI, 1);
return TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
}
return TTI->getCallInstrCost(CI->getCalledFunction(),
CI->getFunctionType()->getReturnType(),
CI->getFunctionType()->params(), CostKind);
};
auto GetVectorCost = [=](InstructionCost CommonCost) {
auto *CI = cast<CallInst>(VL0);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
SmallVector<Type *> ArgTys = buildIntrinsicArgTypes(
CI, ID, getNumElements(VecTy),
It != MinBWs.end() ? It->second.first : 0, TTI);
auto VecCallCosts =
getVectorCallCosts(CI, VecTy, TTI, TLI, ArgTys, CostKind);
return std::min(VecCallCosts.first, VecCallCosts.second) + CommonCost;
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
case Instruction::ShuffleVector: {
if (!Opts.slp_revec || E->isAltShuffle())
assert(E->isAltShuffle() &&
((Instruction::isBinaryOp(E->getOpcode()) &&
Instruction::isBinaryOp(E->getAltOpcode())) ||
(Instruction::isCast(E->getOpcode()) &&
Instruction::isCast(E->getAltOpcode())) ||
(isa<CmpInst>(VL0) && isa<CmpInst>(E->getAltOp()))) &&
"Invalid Shuffle Vector Operand");
// Try to find the previous shuffle node with the same operands and same
// main/alternate ops.
auto TryFindNodeWithEqualOperands = [=]() {
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (TE.get() == E)
break;
if (TE->hasState() && TE->isAltShuffle() &&
((TE->getOpcode() == E->getOpcode() &&
TE->getAltOpcode() == E->getAltOpcode()) ||
(TE->getOpcode() == E->getAltOpcode() &&
TE->getAltOpcode() == E->getOpcode())) &&
TE->hasEqualOperands(*E))
return true;
}
return false;
};
auto GetScalarCost = [&](unsigned Idx) {
if (isa<PoisonValue>(UniqueValues[Idx]))
return InstructionCost(TTI::TCC_Free);
auto *VI = cast<Instruction>(UniqueValues[Idx]);
assert(E->getMatchingMainOpOrAltOp(VI) &&
"Unexpected main/alternate opcode");
(void)E;
if (auto *CI = dyn_cast<CmpInst>(VI))
return TTI->getCmpSelInstrCost(
CI->getOpcode(), CI->getOperand(0)->getType(), CI->getType(),
CI->getPredicate(), CostKind,
TTI::getOperandInfo(CI->getOperand(0)),
TTI::getOperandInfo(CI->getOperand(1)), CI);
if (auto *CI = dyn_cast<CastInst>(VI))
return TTI->getCastInstrCost(CI->getOpcode(), CI->getDestTy(),
CI->getSrcTy(),
TTI::getCastContextHint(CI), CostKind, CI);
if (auto *SV = dyn_cast<ShuffleVectorInst>(VI)) {
int Index;
[[maybe_unused]] bool IsExtractSubvectorMask =
SV->isExtractSubvectorMask(Index);
assert(IsExtractSubvectorMask && "Not supported shufflevector usage.");
auto *SubTy = cast<VectorType>(SV->getType());
return TTI->getShuffleCost(
TTI::SK_ExtractSubvector, SubTy,
cast<VectorType>(SV->getOperand(0)->getType()), CostKind,
SV->getShuffleMask(), Index, SubTy,
{SV->getOperand(0), SV->getOperand(1)}, SV,
TTI::getVectorInstrContextHint(SV));
}
// The backend fuses fmul+fadd/fsub pairs in the scalar code.
InstructionCost FusedCost = InstructionCost::getInvalid();
if (match(VI, m_CombineOr(m_FAdd(m_Value(), m_Value()),
m_FSub(m_Value(), m_Value()))))
FusedCost = GetFMulAddCost(InstructionsState(VI, VI), VI);
else if (match(VI, m_FMul(m_Value(), m_Value())))
FusedCost = GetFusedFMulCost(VI);
if (FusedCost.isValid())
return FusedCost;
return TTI->getArithmeticInstrCost(
VI->getOpcode(), VI->getType(), CostKind,
TTI::getOperandInfo(VI->getOperand(0)),
TTI::getOperandInfo(VI->getOperand(1)),
{VI->getOperand(0), VI->getOperand(1)}, VI, TLI);
};
// Need to clear CommonCost since the final shuffle cost is included into
// vector cost.
auto GetVectorCost = [&, &TTIRef = *TTI](InstructionCost) {
// VecCost is equal to sum of the cost of creating 2 vectors
// and the cost of creating shuffle.
InstructionCost VecCost = 0;
if (TryFindNodeWithEqualOperands()) {
LLVM_DEBUG({
dbgs() << "SLP: diamond match for alternate node found.\n";
E->dump();
});
// No need to add new vector costs here since we're going to reuse
// same main/alternate vector ops, just do different shuffling.
} else if (Instruction::isBinaryOp(E->getOpcode())) {
// Peeled alternate chains fold the operand columns into one pure
// main-opcode chain and one pure alt-opcode chain, followed by a
// single lane-select shuffle; a plain alternate node is a single
// combine. Each combine is priced with the properties of the column
// it folds in; the other operand is the running fold, which stays
// constant while every folded column is constant (such combines
// constant-fold away in codegen) and stays uniform while every
// folded column is uniform.
auto ChainCost = [&](unsigned Opcode) {
InstructionCost Cost = 0;
TTI::OperandValueInfo RunningInfo =
TTI::getOperandInfo(E->getOperand(0));
for (unsigned Idx : seq<unsigned>(1, E->getNumOperands())) {
TTI::OperandValueInfo ColInfo =
TTI::getOperandInfo(E->getOperand(Idx));
if (!RunningInfo.isConstant() || !ColInfo.isConstant())
Cost += TTIRef.getArithmeticInstrCost(
Opcode, VecTy, CostKind, RunningInfo, ColInfo, {},
Opcode == E->getOpcode() ? E->getMainOp() : E->getAltOp(),
TLI);
TTI::OperandValueKind Kind = TTI::OK_AnyValue;
if (RunningInfo.isConstant() && ColInfo.isConstant())
Kind = RunningInfo.Kind == TTI::OK_UniformConstantValue &&
ColInfo.Kind == TTI::OK_UniformConstantValue
? TTI::OK_UniformConstantValue
: TTI::OK_NonUniformConstantValue;
else if (RunningInfo.isUniform() && ColInfo.isUniform())
Kind = TTI::OK_UniformValue;
RunningInfo = {Kind, TTI::OP_None};
}
return Cost;
};
VecCost = ChainCost(E->getOpcode()) + ChainCost(E->getAltOpcode());
} else if (auto *CI0 = dyn_cast<CmpInst>(VL0)) {
auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
TTI::OperandValueInfo Op0Info = TTI::getOperandInfo(E->getOperand(0));
TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(E->getOperand(1));
VecCost = TTIRef.getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy,
CI0->getPredicate(), CostKind,
Op0Info, Op1Info, VL0);
VecCost += TTIRef.getCmpSelInstrCost(
E->getOpcode(), VecTy, MaskTy,
cast<CmpInst>(E->getAltOp())->getPredicate(), CostKind, Op0Info,
Op1Info, E->getAltOp());
} else {
Type *SrcSclTy = E->getMainOp()->getOperand(0)->getType();
auto *SrcTy = getWidenedType(SrcSclTy, VL.size());
if (SrcSclTy->isIntegerTy() && ScalarTy->isIntegerTy()) {
auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
unsigned SrcBWSz =
DL->getTypeSizeInBits(E->getMainOp()->getOperand(0)->getType());
if (SrcIt != MinBWs.end()) {
SrcBWSz = SrcIt->second.first;
SrcSclTy = IntegerType::get(SrcSclTy->getContext(), SrcBWSz);
SrcTy = getWidenedType(SrcSclTy, VL.size());
}
if (BWSz <= SrcBWSz) {
if (BWSz < SrcBWSz)
VecCost = TTIRef.getCastInstrCost(
Instruction::Trunc, VecTy, SrcTy,
GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind);
LLVM_DEBUG({
dbgs()
<< "SLP: alternate extension, which should be truncated.\n";
E->dump();
});
return VecCost;
}
}
VecCost = TTIRef.getCastInstrCost(
E->getOpcode(), VecTy, SrcTy,
GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind,
E->getMainOp());
VecCost += TTIRef.getCastInstrCost(
E->getAltOpcode(), VecTy, SrcTy,
GetCastContextHint(E->getMainOp()->getOperand(0)), CostKind,
E->getAltOp());
}
SmallVector<int> Mask;
E->buildAltOpShuffleMask(
[&](Instruction *I) {
assert(E->getMatchingMainOpOrAltOp(I) &&
"Unexpected main/alternate opcode");
return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),
*TLI);
},
Mask);
VecCost += getShuffleCost(TTIRef, TargetTransformInfo::SK_PermuteTwoSrc,
cast<VectorType>(FinalVecTy), CostKind, Mask);
// Patterns like [fadd,fsub] can be combined into a single instruction
// in x86. Reordering them into [fsub,fadd] blocks this pattern. So we
// need to take into account their order when looking for the most used
// order. Linearized chains emit no alternate-ops pattern.
if (E->hasReassocScalars())
return VecCost;
unsigned Opcode0 = E->getOpcode();
unsigned Opcode1 = E->getAltOpcode();
SmallBitVector OpcodeMask(
getAltInstrMask(E->Scalars, ScalarTy, Opcode0, Opcode1));
// If this pattern is supported by the target then we consider the
// order.
if (TTIRef.isLegalAltInstr(cast<VectorType>(VecTy), Opcode0, Opcode1,
OpcodeMask, E->Scalars)) {
InstructionCost AltVecCost =
TTIRef.getAltInstrCost(cast<VectorType>(VecTy), Opcode0, Opcode1,
OpcodeMask, CostKind, E->Scalars);
return AltVecCost < VecCost ? AltVecCost : VecCost;
}
// TODO: Check the reverse order too.
return VecCost;
};
if (Opts.slp_revec && !E->isAltShuffle())
return GetCostDiff(
GetScalarCost, [&](InstructionCost) -> InstructionCost {
// If a group uses mask in order, the shufflevector can be
// eliminated by instcombine. Then the cost is 0.
assert(isa<ShuffleVectorInst>(VL.front()) &&
"Not supported shufflevector usage.");
auto *SV = cast<ShuffleVectorInst>(VL.front());
unsigned SVNumElements =
cast<FixedVectorType>(SV->getOperand(0)->getType())
->getNumElements();
unsigned GroupSize = SVNumElements / SV->getShuffleMask().size();
for (size_t I = 0, End = VL.size(); I != End; I += GroupSize) {
ArrayRef<Value *> Group = VL.slice(I, GroupSize);
int NextIndex = 0;
if (!all_of(Group, [&](Value *V) {
assert(isa<ShuffleVectorInst>(V) &&
"Not supported shufflevector usage.");
auto *SV = cast<ShuffleVectorInst>(V);
int Index;
[[maybe_unused]] bool IsExtractSubvectorMask =
SV->isExtractSubvectorMask(Index);
assert(IsExtractSubvectorMask &&
"Not supported shufflevector usage.");
if (NextIndex != Index)
return false;
NextIndex += SV->getShuffleMask().size();
return true;
}))
return getShuffleCost(*TTI,
TargetTransformInfo::SK_PermuteSingleSrc,
cast<VectorType>(VecTy), CostKind,
calculateShufflevectorMask(E->Scalars));
}
return TTI::TCC_Free;
});
return GetCostDiffWithPeeled(GetScalarCost, GetVectorCost);
}
case Instruction::Freeze:
return CommonCost;
default:
llvm_unreachable("Unknown instruction");
}
}
bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n");
auto &&AreVectorizableGathers = [this](const TreeEntry *TE, unsigned Limit) {
SmallVector<int> Mask;
return TE->isGather() &&
!any_of(TE->Scalars,
[this](Value *V) { return EphValues.contains(V); }) &&
(allConstant(TE->Scalars) || isSplat(TE->Scalars) ||
TE->Scalars.size() < Limit ||
matchGatheredExtractedFields(TE->Scalars, *DL) ||
// Nodes with copyable lanes may mix in non-extract lanes, which
// are not representable as a shuffle of the source vector.
(((TE->hasState() &&
TE->getOpcode() == Instruction::ExtractElement &&
!TE->hasCopyableElements()) ||
all_of(TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
isFixedVectorShuffle(TE->Scalars, Mask, AC, Cache)) ||
(TE->hasState() && TE->getOpcode() == Instruction::Load &&
!TE->isAltShuffle()) ||
any_of(TE->Scalars, IsaPred<LoadInst>));
};
// We only handle trees of heights 1 and 2.
if (VectorizableTree.size() == 1 &&
(VectorizableTree[0]->State == TreeEntry::Vectorize ||
VectorizableTree[0]->State == TreeEntry::StridedVectorize ||
VectorizableTree[0]->State == TreeEntry::ExpandVectorize ||
VectorizableTree[0]->State == TreeEntry::CompressVectorize ||
VectorizableTree[0]->State == TreeEntry::BlendedLoadVectorize ||
(ForReduction &&
AreVectorizableGathers(VectorizableTree[0].get(),
VectorizableTree[0]->Scalars.size()) &&
VectorizableTree[0]->getVectorFactor() > 2)))
return true;
if (VectorizableTree.size() != 2)
return false;
// Handle splat and all-constants stores. Also try to vectorize tiny trees
// with the second gather nodes if they have less scalar operands rather than
// the initial tree element (may be profitable to shuffle the second gather)
// or they are extractelements, which form shuffle.
if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
AreVectorizableGathers(VectorizableTree[1].get(),
VectorizableTree[0]->Scalars.size()))
return true;
// Gathering cost would be too much for tiny trees.
if (VectorizableTree[0]->isGather() ||
(VectorizableTree[1]->isGather() &&
VectorizableTree[0]->State != TreeEntry::ScatterVectorize &&
VectorizableTree[0]->State != TreeEntry::StridedVectorize &&
VectorizableTree[0]->State != TreeEntry::ExpandVectorize &&
VectorizableTree[0]->State != TreeEntry::CompressVectorize &&
VectorizableTree[0]->State != TreeEntry::BlendedLoadVectorize))
return false;
return true;
}
bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
if (!DebugCounter::shouldExecute(VectorizedGraphs))
return true;
// If we are revectorizing reduction, it may result in same reduction pattern
// with shuffles as leaves of the reduction. Prevent SLP from revectorizing
// that shuffle pattern.
if (Opts.slp_revec && ForReduction && VectorizableTree.size() == 3 &&
VectorizableTree[0]->State == TreeEntry::Vectorize &&
VectorizableTree[0]->getOpcode() == Instruction::ShuffleVector &&
VectorizableTree[1]->isGather() &&
isSplat(VectorizableTree[1]->Scalars) &&
VectorizableTree[2]->isGather() &&
allConstant(VectorizableTree[2]->Scalars)) {
LLVM_DEBUG(dbgs() << "SLP: Rejecting reduction tree with 3 nodes(shuffle "
"as root and remaining are gather nodes).\n");
return true;
}
// Graph is empty - do nothing.
if (VectorizableTree.empty()) {
assert(ExternalUses.empty() && "We shouldn't have any external users");
return true;
}
// FIXME: support buildvector of the gather nodes with struct types.
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() &&
isa<StructType>(TE->Scalars.front()->getType());
})) {
LLVM_DEBUG(
dbgs() << "SLP: rejecting tree with buildvector struct values of size "
<< VectorizableTree.size() << ".\n");
return true;
}
// Cache values from the root node and the cost-threshold options to avoid
// re-querying them inside hot predicates below.
const unsigned TreeSize = VectorizableTree.size();
const TreeEntry &Front = getRootNode();
const bool FrontIsGather = Front.isGather();
const bool FrontHasState = Front.hasState();
const unsigned FrontOpcode = FrontHasState ? Front.getOpcode() : 0u;
const bool ThresholdSet = Opts.slp_threshold.has_value();
const bool ThresholdNonNegative = getCostThreshold() >= 0;
constexpr unsigned Limit = 4;
constexpr unsigned LargeTree = 20;
constexpr unsigned LimitTreeSize = 36;
// The remaining size-1/size-<=MinTreeSize early bail-outs only apply to
// non-reduction trees; group them under a single guard to avoid 3 separate
// !ForReduction short-circuits when reducing.
if (!ForReduction) {
// Single gather node: bail out for ExtractElement or any node containing a
// real Instruction scalar.
if (TreeSize == 1 && FrontIsGather) {
if (FrontHasState && FrontOpcode == Instruction::ExtractElement)
return true;
if (any_of(Front.Scalars, IsaPred<Instruction>))
return true;
}
if (TreeSize <= Opts.slp_min_tree_size &&
all_of(VectorizableTree, [](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() || TE->State == TreeEntry::SplitVectorize;
}))
return true;
if (TreeSize == 1 && getCostThreshold() < 0 && FrontHasState &&
FrontOpcode == Instruction::ExtractElement &&
(Front.getVectorFactor() == 2 ||
all_of(make_isa_range<Instruction>(Front.Scalars),
[&](Instruction *I) {
return !areAllUsersVectorized(I, UserIgnoreList);
})))
return true;
}
// No need to vectorize inserts of gathered values.
if (TreeSize == 2 && isa<InsertElementInst>(Front.Scalars[0]) &&
VectorizableTree[1]->isGather() &&
(VectorizableTree[1]->getVectorFactor() <= 2 ||
!(isSplat(VectorizableTree[1]->Scalars) ||
allConstant(VectorizableTree[1]->Scalars))))
return true;
// The tree with only 3 nodes, where 2 last are gathers/buildvectors, not
// profitable for vectorization.
if (TreeSize == 3 && getCostThreshold() == 0 &&
(!ForReduction || Front.getVectorFactor() <= 2) &&
all_of(ArrayRef(VectorizableTree).drop_front(),
[&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && TE->getVectorFactor() <= Limit &&
!all_of(
TE->Scalars,
IsaPred<ExtractElementInst, UndefValue, Constant>);
}))
return true;
// All remaining bail-out heuristics require !ForReduction. Group them under
// a single guard so reduction trees skip them with one branch instead of one
// per check.
if (!ForReduction) {
// The single vectorized node, fed by the buildvectors, only repacks the
// scalars, the gathers and the extracts are paid for the one vector
// instruction. The vector call replaces the several expensive scalar ones,
// while the store merges the lanes instead of extracting them back.
if (TreeSize > 1 && ThresholdNonNegative &&
(!FrontHasState || (FrontOpcode != Instruction::Call &&
FrontOpcode != Instruction::Store)) &&
count_if(VectorizableTree,
[](const std::unique_ptr<TreeEntry> &TE) {
return TE->State == TreeEntry::Vectorize;
}) == 1 &&
all_of(VectorizableTree, [](const std::unique_ptr<TreeEntry> &TE) {
return TE->State == TreeEntry::Vectorize ||
(TE->isGather() && !isSplat(TE->Scalars) &&
!all_of(TE->Scalars,
IsaPred<ExtractElementInst, UndefValue, Constant>));
}))
return true;
// If the graph includes only PHI nodes and gathers, it is defnitely not
// profitable for the vectorization, we can skip it, if the cost threshold
// is default. The cost of vectorized PHI nodes is almost always 0 + the
// cost of gathers/buildvectors.
if (!ThresholdSet &&
all_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
const bool IsGather = TE->isGather();
const bool HasState = TE->hasState();
const unsigned Op = HasState ? TE->getOpcode() : 0u;
if (IsGather && (!HasState || Op != Instruction::ExtractElement) &&
static_cast<unsigned>(
count_if(TE->Scalars, IsaPred<ExtractElementInst>)) <= Limit)
return true;
return HasState && Op == Instruction::PHI;
}))
return true;
// Do not vectorize small tree of phis only, if all vector phis are also
// gathered.
if (ThresholdSet && TreeSize <= Limit) {
bool HasVectorPhi = false;
auto Compatible = [&](const std::unique_ptr<TreeEntry> &TE) {
const bool IsGather = TE->isGather();
const bool HasState = TE->hasState();
const unsigned Op = HasState ? TE->getOpcode() : 0u;
if (IsGather && (!HasState || Op != Instruction::ExtractElement) &&
static_cast<unsigned>(
count_if(TE->Scalars, IsaPred<ExtractElementInst>)) <= Limit)
return true;
if (!HasState)
return false;
if (Op == Instruction::InsertElement)
return true;
if (Op != Instruction::PHI)
return false;
if (TE->State == TreeEntry::Vectorize)
HasVectorPhi = true;
return all_of(TE->Scalars, [&](Value *V) {
return isa<PoisonValue>(V) || MustGather.contains(V);
});
};
if (all_of(VectorizableTree, Compatible) && HasVectorPhi)
return true;
}
// PHI nodes only and gathers cannot be vectorized, skip.
if (ThresholdNonNegative) {
const bool IsLargeTree = TreeSize >= LargeTree;
bool HasSingleLoad = false;
if (all_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
const bool IsGather = TE->isGather();
const bool HasState = TE->hasState();
const unsigned Op = HasState ? TE->getOpcode() : 0u;
// HasSingleLoad/PrevLoad are only consulted in the
// IsLargeTree branch; skip the bookkeeping otherwise.
if (IsLargeTree) {
const bool PrevLoad = HasSingleLoad;
HasSingleLoad |=
HasState && !IsGather &&
(Op == Instruction::Load || TE->hasCopyableElements()) &&
(TE->getVectorFactor() > 2 || TE->ReorderIndices.empty());
if (HasState) {
if (Op == Instruction::PHI)
return true;
if (TE->getVectorFactor() <= Limit &&
(Op == Instruction::Store ||
(Op == Instruction::Load && !PrevLoad)))
return true;
}
} else if (HasState && Op == Instruction::PHI) {
return true;
}
return IsGather && (!HasState || Op != Instruction::ExtractElement);
}))
return true;
// Single non-phi vector node - skip the tree.
if (TreeSize >= 5 && Front.getVectorFactor() <= 2 &&
Front.Scalars.front()->getType()->isIntegerTy()) {
bool VectorNodeFound = false;
bool AnyNonConst = false;
if (all_of(VectorizableTree,
[&](const std::unique_ptr<TreeEntry> &TE) {
if (TE->State == TreeEntry::Vectorize && TE->hasState()) {
const unsigned Op = TE->getOpcode();
if (Op == Instruction::PHI ||
!TE->ReorderIndices.empty())
return true;
if (VectorNodeFound)
return false;
VectorNodeFound = true;
return true;
}
// Once AnyNonConst is true, skip the O(n) allConstant
// walk for subsequent entries.
if (!AnyNonConst)
AnyNonConst = !allConstant(TE->Scalars);
return TE->isGather() ||
TE->State == TreeEntry::SplitVectorize;
}) &&
AnyNonConst)
return true;
}
}
// Common predicate for "phis, buildvectors, split nodes and small nodes
// with reuses" used by the two checks below. Cheap checks are evaluated
// before expensive Scalars walks.
auto IsBenignNode = [&](const TreeEntry &TE) {
if (TE.State == TreeEntry::SplitVectorize)
return true;
const bool IsGather = TE.isGather();
const bool HasState = TE.hasState();
if (HasState) {
const unsigned Op = TE.getOpcode();
if (Op == Instruction::PHI)
return true;
const unsigned ScalarsSize = TE.Scalars.size();
if (TE.Idx == 0 && ScalarsSize == 2 && Op == Instruction::ICmp &&
TreeSize > LimitTreeSize)
return true;
if (ScalarsSize == 2 &&
(!TE.ReuseShuffleIndices.empty() || !TE.ReorderIndices.empty() ||
TE.isAltShuffle()))
return true;
if (TE.hasCopyableElements() &&
static_cast<unsigned>(count_if(
TE.Scalars, IsaPred<PHINode, Constant>)) >= ScalarsSize / 2)
return true;
}
return IsGather && none_of(TE.Scalars, IsaPred<ExtractElementInst>);
};
// If the tree contains only phis, buildvectors, split nodes and
// small nodes with reuses, we can skip it.
if (!ThresholdSet) {
SmallVector<const TreeEntry *> StoreLoadNodes;
unsigned NumGathers = 0;
if (all_of(VectorizableTree,
[&](const std::unique_ptr<TreeEntry> &TE) {
const bool IsGather = TE->isGather();
if (!IsGather && TE->hasState()) {
const unsigned Op = TE->getOpcode();
if (Op == Instruction::Load || Op == Instruction::Store) {
StoreLoadNodes.push_back(TE.get());
return true;
}
}
if (IsGather)
++NumGathers;
return IsBenignNode(*TE);
}) &&
(StoreLoadNodes.empty() ||
(TreeSize > LimitTreeSize * StoreLoadNodes.size() &&
(NumGathers > 0 ||
none_of(StoreLoadNodes, [&](const TreeEntry *TE) {
return TE->getOpcode() == Instruction::Store ||
all_of(make_isa_range<LoadInst>(TE->Scalars),
[&](LoadInst *LI) {
return areAllUsersVectorized(LI);
});
})))))
return true;
}
// If the tree contains only phis, buildvectors, split nodes and
// small nodes with reuses, we can skip it.
if (ThresholdNonNegative && TreeSize > LimitTreeSize) {
const TreeEntry *VectorNode = nullptr;
if (all_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
if (!TE->isGather() && TE->hasState() &&
TE->State != TreeEntry::SplitVectorize &&
TE->getOpcode() != Instruction::PHI) {
if (VectorNode)
return false;
VectorNode = TE.get();
return true;
}
return IsBenignNode(*TE);
}))
return true;
}
// If the tree contains only buildvector, 2 non-buildvectors (with root
// user tree node) and other buildvectors, we can skip it.
if (ThresholdSet && TreeSize >= Limit &&
Front.State == TreeEntry::SplitVectorize &&
count_if(ArrayRef(VectorizableTree).drop_front(),
[](const std::unique_ptr<TreeEntry> &TE) {
return !TE->isGather() && TE->UserTreeIndex.UserTE &&
TE->UserTreeIndex.UserTE->Idx == 0;
}) == 2)
return true;
// If the tree contains only vectorization of the phi node from the
// buildvector - skip it.
if (ThresholdSet && TreeSize > 2 && Front.State == TreeEntry::Vectorize &&
FrontOpcode == Instruction::InsertElement &&
VectorizableTree[1]->State == TreeEntry::Vectorize &&
VectorizableTree[1]->getOpcode() == Instruction::PHI &&
all_of(ArrayRef(VectorizableTree).drop_front(2),
[](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather();
}))
return true;
}
// We can vectorize the tree if its size is greater than or equal to the
// minimum size specified by the MinTreeSize command line option.
if (TreeSize >= Opts.slp_min_tree_size)
return false;
// If we have a tiny tree (a tree whose size is less than MinTreeSize), we
// can vectorize it if we can prove it fully vectorizable.
if (isFullyVectorizableTinyTree(ForReduction))
return false;
// Check if any of the gather node forms an insertelement buildvector
// somewhere. TreeSize >= 1 is guaranteed, so the multi-node case reduces to
// a simple TreeSize > 1 short-circuit.
// A gather with copyable lanes is not a real instruction node; do not let
// its state qualify it as a buildvector-forming node.
const bool IsAllowedSingleBVNode =
TreeSize > 1 ||
(FrontHasState && !Front.isAltShuffle() && !Front.hasCopyableElements() &&
FrontOpcode != Instruction::PHI &&
FrontOpcode != Instruction::GetElementPtr &&
allSameBlock(Front.Scalars));
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && all_of(TE->Scalars, [&](Value *V) {
return isa<ExtractElementInst, Constant>(V) ||
(IsAllowedSingleBVNode &&
!V->hasNUsesOrMore(UsesLimit) &&
any_of(V->users(), IsaPred<InsertElementInst>));
});
}))
return false;
const TreeEntry &Back = *VectorizableTree.back();
if (Back.isGather() && Back.hasState() && Back.isAltShuffle()) {
const unsigned BackVF = Back.getVectorFactor();
if (BackVF > 2 && allSameBlock(Back.Scalars) &&
!Back.Scalars.front()->getType()->isVectorTy() &&
TTI->getScalarizationOverhead(
cast<VectorType>(
getWidenedType(Back.Scalars.front()->getType(), BackVF)),
APInt::getAllOnes(BackVF),
/*Insert=*/true, /*Extract=*/false, CostKind) > -getCostThreshold())
return false;
}
// Otherwise, we can't vectorize the tree. It is both tiny and not fully
// vectorizable.
return true;
}
bool BoUpSLP::isTreeNotExtendable() const {
if (getCanonicalGraphSize() != getTreeSize()) {
constexpr unsigned SmallTree = 3;
if (getRootNode().isNonPowOf2Vec() &&
getCanonicalGraphSize() <= SmallTree &&
count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
[](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() && TE->hasState() &&
TE->getOpcode() == Instruction::Load &&
!allSameBlock(TE->Scalars);
}) == 1)
return true;
return false;
}
bool Res = false;
for (unsigned Idx : seq<unsigned>(getTreeSize())) {
TreeEntry &E = *VectorizableTree[Idx];
if (E.State == TreeEntry::SplitVectorize)
return false;
if (!E.isGather())
continue;
if ((E.hasState() && E.getOpcode() != Instruction::Load) ||
(!E.hasState() &&
all_of(E.Scalars, IsaPred<ExtractElementInst, LoadInst>)) ||
(isa<ExtractElementInst>(E.Scalars.front()) &&
getSameOpcode(ArrayRef(E.Scalars).drop_front(), *TLI).valid()))
return false;
if (isSplat(E.Scalars) || allConstant(E.Scalars))
continue;
Res = true;
}
return Res;
}
InstructionCost BoUpSLP::getSpillCost() {
// Walk the vectorizable tree from the root towards its leaves, tracking
// which vectorized operand values would be live across each tree edge
// (i.e. between the last instruction of an operand entry and the last
// instruction of its user entry). When the live range crosses a call
// instruction that is not part of the vectorized tree, query TTI for the
// cost of keeping the value live across it (for example, if spills and
// fills are required).
const TreeEntry *Root = &getRootNode();
if (Root->isGather())
return 0;
InstructionCost Cost = 0;
SmallDenseMap<const TreeEntry *, SmallVector<const TreeEntry *>>
EntriesToOperands;
SmallDenseMap<const TreeEntry *, Instruction *> EntriesToLastInstruction;
SmallPtrSet<const Instruction *, 8> LastInstructions;
SmallPtrSet<const TreeEntry *, 8> ScalarOrPseudoEntries;
for (const auto &TEPtr : VectorizableTree) {
if (TEPtr->CombinedOp == TreeEntry::ReducedBitcast ||
TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
TEPtr->CombinedOp == TreeEntry::ReducedBitcastLoads ||
TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
TEPtr->CombinedOp == TreeEntry::ReducedBitPack ||
TEPtr->CombinedOp == TreeEntry::ReducedCmpBitcast) {
ScalarOrPseudoEntries.insert(TEPtr.get());
continue;
}
if (!TEPtr->isGather()) {
Instruction *LastInst = &getLastInstructionInBundle(TEPtr.get());
EntriesToLastInstruction.try_emplace(TEPtr.get(), LastInst);
LastInstructions.insert(LastInst);
}
if (TEPtr->UserTreeIndex)
EntriesToOperands[TEPtr->UserTreeIndex.UserTE].push_back(TEPtr.get());
}
// Cache NoCallIntrinsicOrDoesNotReturn results - the same intrinsic call may
// be queried many times during the spill cost scan, and each computation
// involves two potentially expensive TTI virtual calls.
SmallDenseMap<const IntrinsicInst *, bool> NoCallIntrinsicCache;
auto NoCallIntrinsicOrDoesNotReturn =
[this, &NoCallIntrinsicCache](const Instruction *I) {
const auto *CB = dyn_cast<CallBase>(I);
if (!CB)
return false;
if (CB->doesNotReturn())
return true;
const auto *II = dyn_cast<IntrinsicInst>(CB);
if (!II)
return false;
if (II->isAssumeLikeIntrinsic())
return true;
auto [It, Inserted] = NoCallIntrinsicCache.try_emplace(II);
if (!Inserted)
return It->second;
IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II);
InstructionCost IntrCost = TTI->getIntrinsicInstrCost(ICA, CostKind);
InstructionCost CallCost = TTI->getCallInstrCost(
nullptr, II->getType(), ICA.getArgTypes(), CostKind);
bool Res = IntrCost < CallCost;
It->second = Res;
return Res;
};
// Maps last instruction in the entry to the last instruction for the one of
// operand entries and the flag. If the flag is true, there are no calls in
// between these instructions.
SmallDenseMap<const Instruction *, PointerIntPair<const Instruction *, 1>>
CheckedInstructions;
unsigned Budget = 0;
const unsigned BudgetLimit =
Opts.slp_schedule_budget / VectorizableTree.size();
auto CheckForNonVecCallsInSameBlock = [&](Instruction *First,
const Instruction *Last) {
assert(First->getParent() == Last->getParent() &&
"Expected instructions in same block.");
if (auto It = CheckedInstructions.find(Last);
It != CheckedInstructions.end()) {
const Instruction *Checked = It->second.getPointer();
const bool NoCallsInCachedRange = It->second.getInt() != 0;
if (Checked == First)
return NoCallsInCachedRange;
if (Checked->comesBefore(First))
// In every cached state (full clean scan, call-found, or
// budget-exhausted) the region strictly above `Checked` up to `Last`
// was inspected and proved call-free. Since `First` is above
// `Checked`, the queried range [First, Last] is contained in that
// call-free region, regardless of whether bit is 0 or 1.
return true;
Last = Checked;
} else if (Last == First || Last->comesBefore(First)) {
// Empty range.
return true;
}
BasicBlock::const_reverse_iterator InstIt =
++First->getIterator().getReverse(),
PrevInstIt =
Last->getIterator().getReverse();
SmallVector<const Instruction *> LastInstsInRange;
while (InstIt != PrevInstIt && Budget <= BudgetLimit) {
// Debug information does not impact spill cost.
// Vectorized calls, represented as vector intrinsics, do not impact spill
// cost.
if (const auto *CB = dyn_cast<CallBase>(&*PrevInstIt);
CB && !NoCallIntrinsicOrDoesNotReturn(CB) && !isVectorized(CB)) {
for (const Instruction *LastInst : LastInstsInRange)
CheckedInstructions.try_emplace(LastInst, &*PrevInstIt, 0);
return false;
}
if (LastInstructions.contains(&*PrevInstIt))
LastInstsInRange.push_back(&*PrevInstIt);
++PrevInstIt;
++Budget;
}
// If we reached the scan's lower bound (`PrevInstIt == InstIt`) then the
// whole [First, Last] range was inspected and found call-free, even if
// Budget just overflowed at the very last step; do not mislabel such a
// completed scan as "has call".
const bool Completed = PrevInstIt == InstIt;
const bool NoCallsInRange = Completed || Budget <= BudgetLimit;
for (const Instruction *LastInst : LastInstsInRange)
CheckedInstructions.try_emplace(
LastInst, Completed ? First : &*PrevInstIt, NoCallsInRange ? 1 : 0);
return NoCallsInRange;
};
// The spill/reload is executed once per execution of the call, so the
// cost is scaled by the trip count of the loop containing the call, even
// for hoisted loop-invariant values defined outside of it.
auto GetSpillScale = [&](const BasicBlock *BB) {
return getLoopNestScale(LI->getLoopFor(BB));
};
auto AddCosts = [&](const TreeEntry *Op, uint64_t Scale) {
if (ScalarOrPseudoEntries.contains(Op))
return;
Type *ScalarTy = Op->Scalars.front()->getType();
auto It = MinBWs.find(Op);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, Op->getVectorFactor());
InstructionCost KeepLiveCost = TTI->getCostOfKeepingLiveOverCall(VecTy);
KeepLiveCost *= Scale;
Cost += KeepLiveCost;
if (ScalarTy->isVectorTy()) {
// Handle revec dead vector instructions.
Cost -= Op->Scalars.size() * TTI->getCostOfKeepingLiveOverCall(ScalarTy) *
Scale;
}
};
// Memoize the relationship between blocks, i.e. the spill scale if every
// path between the blocks crosses a non-vectorized call, 0 if there is (at
// least one) call-free path. This allows to skip the analysis of the same
// block paths multiple times.
SmallDenseMap<std::pair<const BasicBlock *, const BasicBlock *>, uint64_t>
ParentOpParentToPreds;
// Memoize whether a basic block contains a non-terminator no-return call.
// Such blocks are dead-end paths in normal control flow (execution does not
// exit them past the no-return call), so the block is excluded from the
// spill cost analysis. Terminator no-return calls (invoke/callbr) are not
// block-killing because they still have live CFG successors (e.g. the
// unwind destination of an invoke).
SmallDenseMap<const BasicBlock *, bool> BlockHasNoReturnCallCache;
auto BlockHasNoReturnCall = [&](const BasicBlock *BB) {
auto [It, Inserted] = BlockHasNoReturnCallCache.try_emplace(BB, false);
if (!Inserted)
return It->second;
for (const CallBase &CB : make_isa_range<CallBase>(*BB)) {
if (CB.doesNotReturn() && !CB.isTerminator()) {
It->second = true;
return true;
}
}
return false;
};
// Memoize whether a loop's body (all blocks of the loop, including
// sub-loops) contains any non-vec call.
SmallDenseMap<const Loop *, bool> LoopBodyHasNonVecCall;
auto LoopBodyHasCall = [&](const Loop *L) {
if (auto It = LoopBodyHasNonVecCall.find(L);
It != LoopBodyHasNonVecCall.end())
return It->second;
auto IsNotCatchSwitch = [](BasicBlock *BB) {
return !isa<CatchSwitchInst>(BB->getTerminator());
};
for (BasicBlock *BB : make_filter_range(L->blocks(), IsNotCatchSwitch)) {
// Blocks containing a no-return call are dead-end paths and never
// actually flow back through the loop's back-edge, so their calls do
// not keep loop-invariant vector values live across calls.
if (BlockHasNoReturnCall(BB))
continue;
for (const Instruction &I : *BB) {
const auto *CB = dyn_cast<CallBase>(&I);
if (!CB || NoCallIntrinsicOrDoesNotReturn(CB) || isVectorized(CB))
continue;
LoopBodyHasNonVecCall.try_emplace(L, true);
return true;
}
}
LoopBodyHasNonVecCall.try_emplace(L, false);
return false;
};
auto GetLoopInvariantSpillRegion = [&](BasicBlock *UseBB,
BasicBlock *DefBB) -> const Loop * {
// If DefBB is outside a loop containing UseBB, the value is live across
// iterations of that loop. A non-vectorized call in the loop may therefore
// require a spill even when the entry path from DefBB to UseBB is
// call-free. Return the outermost such loop containing a relevant call so
// its execution scale can be used to cost the spill.
const Loop *L = LI->getLoopFor(UseBB);
const Loop *Outermost = nullptr;
while (L && !L->contains(DefBB)) {
Outermost = L;
L = L->getParentLoop();
}
return (Outermost && LoopBodyHasCall(Outermost)) ? Outermost : nullptr;
};
auto CheckPredecessors = [&](BasicBlock *Root, BasicBlock *Pred,
BasicBlock *OpParent) {
auto Key = std::make_pair(Root, OpParent);
if (auto It = ParentOpParentToPreds.find(Key);
It != ParentOpParentToPreds.end())
return It->second;
uint64_t Res = 0;
scope_exit Cleanup([&]() { ParentOpParentToPreds.try_emplace(Key, Res); });
if (const auto *L = GetLoopInvariantSpillRegion(Root, OpParent)) {
Res = getLoopNestScale(L);
return Res;
}
SmallVector<BasicBlock *> Worklist;
if (Pred)
Worklist.push_back(Pred);
else
Worklist.append(pred_begin(Root), pred_end(Root));
SmallPtrSet<const BasicBlock *, 16> Visited;
// With "at least one call-free path" semantics we can only reliably
// memoize the exact (Root, OpParent) query. Pairs for intermediate
// blocks that were visited during the BFS are not necessarily
// call-free-reachable to OpParent themselves - we may have reached
// OpParent through a *sibling* path that bypassed them.
// We return 0 (no spill cost) if at least one backward path from
// some predecessor of Root back to OpParent is call-free. Only when
// *every* such path goes through a non-vec call do we charge the spill
// cost: only then is it actually necessary to keep the vectorized value
// live across a call and therefore spill/reload it.
//
// A BB is only explored further (its predecessors added to the worklist)
// when it is itself call-free and not strictly dominated by Root (blocks
// dominated by Root are only reachable via loop back-edges - they sit
// *after* Root in forward execution and must not be counted).
//
// If we ever pop OpParent from the worklist, we have reached it through
// a chain of call-free, non-dominated blocks: a call-free path exists
// and we return 0. If the worklist is exhausted or the scan budget
// overflows without reaching OpParent, no call-free path was found and
// we return the scale of the loop containing Root, so the caller charges
// the spill cost.
while (!Worklist.empty()) {
BasicBlock *BB = Worklist.pop_back_val();
if (BB == OpParent)
return Res;
if (!Visited.insert(BB).second)
continue;
// Blocks strictly dominated by Root are reached only *after* Root in
// forward execution (via loop back-edges); skip them and their
// dominated predecessors.
if (DT->properlyDominates(Root, BB))
continue;
// A block containing a no-return call cannot reach Root via the
// forward edge being analyzed: execution does not continue past the
// no-return call, so the BB -> ... -> Root path is dead. Drop the
// block from the analysis without following its predecessors.
if (BlockHasNoReturnCall(BB))
continue;
auto Pair = std::make_pair(BB, OpParent);
if (auto It = ParentOpParentToPreds.find(Pair);
It != ParentOpParentToPreds.end()) {
// BB is known to reach OpParent via a call-free path.
if (It->second == 0)
return Res;
// BB is known to be blocked from OpParent by calls; keep checking
// other paths.
continue;
}
unsigned BlockSize = BB->size();
if (BlockSize > static_cast<unsigned>(Opts.slp_schedule_budget))
continue;
Budget += BlockSize;
if (Budget > BudgetLimit)
break;
if (!isa<CatchSwitchInst>(BB->getTerminator()) &&
!CheckForNonVecCallsInSameBlock(&*BB->getFirstNonPHIOrAlloca(),
BB->getTerminator()))
continue;
Worklist.append(pred_begin(BB), pred_end(BB));
}
Res = GetSpillScale(Root);
return Res;
};
SmallVector<const TreeEntry *> LiveEntries(1, Root);
auto FindNonScalarParentEntry = [&](const TreeEntry *E) -> const TreeEntry * {
assert(ScalarOrPseudoEntries.contains(E) &&
"Expected scalar or pseudo entry.");
const TreeEntry *Entry = E;
while (Entry->UserTreeIndex) {
Entry = Entry->UserTreeIndex.UserTE;
if (!ScalarOrPseudoEntries.contains(Entry))
return Entry;
}
return nullptr;
};
auto IsCoveredByMatchingVectorEntry = [&](const TreeEntry *Gather,
const Loop *SpillLoop) -> bool {
assert(Gather->isGather() && "Expected a gather/buildvector entry.");
Value *LookupValue = nullptr;
if (Gather->hasState()) {
LookupValue = Gather->getMainOp();
} else {
auto *It = find_if_not(Gather->Scalars, IsaPred<UndefValue>);
if (It == Gather->Scalars.end())
return false;
LookupValue = *It;
}
// Find the real vector entry reused by this perfect-diamond gather.
const TreeEntry *SameTE =
getSameValuesTreeEntry(LookupValue, Gather->Scalars, /*SameVF=*/true);
if (!SameTE || SameTE == Gather || SameTE->State != TreeEntry::Vectorize ||
ScalarOrPseudoEntries.contains(SameTE) || !SameTE->UserTreeIndex)
return false;
// A reordered or reuse-shuffled match materializes a separate shuffled
// vector that is itself live across the call; only an exact match reuses
// the same vector value.
if (!SameTE->ReorderIndices.empty() || !SameTE->ReuseShuffleIndices.empty())
return false;
// Only permit an ordinary vectorized user.
const TreeEntry *UserTE = SameTE->UserTreeIndex.UserTE;
assert(UserTE && "Expected a user tree entry.");
if (UserTE->State != TreeEntry::Vectorize ||
ScalarOrPseudoEntries.contains(UserTE) ||
UserTE->getOpcode() == Instruction::PHI)
return false;
// Different demoted bitwidth would make the two edge costs unequal.
if (MinBWs.lookup(SameTE).first != MinBWs.lookup(Gather).first)
return false;
// The spill walk does not descend through gather entries; if any ancestor
// of the matching entry's user is a gather or a combined scalar/pseudo
// entry, the matching edge is never charged and de-duplicating would lose
// the spill cost entirely.
for (const TreeEntry *E = UserTE; E != Root;) {
if (!E->UserTreeIndex)
return false;
E = E->UserTreeIndex.UserTE;
if (E->isGather() || ScalarOrPseudoEntries.contains(E))
return false;
}
Instruction *Def = EntriesToLastInstruction.at(SameTE);
Instruction *Use = EntriesToLastInstruction.at(UserTE);
// Require the matching entry to be defined outside the loop and its use to
// execute directly in the same loop as the gather user.
if (SpillLoop->contains(Def->getParent()) ||
LI->getLoopFor(Use->getParent()) != SpillLoop)
return false;
// Ensure that the matching entry is also loop invariant.
if (!all_of(make_isa_range<Instruction>(SameTE->Scalars),
[&](Instruction *I) { return SpillLoop->isLoopInvariant(I); }))
return false;
// A non-vec call between Def and the end of its block preempts the
// loop-invariant charge for the matching entry's edge with a smaller
// scale, so de-duplicating would drop the in-loop spill cost.
if (!CheckForNonVecCallsInSameBlock(Def, Def->getParent()->getTerminator()))
return false;
return GetLoopInvariantSpillRegion(Use->getParent(), Def->getParent()) ==
SpillLoop;
};
while (!LiveEntries.empty()) {
const TreeEntry *Entry = LiveEntries.pop_back_val();
const auto OpIt = EntriesToOperands.find(Entry);
if (OpIt == EntriesToOperands.end())
continue;
ArrayRef<const TreeEntry *> Operands = OpIt->second;
if (Operands.empty())
continue;
if (ScalarOrPseudoEntries.contains(Entry)) {
Entry = FindNonScalarParentEntry(Entry);
if (!Entry) {
for (const TreeEntry *Op : Operands) {
if (!Op->isGather())
LiveEntries.push_back(Op);
}
continue;
}
}
Instruction *LastInst = EntriesToLastInstruction.at(Entry);
BasicBlock *Parent = LastInst->getParent();
for (const TreeEntry *Op : Operands) {
if (!Op->isGather())
LiveEntries.push_back(Op);
if (ScalarOrPseudoEntries.contains(Op))
continue;
if (Entry->State == TreeEntry::SplitVectorize ||
(Op->isGather() && allConstant(Op->Scalars)))
continue;
// Reset the scan budget for analysis of each edge.
Budget = 0;
// A gather with all loop-invariant lanes is hoisted to the loop
// preheader by optimizeGatherSequence, so its vector value becomes live
// across any non-vectorized call in the loop body. Charge it like any
// other vectorized value live over a call instead of skipping it.
if (Entry->hasState() && Entry->getOpcode() != Instruction::PHI &&
Op->isGather()) {
if (const Loop *L = LI->getLoopFor(Parent);
L && L->getLoopPreheader() && LoopBodyHasCall(L) &&
all_of(make_isa_range<Instruction>(Op->Scalars),
[&](Instruction *I) { return L->isLoopInvariant(I); }))
if (!IsCoveredByMatchingVectorEntry(Op, L))
AddCosts(Op, GetSpillScale(Parent));
continue;
}
BasicBlock *Pred = nullptr;
if (auto *Phi = dyn_cast<PHINode>(Entry->getMainOp()))
Pred = Phi->getIncomingBlock(Op->UserTreeIndex.EdgeIdx);
BasicBlock *OpParent;
Instruction *OpLastInst;
if (Op->isGather()) {
assert(Entry->getOpcode() == Instruction::PHI &&
"Expected phi node only.");
OpParent = cast<PHINode>(Entry->getMainOp())
->getIncomingBlock(Op->UserTreeIndex.EdgeIdx);
OpLastInst = OpParent->getTerminator();
for (Instruction *Inst : make_isa_range<Instruction>(Op->Scalars)) {
if (isVectorized(Inst)) {
OpParent = Inst->getParent();
OpLastInst = Inst;
break;
}
}
} else {
OpLastInst = EntriesToLastInstruction.at(Op);
OpParent = OpLastInst->getParent();
}
// Check the call instructions within the same basic blocks.
if (OpParent == Parent) {
if (Entry->getOpcode() == Instruction::PHI) {
if (!CheckForNonVecCallsInSameBlock(LastInst, OpLastInst))
AddCosts(Op, GetSpillScale(Parent));
continue;
}
if (!CheckForNonVecCallsInSameBlock(OpLastInst, LastInst))
AddCosts(Op, GetSpillScale(Parent));
continue;
}
// Check for call instruction in between blocks.
// 1. Check entry's block to the head.
if (Entry->getOpcode() != Instruction::PHI &&
!CheckForNonVecCallsInSameBlock(&*Parent->getFirstNonPHIOrAlloca(),
LastInst)) {
AddCosts(Op, GetSpillScale(Parent));
continue;
}
// 2. Check op's block from the end.
if (!CheckForNonVecCallsInSameBlock(OpLastInst,
OpParent->getTerminator())) {
AddCosts(Op, GetSpillScale(OpParent));
continue;
}
// 3. Check the predecessors of entry's block till op's block.
if (uint64_t Scale = CheckPredecessors(Parent, Pred, OpParent))
AddCosts(Op, Scale);
}
}
return Cost;
}
namespace {
/// Returns incoming Value *, if the requested type is Value * too, or a default
/// value, otherwise.
struct ValueSelect {
template <typename U>
static std::enable_if_t<std::is_same_v<Value *, U>, Value *> get(Value *V) {
return V;
}
template <typename U>
static std::enable_if_t<!std::is_same_v<Value *, U>, U> get(Value *) {
return U();
}
};
} // namespace
/// Does the analysis of the provided shuffle masks and performs the requested
/// actions on the vectors with the given shuffle masks. It tries to do it in
/// several steps.
/// 1. If the Base vector is not undef vector, resizing the very first mask to
/// have common VF and perform action for 2 input vectors (including non-undef
/// Base). Other shuffle masks are combined with the resulting after the 1 stage
/// and processed as a shuffle of 2 elements.
/// 2. If the Base is undef vector and have only 1 shuffle mask, perform the
/// action only for 1 vector with the given mask, if it is not the identity
/// mask.
/// 3. If > 2 masks are used, perform the remaining shuffle actions for 2
/// vectors, combing the masks properly between the steps.
template <typename T>
static T *performExtractsShuffleAction(
MutableArrayRef<std::pair<T *, SmallVector<int>>> ShuffleMask, Value *Base,
function_ref<unsigned(T *)> GetVF,
function_ref<std::pair<T *, bool>(T *, ArrayRef<int>, bool)> ResizeAction,
function_ref<T *(ArrayRef<int>, ArrayRef<T *>)> Action) {
assert(!ShuffleMask.empty() && "Empty list of shuffles for inserts.");
SmallVector<int> Mask(ShuffleMask.begin()->second);
auto VMIt = std::next(ShuffleMask.begin());
T *Prev = nullptr;
SmallBitVector UseMask =
buildUseMask(Mask.size(), Mask, UseMask::UndefsAsMask);
SmallBitVector IsBaseUndef = isUndefVector(Base, UseMask);
if (!IsBaseUndef.all()) {
// Base is not undef, need to combine it with the next subvectors.
std::pair<T *, bool> Res =
ResizeAction(ShuffleMask.begin()->first, Mask, /*ForSingleMask=*/false);
SmallBitVector IsBasePoison = isUndefVector<true>(Base, UseMask);
for (unsigned Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) {
if (Mask[Idx] == PoisonMaskElem)
Mask[Idx] = IsBasePoison.test(Idx) ? PoisonMaskElem : Idx;
else
Mask[Idx] = (Res.second ? Idx : Mask[Idx]) + VF;
}
[[maybe_unused]] auto *V = ValueSelect::get<T *>(Base);
assert((!V || GetVF(V) == Mask.size()) &&
"Expected base vector of VF number of elements.");
Prev = Action(Mask, {nullptr, Res.first});
} else if (ShuffleMask.size() == 1) {
// Base is undef and only 1 vector is shuffled - perform the action only for
// single vector, if the mask is not the identity mask.
std::pair<T *, bool> Res = ResizeAction(ShuffleMask.begin()->first, Mask,
/*ForSingleMask=*/true);
if (Res.second)
// Identity mask is found.
Prev = Res.first;
else
Prev = Action(Mask, {ShuffleMask.begin()->first});
} else {
// Base is undef and at least 2 input vectors shuffled - perform 2 vectors
// shuffles step by step, combining shuffle between the steps.
unsigned Vec1VF = GetVF(ShuffleMask.begin()->first);
unsigned Vec2VF = GetVF(VMIt->first);
if (Vec1VF == Vec2VF) {
// No need to resize the input vectors since they are of the same size, we
// can shuffle them directly.
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
if (SecMask[I] != PoisonMaskElem) {
assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
Mask[I] = SecMask[I] + Vec1VF;
}
}
Prev = Action(Mask, {ShuffleMask.begin()->first, VMIt->first});
} else {
// Vectors of different sizes - resize and reshuffle.
std::pair<T *, bool> Res1 = ResizeAction(ShuffleMask.begin()->first, Mask,
/*ForSingleMask=*/false);
std::pair<T *, bool> Res2 =
ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
if (Mask[I] != PoisonMaskElem) {
assert(SecMask[I] == PoisonMaskElem && "Multiple uses of scalars.");
if (Res1.second)
Mask[I] = I;
} else if (SecMask[I] != PoisonMaskElem) {
assert(Mask[I] == PoisonMaskElem && "Multiple uses of scalars.");
Mask[I] = (Res2.second ? I : SecMask[I]) + VF;
}
}
Prev = Action(Mask, {Res1.first, Res2.first});
}
VMIt = std::next(VMIt);
}
[[maybe_unused]] bool IsBaseNotUndef = !IsBaseUndef.all();
// Perform requested actions for the remaining masks/vectors.
for (auto E = ShuffleMask.end(); VMIt != E; ++VMIt) {
// Shuffle other input vectors, if any.
std::pair<T *, bool> Res =
ResizeAction(VMIt->first, VMIt->second, /*ForSingleMask=*/false);
ArrayRef<int> SecMask = VMIt->second;
for (unsigned I = 0, VF = Mask.size(); I < VF; ++I) {
if (SecMask[I] != PoisonMaskElem) {
assert((Mask[I] == PoisonMaskElem || IsBaseNotUndef) &&
"Multiple uses of scalars.");
Mask[I] = (Res.second ? I : SecMask[I]) + VF;
} else if (Mask[I] != PoisonMaskElem) {
Mask[I] = I;
}
}
Prev = Action(Mask, {Prev, Res.first});
}
return Prev;
}
void BoUpSLP::detectBooleanizedNodes() {
// An absorbable boolean leaf is a one-use trunc of a wide value or a
// one-use zero-test of a value from [0, 1]; bit 0 of the wide value is the
// i1 lane value in both cases.
auto GetBoolLeafWideTy = [&](const TreeEntry *OpE) -> Type * {
if (OpE->State != TreeEntry::Vectorize || OpE->isAltShuffle() ||
OpE->hasCopyableElements() || !OpE->ReuseShuffleIndices.empty() ||
OpE->CombinedOp != TreeEntry::NotCombinedOp)
return nullptr;
unsigned WideOpIdx;
bool IsZeroTest = OpE->getOpcode() == Instruction::ICmp;
if (OpE->getOpcode() == Instruction::Trunc) {
if (!all_of(OpE->Scalars, [](Value *V) {
return match(V, m_OneUse(m_Trunc(m_Value())));
}))
return nullptr;
WideOpIdx = 0;
} else if (IsZeroTest) {
if (!all_of(OpE->Scalars, [](Value *V) {
auto *CI = dyn_cast<ICmpInst>(V);
return CI && CI->hasOneUse() &&
CI->getPredicate() == ICmpInst::ICMP_NE;
}))
return nullptr;
if (all_of(OpE->getOperand(0),
[](Value *V) { return match(V, m_Zero()); }))
WideOpIdx = 1;
else if (all_of(OpE->getOperand(1),
[](Value *V) { return match(V, m_Zero()); }))
WideOpIdx = 0;
else
return nullptr;
} else {
return nullptr;
}
Type *WideTy = OpE->getOperand(WideOpIdx).front()->getType();
if (!WideTy->isIntegerTy())
return nullptr;
for (Value *W : OpE->getOperand(WideOpIdx)) {
if (W->getType() != WideTy || !isGuaranteedNotToBePoison(W) ||
(IsZeroTest && computeKnownBits(W, *DL).getMaxValue() != 1))
return nullptr;
}
return WideTy;
};
// The wide value of a booleanized node is valid only in bit 0, so every
// tree user must consume it as an i1 vector: either a booleanized and/or
// node (which uses the wide value directly) or a node whose emission
// truncates the operand back to the i1 vector type.
auto IsSafeBoolUser = [&](const TreeEntry *UserTE, unsigned EdgeIdx) {
if (!UserTE->hasState() || UserTE->isAltShuffle())
return false;
if (Instruction::isBinaryOp(UserTE->getOpcode()))
return true;
switch (UserTE->getOpcode()) {
case Instruction::Store:
return EdgeIdx == 0;
case Instruction::InsertElement:
return EdgeIdx == 1;
case Instruction::Select:
return EdgeIdx != 0;
case Instruction::Freeze:
case Instruction::PHI:
return true;
default:
return false;
}
};
// Visit operand nodes before their users so that booleanized and/or nodes
// chain: an and/or node whose operand is itself a booleanized and/or node
// consumes the wide value directly.
for (const std::unique_ptr<TreeEntry> &TE : reverse(VectorizableTree)) {
// Logical and/or on i1 with booleanized wide leaves is emitted in the
// wide type: bit 0 of a bitwise and/or is the and/or of the operands'
// bit 0. The leaf conversions are absorbed and a single trunc of the
// wide vector result restores the i1 lanes. Wide leaves must be
// non-poison: and/or may mask poison in the i1 type, which the wide ops
// would propagate instead.
if (TE->State != TreeEntry::Vectorize || TE->isAltShuffle() ||
(TE->getOpcode() != Instruction::And &&
TE->getOpcode() != Instruction::Or) ||
!TE->getMainOp()->getType()->isIntegerTy(1) ||
!TE->ReuseShuffleIndices.empty() || TE->hasCopyableElements() ||
TE->CombinedOp != TreeEntry::NotCombinedOp ||
// The i1 flags (e.g. disjoint) do not apply to the wide op.
any_of(TE->Scalars,
[](Value *V) {
auto *I = dyn_cast<Instruction>(V);
return !I || I->hasPoisonGeneratingFlags();
}) ||
any_of(TE->getReassocScalars(),
[](Value *V) {
return cast<Instruction>(V)->hasPoisonGeneratingFlags();
}) ||
// Reduction trees are handled by the reduction's own booleanization.
UserIgnoreList ||
(TE->UserTreeIndex.UserTE &&
!IsSafeBoolUser(TE->UserTreeIndex.UserTE,
TE->UserTreeIndex.EdgeIdx)) ||
!all_of(TE->Scalars,
[&](Value *V) { return getTreeEntries(V).size() == 1; }))
continue;
Type *WideTy = nullptr;
bool Bail = false;
for (unsigned I : seq<unsigned>(TE->getNumOperands())) {
const TreeEntry *OpE = getOperandEntry(TE.get(), I);
Type *ColTy = GetBoolLeafWideTy(OpE);
if (!ColTy && isBooleanizedNode(OpE) &&
(OpE->getOpcode() == Instruction::And ||
OpE->getOpcode() == Instruction::Or))
ColTy = IntegerType::get(F->getContext(), MinBWs.lookup(OpE).first);
if (!ColTy || (WideTy && ColTy != WideTy)) {
Bail = true;
break;
}
WideTy = ColTy;
}
if (Bail || !WideTy)
continue;
unsigned WideBW = WideTy->getIntegerBitWidth();
MinBWs.try_emplace(TE.get(), WideBW, /*IsSigned=*/false);
for (unsigned I : seq<unsigned>(TE->getNumOperands()))
MinBWs.try_emplace(getOperandEntry(TE.get(), I), WideBW,
/*IsSigned=*/false);
}
}
InstructionCost
BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
Instruction *RdxRoot) {
// FIXME: support buildvector of the gather nodes with struct types.
if (any_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->isGather() &&
isa<StructType>(TE->Scalars.front()->getType());
})) {
LLVM_DEBUG(
dbgs() << "SLP: rejecting tree with buildvector struct values of size "
<< VectorizableTree.size() << ".\n");
return InstructionCost::getInvalid();
}
// The tree is priced, so its minimal vector nodes are analyzed, including the
// ones, trimmed below as non-profitable. The verdict of the tree itself does
// not matter: the retry with the single-user seeds rebuilds the same narrow
// nodes, and their cost without the tree is not better. The wider nodes are
// not registered: their rejection does not prevent the members from forming
// the profitable narrower node. Same for the trees, rejected before the cost
// estimation, and for the buildvector trees, rejected by the insert/extract
// overhead rather than by the members themselves.
const TreeEntry &Root = getRootNode();
if (!Root.hasState() ||
!isa<InsertElementInst, InsertValueInst>(Root.getMainOp())) {
for (const std::unique_ptr<TreeEntry> &TE : make_filter_range(
VectorizableTree, [](const std::unique_ptr<TreeEntry> &TE) {
return !TE->isGather() && TE->Scalars.size() <= 2;
})) {
for (Value *V : TE->Scalars)
if (isa_and_present<Instruction>(V) && !TE->isCopyableElement(V))
AnalyzedScalars.insert(V);
}
}
SmallDenseMap<const TreeEntry *, InstructionCost> NodesCosts;
SmallPtrSet<Value *, 4> CheckedExtracts;
SmallSetVector<TreeEntry *, 4> GatheredLoadsNodes;
SmallDenseMap<const TreeEntry *, InstructionCost> ExtractCosts;
LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n");
// The V-only-dependent part of the predicate. Same V is commonly seen in
// multiple TEs (shared scalars), so cache the result across calls.
// DeletedNodes is read-only during this cost loop, so caching is safe.
const size_t NumVectScalars = ScalarToTreeEntries.size() + 1;
SmallDenseMap<Value *, bool> ExternalUseVCache;
auto IsExternallyUsedV = [&](Value *V) {
auto [It, Inserted] = ExternalUseVCache.try_emplace(V);
if (!Inserted)
return It->second;
bool Res = false;
if (V->hasOneUse() || V->getType()->isVoidTy()) {
// Res stays false.
} else if (V->hasNUsesOrMore(NumVectScalars)) {
Res = true;
} else if (auto *I = dyn_cast<Instruction>(V)) {
Res = any_of(I->users(), [&](const User *U) {
// store/insertelt v, [cast]U will likely be vectorized.
if (match(U,
m_InsertElt(m_Value(), m_OneUse(m_CastOrSelf(m_Specific(I))),
m_ConstantInt())))
return false;
if (match(U, m_InsertElt(m_Value(), m_Specific(I), m_ConstantInt())))
return false;
if (match(U, m_Store(m_OneUse(m_CastOrSelf(m_Specific(I))), m_Value())))
return false;
if (match(U, m_Store(m_Specific(I), m_Value())))
return false;
ArrayRef<TreeEntry *> Entries = getTreeEntries(U);
if (Entries.empty() && !MustGather.contains(U))
return true;
if (any_of(Entries,
[&](TreeEntry *TE) { return DeletedNodes.contains(TE); }))
return true;
return any_of(ValueToGatherNodes.lookup(U), [&](const TreeEntry *TE) {
return DeletedNodes.contains(TE);
});
});
}
It->second = Res;
return Res;
};
auto IsExternallyUsed = [&](const TreeEntry &TE, Value *V) {
assert(TE.hasState() && !TE.isGather() &&
TE.State != TreeEntry::SplitVectorize && "Expected vector node.");
if (TE.hasCopyableElements() && TE.isCopyableElement(V))
return false;
return IsExternallyUsedV(V);
};
InstructionCost Cost = 0;
SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale;
uint64_t PrevScale = 0;
BasicBlock *PrevVecParent = nullptr;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
TreeEntry &TE = *Ptr;
// No need to count the cost for combined entries, they are combined and
// just skip their cost.
if (TE.State == TreeEntry::CombinedVectorize) {
LLVM_DEBUG(
dbgs() << "SLP: Skipping cost for combined node that starts with "
<< *TE.Scalars[0] << ".\n";
TE.dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
NodesCosts.try_emplace(&TE);
continue;
}
if (TE.hasState() &&
(TE.isGather() || TE.State == TreeEntry::SplitVectorize)) {
if (const TreeEntry *E =
getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars);
E && E->getVectorFactor() == TE.getVectorFactor()) {
// Some gather nodes might be absolutely the same as some vectorizable
// nodes after reordering, need to handle it.
LLVM_DEBUG(dbgs() << "SLP: Adding cost 0 for bundle "
<< shortBundleName(TE.Scalars, TE.Idx) << ".\n"
<< "SLP: Current total cost = " << Cost << "\n");
NodesCosts.try_emplace(&TE);
continue;
}
}
// Exclude cost of gather loads nodes which are not used. These nodes were
// built as part of the final attempt to vectorize gathered loads.
assert((!TE.isGather() || TE.Idx == 0 || TE.UserTreeIndex) &&
"Expected gather nodes with users only.");
InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts);
uint64_t Scale = 0;
bool CostIsFree = C == 0;
// For gather/buildvector (and split-vectorize) entries, prefer the
// per-lane refined scale that accounts for LICM-hoistable insertelements
// when an operand is invariant in the current loop nest but defined in
// an outer loop. This prevents over-costing cross-loop-nest buildvectors.
if (!CostIsFree && !TE.isGather() && TE.hasState()) {
if (PrevVecParent == TE.getMainOp()->getParent()) {
Scale = PrevScale;
C *= Scale;
EntryToScale.try_emplace(&TE, Scale);
}
}
if (!CostIsFree && !Scale) {
Scale = getEntryEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr);
C *= Scale;
EntryToScale.try_emplace(&TE, Scale);
if (!TE.isGather() && TE.hasState()) {
PrevVecParent = TE.getMainOp()->getParent();
PrevScale = Scale;
}
}
Cost += C;
NodesCosts.try_emplace(&TE, C);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle "
<< shortBundleName(TE.Scalars, TE.Idx) << ".\n"
<< "SLP: Current total cost = " << Cost << "\n");
// Add gathered loads nodes to the set for later processing.
if (TE.Idx > 0 && !TE.UserTreeIndex && TE.hasState() &&
TE.getOpcode() == Instruction::Load)
GatheredLoadsNodes.insert(&TE);
if (!TE.isGather() && TE.State != TreeEntry::SplitVectorize &&
!(TE.Idx == 0 && (TE.getOpcode() == Instruction::InsertElement ||
TE.getOpcode() == Instruction::InsertValue ||
TE.getOpcode() == Instruction::Store)) &&
!isa<StructType>(getValueType(TE.Scalars.front(), Opts.slp_revec))) {
// Calculate costs of external uses.
APInt DemandedElts = APInt::getZero(TE.getVectorFactor());
for (Value *V : TE.Scalars) {
if (IsExternallyUsed(TE, V))
DemandedElts.setBit(TE.findLaneForValue(V));
}
if (!DemandedElts.isZero()) {
Type *ScalarTy = TE.Scalars.front()->getType();
auto It = MinBWs.find(&TE);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, TE.getVectorFactor());
InstructionCost ExtCost =
getScalarizationOverhead(*TTI, Opts.slp_revec, ScalarTy,
cast<VectorType>(VecTy), DemandedElts,
/*Insert=*/false,
/*Extract=*/true, CostKind);
if (ExtCost.isValid() && ExtCost != 0) {
if (!Scale)
Scale = getScaleToLoopIterations(TE);
ExtCost *= Scale;
EntryToScale.try_emplace(&TE, Scale);
}
ExtractCosts.try_emplace(&TE, ExtCost);
}
}
}
// Bail out if the cost threshold is negative and cost already below it.
// The splat subtrees may still force extracts of their scalars on top of
// the node cost and have to be trimmed, so do not bail out if there are
// any.
if (getCostThreshold() < 0 && Cost < -getCostThreshold() &&
SplatGatheredScalarsRoots.empty())
return Cost;
// The narrow non-profitable tree in loop? Skip, may cause regressions.
constexpr unsigned PartLimit = 2;
// Measure the narrowness by the width the tree is vectorized at.
const TreeEntry *RootTE = &getRootNode();
unsigned Sz;
if (auto It = MinBWs.find(RootTE); It != MinBWs.end()) {
Sz = It->second.first;
} else {
Sz = std::max<unsigned>(
getVectorElementSize(RootTE->Scalars.front()),
DL->getTypeSizeInBits(
getValueType(RootTE->Scalars.front(), Opts.slp_revec)
->getScalarType()));
}
const unsigned MinVF = getMinVF(Sz);
if (Cost >= -getCostThreshold() &&
getRootNodeScalars().size() * PartLimit <= MinVF &&
(!getRootNode().hasState() ||
(getRootNode().getOpcode() != Instruction::Store &&
LI->getLoopFor(getRootNode().getMainOp()->getParent()))))
return Cost;
// Store the cost + external uses estimation as the first element of the
// tuple, just the cost as the second element of the tuple. Required to return
// correct cost estimation for the tree, extracts are calculated separately.
// Extracts, calculated here, are just quick estimations.
SmallVector<
std::tuple<InstructionCost, InstructionCost, SmallVector<unsigned>>>
SubtreeCosts(VectorizableTree.size());
auto IsNotDeleted = [&](const TreeEntry *TE) {
return !DeletedNodes.contains(TE);
};
// (Re)computes the subtree costs for the nodes that are not deleted.
auto ComputeSubtreeCosts = [&]() {
SubtreeCosts.assign(
VectorizableTree.size(),
std::tuple<InstructionCost, InstructionCost, SmallVector<unsigned>>());
auto UpdateParentNodes =
[&](const TreeEntry *UserTE, const TreeEntry *TE,
InstructionCost TotalCost, InstructionCost Cost,
SmallDenseSet<std::pair<const TreeEntry *, const TreeEntry *>, 4>
&VisitedUser,
bool AddToList = true) {
while (UserTE &&
VisitedUser.insert(std::make_pair(TE, UserTE)).second) {
std::get<0>(SubtreeCosts[UserTE->Idx]) += TotalCost;
std::get<1>(SubtreeCosts[UserTE->Idx]) += Cost;
if (AddToList)
std::get<2>(SubtreeCosts[UserTE->Idx]).push_back(TE->Idx);
UserTE = UserTE->UserTreeIndex.UserTE;
}
};
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
TreeEntry &TE = *Ptr;
if (DeletedNodes.contains(&TE))
continue;
// Combined subnodes are not costed on their own (their cost is 0), but
// must be included into the ancestors' subtree node lists, so that they
// get deleted together with the trimmed combined root.
InstructionCost C = NodesCosts.at(&TE);
InstructionCost ExtractCost = ExtractCosts.lookup(&TE);
std::get<0>(SubtreeCosts[TE.Idx]) += C + ExtractCost;
std::get<1>(SubtreeCosts[TE.Idx]) += C;
if (const TreeEntry *UserTE = TE.UserTreeIndex.UserTE) {
SmallDenseSet<std::pair<const TreeEntry *, const TreeEntry *>, 4>
VisitedUser;
UpdateParentNodes(UserTE, &TE, C + ExtractCost, C, VisitedUser);
}
}
SmallDenseSet<std::pair<const TreeEntry *, const TreeEntry *>, 4> Visited;
for (TreeEntry *TE : make_filter_range(GatheredLoadsNodes, IsNotDeleted)) {
InstructionCost TotalCost = std::get<0>(SubtreeCosts[TE->Idx]);
InstructionCost Cost = std::get<1>(SubtreeCosts[TE->Idx]);
for (Value *V : TE->Scalars) {
const auto Nodes = ValueToGatherNodes.lookup(V);
for (const TreeEntry *BVTE : make_filter_range(Nodes, IsNotDeleted))
UpdateParentNodes(BVTE, TE, TotalCost, Cost, Visited,
/*AddToList=*/false);
}
}
};
ComputeSubtreeCosts();
using ValuesToInsertTy =
SmallDenseMap<const TreeEntry *, SmallVector<Value *>>;
auto GetScalarTy = [&](const TreeEntry *TE) {
Type *ScalarTy = TE->Scalars.front()->getType();
auto It = MinBWs.find(TE);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
return ScalarTy;
};
// Lanes of the subtree scalars used by the surviving gather nodes, and the
// values to materialize in those gathers if the subtree is deleted.
auto FindDemandedElts = [&](TreeEntry *TE, ValuesToInsertTy &ValuesToInsert) {
APInt DemandedElts = APInt::getZero(TE->getVectorFactor());
for (Value *V : TE->Scalars) {
unsigned Pos = TE->findLaneForValue(V);
const auto Nodes = ValueToGatherNodes.lookup(V);
for (const TreeEntry *BVE : make_filter_range(Nodes, IsNotDeleted)) {
DemandedElts.setBit(Pos);
ValuesToInsert.try_emplace(BVE).first->second.push_back(V);
}
}
return DemandedElts;
};
// Cost of materializing the values directly in the surviving gather nodes
// that use them.
auto GetGatherInsertCost = [&](Type *ScalarTy,
const ValuesToInsertTy &ValuesToInsert) {
InstructionCost BVCost = 0;
for (const auto &[BVE, Values] : ValuesToInsert) {
APInt BVDemandedElts = APInt::getZero(BVE->getVectorFactor());
SmallVector<Value *> BVValues(BVE->getVectorFactor(),
PoisonValue::get(ScalarTy));
for (Value *V : Values) {
unsigned Pos = BVE->findLaneForValue(V);
BVValues[Pos] = V;
BVDemandedElts.setBit(Pos);
}
BVCost += getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy,
cast<VectorType>(getWidenedType(ScalarTy, BVE->getVectorFactor())),
BVDemandedElts, /*Insert=*/true, /*Extract=*/false, CostKind,
BVDemandedElts.isAllOnes(), BVValues);
}
return BVCost;
};
auto RecostEntry = [&](const TreeEntry *TE) {
InstructionCost C = getEntryCost(TE, VectorizedVals, CheckedExtracts);
if (!C.isValid() || C == 0)
return C;
uint64_t Scale = EntryToScale.lookup(TE);
if (!Scale)
Scale = getEntryEffectiveScale(*TE, TE->Idx == 0 ? RdxRoot : nullptr);
return C * Scale;
};
// A splat subtree pays off only if some surviving gather node reuses it and
// its price plus the extracts of its scalars used by the remaining scalar
// code plus the current cost of the gather nodes that reuse it is not worse
// than re-emitting those gathers without the subtree.
auto IsSplatSubtreeProfitable = [&](TreeEntry *TE,
ValuesToInsertTy &ValuesToInsert,
InstructionCost &CurrentGathersCost,
InstructionCost &DroppedGathersCost) {
if (FindDemandedElts(TE, ValuesToInsert).isZero())
return false;
APInt ExtractElts = APInt::getZero(TE->getVectorFactor());
for (Instruction *I : make_isa_range<Instruction>(TE->Scalars)) {
if (TE->isCopyableElement(I))
continue;
// Too many users - the scalar is extracted anyway.
if (I->hasNUsesOrMore(UsesLimit) || any_of(I->users(), [&](User *U) {
return none_of(getTreeEntries(U), [&](const TreeEntry *UseTE) {
return !DeletedNodes.contains(UseTE) &&
!TransformedToGatherNodes.contains(UseTE);
});
}))
ExtractElts.setBit(TE->findLaneForValue(I));
}
Type *ScalarTy = GetScalarTy(TE);
InstructionCost KeepCost = getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy,
cast<VectorType>(getWidenedType(ScalarTy, TE->getVectorFactor())),
ExtractElts, /*Insert=*/false, /*Extract=*/true, CostKind);
// Scale the extract cost to the subtree's execution frequency: the
// subtree and gather costs it is compared against are already loop-scaled.
if (KeepCost.isValid() && KeepCost != 0)
KeepCost *= getEntryEffectiveScale(*TE);
// Add the cost of the subtree itself, computed before any trimming:
// trimming of the subtree's own nodes would otherwise make it look
// artificially cheap. The inner nodes of the subtree keep their external
// scalar users too, so their extracts are part of the price as well.
KeepCost += std::get<1>(SubtreeCosts[TE->Idx]);
for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx]))
KeepCost += ExtractCosts.lookup(VectorizableTree[Idx].get());
// The reusing gather nodes currently pay the broadcast cost; without
// the subtree they fall back to plain insertion sequences. Gather
// nodes already erased from NodesCosts are being deleted and do not
// count on either side.
CurrentGathersCost = 0;
for (const auto &[BVE, _] : ValuesToInsert)
CurrentGathersCost += NodesCosts.lookup(BVE);
KeepCost += CurrentGathersCost;
// Re-cost the gather nodes with the subtree tentatively deleted.
DeletedNodes.insert(TE);
SmallVector<TreeEntry *> TempDeleted;
for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx])) {
TreeEntry *Child = VectorizableTree[Idx].get();
if (DeletedNodes.insert(Child).second)
TempDeleted.push_back(Child);
}
DroppedGathersCost = 0;
auto HasCost = [&](const auto &P) { return NodesCosts.contains(P.first); };
for (const auto &[BVE, _] : make_filter_range(ValuesToInsert, HasCost))
DroppedGathersCost += RecostEntry(BVE);
DeletedNodes.erase(TE);
for (TreeEntry *Child : TempDeleted)
DeletedNodes.erase(Child);
// On a cost tie prefer dropping the subtree: the reusing gathers
// materialize the scalars at the same price with fewer instructions.
return KeepCost < DroppedGathersCost;
};
// Re-price the gathers that reused a dropped splat subtree, so the
// remaining subtrees are checked against the costs with it deleted.
auto SyncGatherCosts = [&](const ValuesToInsertTy &ValuesToInsert) {
for (const auto &[BVE, _] : ValuesToInsert)
if (!DeletedNodes.contains(BVE))
NodesCosts[BVE] = RecostEntry(BVE);
};
// Deletes the subtree. The per-node costs are erased eagerly only where the
// total is recomputed from them right after the deletion.
auto DeleteSubtree = [&](TreeEntry *TE, bool EraseCosts) {
DeletedNodes.insert(TE);
if (EraseCosts)
NodesCosts.erase(TE);
for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx])) {
TreeEntry *Child = VectorizableTree[Idx].get();
DeletedNodes.insert(Child);
if (EraseCosts)
NodesCosts.erase(Child);
}
};
// Gathered loads subtrees left without surviving gather users are dead.
auto DropDeadGatheredLoads = [&](bool EraseCosts) {
for (TreeEntry *TE : make_filter_range(GatheredLoadsNodes, IsNotDeleted)) {
ValuesToInsertTy ValuesToInsert;
if (!FindDemandedElts(TE, ValuesToInsert).isZero())
continue;
DeleteSubtree(TE, EraseCosts);
}
};
// Drops the splat subtrees that are unused by the surviving gathers or do
// not pay off; returns true if anything was dropped.
auto DropUnprofitableSplatSubtrees = [&](bool EraseCosts) {
bool Dropped = false;
for (TreeEntry *TE :
make_filter_range(SplatGatheredScalarsRoots, IsNotDeleted)) {
ValuesToInsertTy ValuesToInsert;
InstructionCost CurrentGathersCost = 0, DroppedGathersCost = 0;
if (IsSplatSubtreeProfitable(TE, ValuesToInsert, CurrentGathersCost,
DroppedGathersCost))
continue;
DeleteSubtree(TE, EraseCosts);
SyncGatherCosts(ValuesToInsert);
Dropped = true;
}
return Dropped;
};
auto SumNodesCosts = [&]() {
InstructionCost C = 0;
for (const auto &P : NodesCosts)
C += P.second;
return C;
};
// The splat subtrees are speculative. The ones that do not pay off even in
// the full tree are dropped before the trimming: the reusing gathers are
// priced for the broadcast from the subtree vector while the subtree is
// around, which distorts the trimming decisions of the main tree, and fewer
// surviving gathers after the trimming can only make keeping a subtree less
// profitable.
if (DropUnprofitableSplatSubtrees(/*EraseCosts=*/true)) {
DropDeadGatheredLoads(/*EraseCosts=*/true);
ComputeSubtreeCosts();
Cost = SumNodesCosts();
}
using CostIndicesTy =
std::pair<TreeEntry *, std::tuple<InstructionCost, InstructionCost,
SmallVector<unsigned>>>;
struct FirstGreater {
bool operator()(const CostIndicesTy &LHS, const CostIndicesTy &RHS) const {
return std::get<0>(LHS.second) < std::get<0>(RHS.second) ||
(std::get<0>(LHS.second) == std::get<0>(RHS.second) &&
LHS.first->Idx < RHS.first->Idx);
}
};
PriorityQueue<CostIndicesTy, SmallVector<CostIndicesTy>, FirstGreater>
Worklist;
// Combined subnodes are not trimmed on their own, only as a whole combined
// node, so only the root nodes are checked and included into the worklist.
for (const auto [Idx, P] :
make_filter_range(enumerate(SubtreeCosts), [&](const auto &Elem) {
return VectorizableTree[Elem.index()]->State !=
TreeEntry::CombinedVectorize;
}))
Worklist.emplace(VectorizableTree[Idx].get(), P);
// Narrow store trees with non-profitable immediate values - exit.
if (!UserIgnoreList && getRootNode().getVectorFactor() < MinVF &&
getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::Store &&
(Worklist.top().first->Idx == 0 || Worklist.top().first->Idx == 1))
return Cost;
// Node costs and deleted nodes before any trimming; the restored tree is
// priced from them if the trimming is reverted.
const SmallDenseMap<const TreeEntry *, InstructionCost> OrigNodesCosts =
NodesCosts;
const SmallPtrSet<const TreeEntry *, 8> OrigDeletedNodes = DeletedNodes;
bool Changed = false;
bool PreferTrimmedTree = false;
while (!Worklist.empty() && std::get<0>(Worklist.top().second) > 0) {
TreeEntry *TE = Worklist.top().first;
if (TE->isGather() || TE->Idx == 0 || DeletedNodes.contains(TE) ||
isa<StructType>(getValueType(TE->Scalars.front(), Opts.slp_revec)) ||
// Splat subtree roots are decided by the keep/drop check below: their
// scalars are materialized in the reusing splat gathers, not in a
// gather of the root's own scalars.
is_contained(SplatGatheredScalarsRoots, TE) ||
// Exit early if the parent node is split node and any of scalars is
// used in other split nodes.
(TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->State == TreeEntry::SplitVectorize &&
any_of(TE->Scalars, [&](Value *V) {
ArrayRef<TreeEntry *> Entries = getSplitTreeEntries(V);
return Entries.size() > 1;
}))) {
Worklist.pop();
continue;
}
// Skip inversed compare nodes, they cannot be transformed to buildvectors.
if (TE->State == TreeEntry::Vectorize && !TE->isAltShuffle() &&
(TE->getOpcode() == Instruction::ICmp ||
TE->getOpcode() == Instruction::FCmp) &&
any_of(make_isa_range<CmpInst>(TE->Scalars), [&](CmpInst *I) {
return I->getPredicate() !=
cast<CmpInst>(TE->getMainOp())->getPredicate();
})) {
Worklist.pop();
continue;
}
// A zext feeding a bit pack is priced as part of the pack: its vectorized
// form makes the compaction back to the source width free, so keep it.
if (TE->UserTreeIndex && TE->UserTreeIndex.EdgeIdx == 0 &&
TE->UserTreeIndex.UserTE->CombinedOp == TreeEntry::ReducedBitPack &&
getZExtSrcWidth(*TE) != 0) {
Worklist.pop();
continue;
}
// Calculate the gather cost of the root node.
InstructionCost TotalSubtreeCost = std::get<0>(Worklist.top().second);
InstructionCost SubtreeCost = std::get<1>(Worklist.top().second);
if (TotalSubtreeCost < TE->Scalars.size()) {
Worklist.pop();
continue;
}
if (!TransformedToGatherNodes.empty()) {
for (unsigned Idx : std::get<2>(Worklist.top().second)) {
auto It = TransformedToGatherNodes.find(VectorizableTree[Idx].get());
if (It != TransformedToGatherNodes.end()) {
TotalSubtreeCost -= std::get<0>(SubtreeCosts[Idx]);
SubtreeCost -= std::get<1>(SubtreeCosts[Idx]);
TotalSubtreeCost += It->second;
SubtreeCost += It->second;
}
}
}
if (TotalSubtreeCost < 0 || TotalSubtreeCost < TE->Scalars.size()) {
Worklist.pop();
continue;
}
const unsigned EntryVF = TE->getVectorFactor();
APInt DemandedElts = APInt::getZero(EntryVF);
for (auto [Idx, V] : enumerate(TE->Scalars)) {
if (!isConstant(V))
DemandedElts.setBit(Idx);
}
Type *ScalarTy = getValueType(TE->Scalars.front(), Opts.slp_revec);
auto It = MinBWs.find(TE);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, EntryVF);
InstructionCost GatherCost = getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, cast<VectorType>(VecTy), DemandedElts,
/*Insert=*/true, /*Extract=*/false, CostKind);
SmallVector<int> Mask;
if (!TE->ReorderIndices.empty() &&
TE->State != TreeEntry::CompressVectorize &&
(TE->State != TreeEntry::StridedVectorize || !TE->isReverse() ||
isWidenedStridedLoad(TE))) {
SmallVector<int> NewMask;
if (TE->getOpcode() == Instruction::Store) {
// For stores the order is actually a mask.
NewMask.resize(TE->ReorderIndices.size());
copy(TE->ReorderIndices, NewMask.begin());
} else {
inversePermutation(TE->ReorderIndices, NewMask);
}
addMask(Mask, NewMask);
}
if (!TE->ReuseShuffleIndices.empty())
addMask(Mask, TE->ReuseShuffleIndices);
if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, EntryVF))
GatherCost += getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(VecTy), CostKind, Mask);
// If all scalars are reused in gather node(s) or other vector nodes, there
// might be extra cost for inserting them.
if ((!TE->hasState() || !TE->isAltShuffle()) &&
all_of(TE->Scalars, [&](Value *V) {
return (TE->hasCopyableElements() && TE->isCopyableElement(V)) ||
isConstant(V) || isGathered(V) || getTreeEntries(V).size() > 1;
}))
GatherCost *= 2;
// The fmuls, priced as fused into their vectorized fadd/fsub users, stay
// scalar and unfused in the gathered form.
GatherCost += getUnfusedFMulsPenalty(*TE);
// Erase subtree if it is non-profitable.
ArrayRef<unsigned> Nodes = std::get<2>(Worklist.top().second);
// Prefer trimming equal-cost alternate-shuffle subtrees rooted at binary
// ops: alt-shuffles introduce runtime shuffle overhead that the cost model
// may underestimate. Skip if the subtree contains ExtractElement nodes,
// since those operate on already-materialized vectors where the cost model
// is more accurate.
auto IsEqualCostAltShuffleToTrim = [&]() {
return TotalSubtreeCost == GatherCost && TE->hasState() &&
TE->isAltShuffle() && Instruction::isBinaryOp(TE->getOpcode()) &&
none_of(Nodes, [&](unsigned Idx) {
return VectorizableTree[Idx]->hasState() &&
VectorizableTree[Idx]->getOpcode() ==
Instruction::ExtractElement;
});
};
// Non-power-of-2 entries may have inflated costs - add a margin of 1
// before trimming to avoid over-pruning.
bool HasNonPowerOf2 = any_of(Nodes, [&](unsigned Idx) {
return !has_single_bit(VectorizableTree[Idx]->Scalars.size());
});
InstructionCost TrimMargin = HasNonPowerOf2 ? 1 : 0;
if (TotalSubtreeCost > GatherCost + TrimMargin ||
IsEqualCostAltShuffleToTrim()) {
PreferTrimmedTree |= TotalSubtreeCost == GatherCost;
// If the remaining tree is just a buildvector - exit, it will cause
// endless attempts to vectorize. When the tree is already profitable,
// skip trimming this node and let the post-loop logic (including
// gathered loads processing) decide.
if (getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::InsertElement &&
TE->Idx == 1) {
if (Cost < -getCostThreshold()) {
LLVM_DEBUG(dbgs() << "SLP: Skipping trim of node " << TE->Idx
<< " - tree already profitable with cost " << Cost
<< ".\n");
Worklist.pop();
continue;
}
return InstructionCost::getInvalid();
}
LLVM_DEBUG(dbgs() << "SLP: Trimming unprofitable subtree at node "
<< TE->Idx << " with cost "
<< std::get<0>(Worklist.top().second)
<< " and gather cost " << GatherCost << ".\n");
if (TE->UserTreeIndex) {
TransformedToGatherNodes.try_emplace(TE, GatherCost);
NodesCosts.erase(TE);
} else {
DeletedNodes.insert(TE);
TransformedToGatherNodes.erase(TE);
NodesCosts.erase(TE);
}
for (unsigned Idx : Nodes) {
TreeEntry &ChildTE = *VectorizableTree[Idx];
DeletedNodes.insert(&ChildTE);
TransformedToGatherNodes.erase(&ChildTE);
NodesCosts.erase(&ChildTE);
}
Changed = true;
}
Worklist.pop();
}
// The VF=2 instruction-count veto rejects the whole tree when the vector code
// has more instructions than the scalar code. The auxiliary subtrees - splat
// gather roots and gathered loads - are optional: the surviving gathers can
// re-materialize their scalars. Drop the reused ones while the tree exceeds
// the budget so a kept subtree cannot reject the main tree. The gate mirrors
// the veto conditions.
const bool ApplyInstCountVeto =
(!SplatGatheredScalarsRoots.empty() || !GatheredLoadsNodes.empty()) &&
CostKind != TTI::TCK_CodeSize && Opts.slp_inst_count_check &&
TTI->preferSLPInstCountCheck() && getRootNode().getVectorFactor() == 2 &&
getCostThreshold() == 0 &&
(!Opts.slp_revec ||
!isa<VectorType>(getRootNodeScalars().front()->getType()));
const Loop *InstCountLoop = nullptr;
if (ApplyInstCountVeto && Opts.slp_cost_loop_trip_count != 0 &&
getRootNode().hasState())
InstCountLoop = LI->getLoopFor(getRootNode().getMainOp()->getParent());
auto TrimAuxSubtreesForInstCount = [&](InstructionCost &Cost,
bool UseCurrentNodeCosts) {
if (!ApplyInstCountVeto || bypassesInstCountCheck(Cost))
return;
// Current cost of the subtree rooted at TE from the per-node costs.
auto GetCurrentSubtreeCost = [&](const TreeEntry *TE) {
InstructionCost C = NodesCosts.lookup(TE);
for (unsigned Idx : std::get<2>(SubtreeCosts[TE->Idx]))
C += NodesCosts.lookup(VectorizableTree[Idx].get());
return C;
};
auto DropReusedSubtree = [&](TreeEntry *TE, bool DeadOnly) {
if (DeletedNodes.contains(TE) || TransformedToGatherNodes.contains(TE))
return false;
ValuesToInsertTy ValuesToInsert;
const bool IsDead = FindDemandedElts(TE, ValuesToInsert).isZero();
if (!IsDead && DeadOnly)
return false;
InstructionCost CurrentGathersCost = 0;
for (const auto &[BVE, _] : ValuesToInsert)
CurrentGathersCost += NodesCosts.lookup(BVE);
DeleteSubtree(TE, /*EraseCosts=*/false);
// All reusers were dropped: the subtree is dead. Without current node
// costs the total conservatively keeps the subtree cost.
if (IsDead) {
if (UseCurrentNodeCosts)
Cost -= GetCurrentSubtreeCost(TE);
return true;
}
InstructionCost DroppedGathersCost = 0;
for (const auto &[BVE, _] : ValuesToInsert)
if (NodesCosts.contains(BVE))
DroppedGathersCost += RecostEntry(BVE);
// The pre-trimming subtree cost matches the total only while the tree
// is intact or restored; an accepted trimming leaves the total holding
// just the current costs of the surviving subtree nodes.
InstructionCost SubtreeCost = UseCurrentNodeCosts
? GetCurrentSubtreeCost(TE)
: std::get<1>(SubtreeCosts[TE->Idx]);
Cost += DroppedGathersCost - CurrentGathersCost - SubtreeCost;
SyncGatherCosts(ValuesToInsert);
return true;
};
// Recompute the counts only after an actual drop. Dropping a root can
// kill the reusers of a root from the other list, so repeat until no more
// drops; dead subtrees are removed even under the budget.
uint64_t NumScalar = getNumScalarInsts(InstCountLoop);
uint64_t NumVector = getNumVectorInsts(InstCountLoop);
bool Retry;
do {
Retry = false;
for (ArrayRef<TreeEntry *> Roots :
{ArrayRef<TreeEntry *>(SplatGatheredScalarsRoots),
GatheredLoadsNodes.getArrayRef()})
for (TreeEntry *TE : Roots)
if (DropReusedSubtree(TE, /*DeadOnly=*/NumVector <= NumScalar)) {
NumScalar = getNumScalarInsts(InstCountLoop);
NumVector = getNumVectorInsts(InstCountLoop);
Retry = true;
}
} while (Retry);
};
if (!Changed) {
// The splat subtrees are not linked to the tree root, so their cost is
// not included in the root's subtree cost; add it explicitly. Drop the
// unprofitable ones instead of letting them reject the whole tree.
InstructionCost TotalCost = std::get<1>(SubtreeCosts.front());
for (TreeEntry *TE :
make_filter_range(SplatGatheredScalarsRoots, [&](TreeEntry *TE) {
return !DeletedNodes.contains(TE);
})) {
ValuesToInsertTy ValuesToInsert;
InstructionCost CurrentGathersCost = 0, DroppedGathersCost = 0;
if (IsSplatSubtreeProfitable(TE, ValuesToInsert, CurrentGathersCost,
DroppedGathersCost)) {
TotalCost += std::get<1>(SubtreeCosts[TE->Idx]);
continue;
}
DeleteSubtree(TE, /*EraseCosts=*/false);
// The gather nodes that reused the subtree are re-emitted without it.
TotalCost += DroppedGathersCost - CurrentGathersCost;
SyncGatherCosts(ValuesToInsert);
}
DropDeadGatheredLoads(/*EraseCosts=*/false);
TrimAuxSubtreesForInstCount(TotalCost, /*UseCurrentNodeCosts=*/false);
return TotalCost;
}
SmallPtrSet<TreeEntry *, 4> SubtreesToDelete;
InstructionCost LoadsExtractsCost = 0;
// Check if all gather nodes that reuse the splat subtrees are marked for
// deletion. In this case the whole splat subtree must be deleted. If only
// some of the gathers are trimmed, keeping the subtree still costs its full
// price plus the extracts of the scalars used by the remaining scalar code,
// while the surviving gathers can materialize the splatted scalars
// directly. Drop the subtree if it does not pay off.
for (TreeEntry *TE :
make_filter_range(SplatGatheredScalarsRoots, [&](TreeEntry *TE) {
return !DeletedNodes.contains(TE);
})) {
ValuesToInsertTy ValuesToInsert;
InstructionCost CurrentGathersCost, DroppedGathersCost;
if (IsSplatSubtreeProfitable(TE, ValuesToInsert, CurrentGathersCost,
DroppedGathersCost))
continue;
SubtreesToDelete.insert(TE);
NodesCosts.erase(TE);
}
// Check if all loads of gathered loads nodes are marked for deletion. In this
// case the whole gathered loads subtree must be deleted.
// Also, try to account for extracts, which might be required, if only part of
// gathered load must be vectorized. Keep partially vectorized nodes, if
// extracts are cheaper than gathers.
for (TreeEntry *TE :
make_filter_range(GatheredLoadsNodes, [&](TreeEntry *TE) {
return !DeletedNodes.contains(TE) &&
!TransformedToGatherNodes.contains(TE);
})) {
SubtreesToDelete.insert(TE);
// All loads are removed from gathered? Need to delete the subtree.
ValuesToInsertTy ValuesToInsert;
APInt DemandedElts = FindDemandedElts(TE, ValuesToInsert);
if (!DemandedElts.isZero()) {
Type *ScalarTy = GetScalarTy(TE);
auto *VecTy = getWidenedType(ScalarTy, TE->getVectorFactor());
InstructionCost ExtractsCost = getScalarizationOverhead(
*TTI, Opts.slp_revec, ScalarTy, cast<VectorType>(VecTy), DemandedElts,
/*Insert=*/false, /*Extract=*/true, CostKind);
InstructionCost BVCost = GetGatherInsertCost(ScalarTy, ValuesToInsert);
if (ExtractsCost < BVCost) {
LoadsExtractsCost += ExtractsCost;
SubtreesToDelete.erase(TE);
continue;
}
LoadsExtractsCost += BVCost;
}
NodesCosts.erase(TE);
}
// Deleted all subtrees rooted at gathered loads nodes or splat subtrees.
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (TE->UserTreeIndex &&
SubtreesToDelete.contains(TE->UserTreeIndex.UserTE)) {
DeletedNodes.insert(TE.get());
NodesCosts.erase(TE.get());
SubtreesToDelete.insert(TE.get());
}
if (SubtreesToDelete.contains(TE.get()))
DeletedNodes.insert(TE.get());
}
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
if (!TE->UserTreeIndex && TransformedToGatherNodes.contains(TE.get())) {
assert(TE->getOpcode() == Instruction::Load && "Expected load only.");
continue;
}
if (DeletedNodes.contains(TE.get()))
continue;
// Gather costs depend on the set of vectorized nodes available for
// reuse, which changes during trimming, so recalculate them for all
// gather nodes, not just for the transformed ones.
if (TE->isGather() || !NodesCosts.contains(TE.get()))
NodesCosts[TE.get()] = RecostEntry(TE.get());
}
LLVM_DEBUG(dbgs() << "SLP: Recalculate costs after tree trimming.\n");
InstructionCost NewCost = 0;
for (const auto &P : NodesCosts) {
NewCost += P.second;
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << P.second << " for bundle "
<< shortBundleName(P.first->Scalars, P.first->Idx)
<< ".\n"
<< "SLP: Current total cost = " << NewCost << "\n");
}
// The trimming is reverted if the restored tree is not more expensive. The
// restored tree is priced the same way as the trimmed one, from the per-node
// costs: adjusting the pre-trimming total incrementally is not exact with
// the auxiliary subtrees around, since the subtree costs recorded before the
// trimming fold in the gathered loads they reuse, and the gathers inside a
// subtree change their price once their reuse sources are dropped.
auto ComputeRestoredCost = [&]() {
DeletedNodes = OrigDeletedNodes;
TransformedToGatherNodes.clear();
NodesCosts = OrigNodesCosts;
if (!SplatGatheredScalarsRoots.empty()) {
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree)
if (TE->isGather() && !DeletedNodes.contains(TE.get()))
NodesCosts[TE.get()] = RecostEntry(TE.get());
DropUnprofitableSplatSubtrees(/*EraseCosts=*/false);
// Dropping the splat subtrees may leave the gathered loads subtrees
// built for them without surviving gather users.
DropDeadGatheredLoads(/*EraseCosts=*/false);
for (const TreeEntry *TE : DeletedNodes)
NodesCosts.erase(TE);
}
return SumNodesCosts();
};
const InstructionCost TrimmedCost = NewCost + LoadsExtractsCost;
auto TrimmedDeletedNodes = DeletedNodes;
auto TrimmedTransformedNodes = TransformedToGatherNodes;
auto TrimmedNodesCosts = NodesCosts;
const InstructionCost RestoredCost = ComputeRestoredCost();
LLVM_DEBUG(dbgs() << "SLP: Trimmed tree cost = " << TrimmedCost
<< ", restored tree cost = " << RestoredCost << ".\n");
const bool Reverted = TrimmedCost > RestoredCost ||
(!PreferTrimmedTree && TrimmedCost == RestoredCost);
if (Reverted) {
NewCost = RestoredCost;
} else {
DeletedNodes = std::move(TrimmedDeletedNodes);
TransformedToGatherNodes = std::move(TrimmedTransformedNodes);
NodesCosts = std::move(TrimmedNodesCosts);
// If the remaining tree is just a buildvector - exit, it will cause
// endless attempts to vectorize.
if (VectorizableTree.size() >= 2 && getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::InsertElement &&
TransformedToGatherNodes.contains(VectorizableTree[1].get()))
return InstructionCost::getInvalid();
if (VectorizableTree.size() >= 3 && getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::InsertElement &&
VectorizableTree[1]->hasState() &&
VectorizableTree[1]->State == TreeEntry::Vectorize &&
(VectorizableTree[1]->getOpcode() == Instruction::ZExt ||
VectorizableTree[1]->getOpcode() == Instruction::SExt ||
VectorizableTree[1]->getOpcode() == Instruction::Trunc) &&
TransformedToGatherNodes.contains(VectorizableTree[2].get()))
return InstructionCost::getInvalid();
}
TrimAuxSubtreesForInstCount(NewCost, /*UseCurrentNodeCosts=*/true);
return NewCost;
}
namespace {
/// Data type for handling buildvector sequences with the reused scalars from
/// other tree entries.
template <typename T> struct ShuffledInsertData {
/// List of insertelements to be replaced by shuffles.
SmallVector<InsertElementInst *> InsertElements;
/// The parent vectors and shuffle mask for the given list of inserts.
MapVector<T, SmallVector<int>> ValueMasks;
};
} // namespace
/// \returns true if more values with the scalar type \p Ty than registers are
/// live at some point of the block \p BB. The blocks larger than the scheduling
/// budget are skipped to save compile time.
static bool hasHighRegisterPressure(const VectorizeOptions &Opts,
const BasicBlock &BB, Type *Ty,
const TargetTransformInfo &TTI) {
if (hasNItemsOrMore(BB, Opts.slp_schedule_budget))
return false;
const unsigned NumRegs = TTI.getNumberOfRegisters(
TTI.getRegisterClassForType(/*Vector=*/false, Ty));
auto IsLiveCandidate = [&](const Value *V) {
return isa<Instruction, Argument>(V) && V->getType()->getScalarType() == Ty;
};
SmallPtrSet<const Value *, 32> Live;
for (const Instruction &I : reverse(BB)) {
// The operands of the PHIs are live in the predecessors.
if (isa<PHINode>(I))
break;
Live.erase(&I);
Live.insert_range(make_filter_range(I.operand_values(), IsLiveCandidate));
if (Live.size() > NumRegs)
return true;
}
return false;
}
InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals,
InstructionCost ReductionCost,
Instruction *RdxRoot) {
// Reject vectorization if the vector code would produce more instructions
// than the scalar code. The cost model may underestimate overhead from
// shuffles, inserts, and extracts.
// FIXME: remove this as soon as correct fractional model is landed for all
// targets.
if (CostKind != TTI::TCK_CodeSize && Opts.slp_inst_count_check &&
TTI->preferSLPInstCountCheck() && getRootNode().getVectorFactor() == 2 &&
getCostThreshold() == 0 &&
(!Opts.slp_revec ||
!isa<VectorType>(getRootNodeScalars().front()->getType()))) {
// Loop containing the tree root; null for flat code or disabled
// loop-aware modeling. Shared by both calls below.
const Loop *TreeLoop = nullptr;
if (Opts.slp_cost_loop_trip_count != 0 && getRootNode().hasState())
TreeLoop = LI->getLoopFor(getRootNode().getMainOp()->getParent());
uint64_t NumScalar = getNumScalarInsts(TreeLoop);
uint64_t NumVector = getNumVectorInsts(TreeLoop);
// In loops, the extracts are not scaled by the trip count and just break
// the ties of the scaled counts. Ignore them under the high register
// pressure: the vector code holds several lanes per register.
if (NumVector > NumScalar && TreeLoop && !ExternalUses.empty()) {
const uint64_t NumNoExtracts =
getNumVectorInsts(TreeLoop, /*CountExtracts=*/false);
Instruction *Root = getRootNode().getMainOp();
if (NumNoExtracts <= NumScalar &&
hasHighRegisterPressure(Opts, *Root->getParent(),
getValueType(Root, Opts.slp_revec), *TTI))
NumVector = NumNoExtracts;
}
LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector
<< " scalar=" << NumScalar << "\n");
if (NumVector > NumScalar && !bypassesInstCountCheck(TreeCost)) {
LLVM_DEBUG(dbgs() << "SLP: Rejecting tree: vector inst count "
<< NumVector << " > scalar inst count " << NumScalar
<< ".\n");
return InstructionCost::getInvalid();
}
}
InstructionCost Cost = TreeCost;
SmallDenseMap<std::tuple<const TreeEntry *, Value *, Instruction *>, uint64_t>
EntryToScale;
auto ScaleCost = [&](InstructionCost C, const TreeEntry &TE,
Value *Scalar = nullptr, Instruction *U = nullptr) {
if (!C.isValid() || C == 0)
return C;
uint64_t &Scale =
EntryToScale.try_emplace(std::make_tuple(&TE, Scalar, U), 0)
.first->getSecond();
if (!Scale)
Scale = getScaleToLoopIterations(TE, Scalar, U);
LLVM_DEBUG(dbgs() << "Scale " << Scale << " For entry " << TE.Idx << "\n");
return C * Scale;
};
Instruction *ReductionRoot = RdxRoot;
if (UserIgnoreList) {
// Scale reduction cost to the factor of the loop nest trip count.
ReductionCost = ScaleCost(ReductionCost, getRootNode(), /*Scalar=*/nullptr,
ReductionRoot);
}
// Add the cost for reduction.
Cost += ReductionCost;
if (any_of(ExternalUses, [](const ExternalUser &EU) {
return isa<StructType>(EU.Scalar->getType()) &&
(EU.E.Idx != 0 || EU.E.State != TreeEntry::Vectorize ||
EU.E.getOpcode() != Instruction::InsertValue) &&
!isa_and_nonnull<ExtractValueInst>(EU.User);
}))
return InstructionCost::getInvalid();
// Skip trees, which are non-profitable even if there are insertelements with
// external uses.
constexpr unsigned CostLimit = 100;
if (Cost >= -getCostThreshold() + CostLimit &&
(VectorizableTree.size() - DeletedNodes.size()) *
getRootNode().getVectorFactor() <
CostLimit)
return Cost;
if (Cost >= -getCostThreshold() &&
none_of(ExternalUses, [](const ExternalUser &EU) {
return isa_and_nonnull<InsertElementInst>(EU.User);
}))
return Cost;
SmallDenseSet<std::pair<Value *, SmallVector<unsigned, 1>>, 16>
ExtractCostCalculated;
InstructionCost ExtractCost = 0;
SmallVector<ShuffledInsertData<const TreeEntry *>> ShuffledInserts;
SmallVector<APInt> DemandedElts;
SmallDenseSet<Value *, 4> UsedInserts;
DenseSet<std::pair<const TreeEntry *, Type *>> VectorCasts;
std::optional<DenseMap<Value *, unsigned>> ValueToExtUses;
DenseMap<const TreeEntry *, DenseSet<Value *>> ExtractsCount;
SmallPtrSet<Value *, 4> ScalarOpsFromCasts;
// Keep track {Scalar, Index, User} tuple.
// On AArch64, this helps in fusing a mov instruction, associated with
// extractelement, with fmul in the backend so that extractelement is free.
SmallVector<std::tuple<Value *, User *, int>, 4> ScalarUserAndIdx;
// Record every external use: a missing entry is indistinguishable from
// lane 0 and is priced as a free extract by the extract-fusion cost model.
for (ExternalUser &EU : ExternalUses)
ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
// Detect external uses that drive address computations: the scalar (through
// an optional single-use index-promotion cast) is used as a GEP index.
bool AllUsersGEPSWithStoresLoads = true;
SmallVector<const Value *> Pointers;
Type *UserScalarTy = nullptr;
for (ExternalUser &EU : ExternalUses) {
Value *Usr = EU.User;
if (Usr && match(Usr, m_OneUse(m_ZExtOrSExt(m_Value()))))
Usr = cast<Instruction>(Usr)->user_back();
auto *User = dyn_cast_if_present<GetElementPtrInst>(Usr);
// Only a GEP that feeds a single load/store of a fixed access type drives
// a real memory address computation.
Type *AccessTy = nullptr;
if (User && User->hasOneUse() &&
isa<LoadInst, StoreInst>(User->user_back()))
AccessTy = getValueType(User->user_back(), Opts.slp_revec);
if (AccessTy && !isa<ScalableVectorType>(AccessTy) &&
(!UserScalarTy || UserScalarTy == AccessTy)) {
UserScalarTy = AccessTy;
Pointers.push_back(User);
} else {
AllUsersGEPSWithStoresLoads = false;
break;
}
}
// Pre-pass: for each externally-used scalar, find the basic block at which
// the extractelement will be placed by codegen. This mirrors what
// vectorizeTree does: the extract is placed at the nearest common dominator
// of all effective use sites. For a non-PHI user the effective site is the
// user's own block; for a PHI user it is the incoming block for the scalar
// operand (the predecessor of the PHI on the edge that carries the scalar).
// Using the NCD of all effective sites rather than the first-encountered
// user's block makes the extract-cost scale order-independent and correct
// even when users live in different loop nests.
SmallDenseMap<Value *, BasicBlock *> ScalarToExtractBlock;
for (const ExternalUser &EU :
make_filter_range(ExternalUses, [](const ExternalUser &EU) {
return EU.User && !isa<InsertElementInst>(EU.User);
})) {
if (EphValues.count(EU.User))
continue;
BasicBlock *UserParent = cast<Instruction>(EU.User)->getParent();
if (!DT->isReachableFromEntry(UserParent) || UserParent->isEHPad() ||
isa<UnreachableInst>(UserParent->getTerminator()))
continue;
BasicBlock *UseBlock = nullptr;
if (auto *PHI = dyn_cast<PHINode>(EU.User)) {
// When the PHI itself is inside a loop, the extractelement is placed
// in the incoming block for the scalar operand (the predecessor edge),
// not in the PHI's own block. This applies to LCSSA phis at an inner-
// loop exit that are still inside an outer loop: the incoming block is
// in the inner loop while the PHI block is in the outer loop.
// When the PHI is outside all loops (a true loop-exit phi), codegen
// uses a vector phi at the exit block and the extract stays there
// (scale = 1), so we keep the PHI's own block as the effective site.
if (LI->getLoopFor(PHI->getParent())) {
for (unsigned Idx : seq<unsigned>(PHI->getNumIncomingValues())) {
if (PHI->getIncomingValue(Idx) != EU.Scalar)
continue;
BasicBlock *InBB = PHI->getIncomingBlock(Idx);
UseBlock =
UseBlock ? DT->findNearestCommonDominator(UseBlock, InBB) : InBB;
}
}
if (!UseBlock)
UseBlock = cast<Instruction>(EU.User)->getParent();
} else {
UseBlock = cast<Instruction>(EU.User)->getParent();
}
auto [It, Inserted] = ScalarToExtractBlock.try_emplace(EU.Scalar, UseBlock);
if (!Inserted && It->second && UseBlock)
It->second = DT->findNearestCommonDominator(It->second, UseBlock);
}
SmallDenseSet<std::pair<Value *, Value *>, 8> CheckedScalarUser;
for (ExternalUser &EU : ExternalUses) {
LLVM_DEBUG(dbgs() << "SLP: Computing cost for external use of TreeEntry "
<< EU.E.Idx << " in lane " << EU.Lane << "\n");
LLVM_DEBUG(if (EU.User) dbgs() << " User:" << *EU.User << "\n";
else dbgs() << " User: nullptr\n");
LLVM_DEBUG(dbgs() << " Use: " << EU.Scalar->getNameOrAsOperand() << "\n");
// Uses by ephemeral values are free (because the ephemeral value will be
// removed prior to code generation, and so the extraction will be
// removed as well).
if (EphValues.count(EU.User))
continue;
// Check if the scalar for the given user or all users is accounted already.
if (!CheckedScalarUser.insert(std::make_pair(EU.Scalar, EU.User)).second ||
(EU.User &&
CheckedScalarUser.contains(std::make_pair(EU.Scalar, nullptr))))
continue;
// Used in unreachable blocks or in EH pads (rarely executed) or is
// terminated with unreachable instruction.
if (BasicBlock *UserParent =
EU.User ? cast<Instruction>(EU.User)->getParent() : nullptr;
UserParent &&
(!DT->isReachableFromEntry(UserParent) || UserParent->isEHPad() ||
isa<UnreachableInst>(UserParent->getTerminator())))
continue;
// No extract cost for vector "scalar" if REVEC is disabled.
if (isVectorizedTy(EU.Scalar->getType()) &&
(!Opts.slp_revec ||
(EU.E.hasState() && EU.E.getOpcode() == Instruction::InsertElement)))
continue;
if (isa<InsertValueInst>(EU.Scalar))
continue;
// If found user is an insertelement, do not calculate extract cost but try
// to detect it as a final shuffled/identity match.
// TODO: what if a user is insertvalue when REVEC is enabled?
if (auto *VU = dyn_cast_if_present<InsertElementInst>(EU.User);
VU && VU->getOperand(1) == EU.Scalar) {
if (auto *FTy = dyn_cast<FixedVectorType>(VU->getType())) {
if (!UsedInserts.insert(VU).second)
continue;
std::optional<unsigned> InsertIdx = getElementIndex(VU);
if (InsertIdx) {
const TreeEntry *ScalarTE = &EU.E;
auto *It = find_if(
ShuffledInserts,
[this, VU](const ShuffledInsertData<const TreeEntry *> &Data) {
// Checks if 2 insertelements are from the same buildvector.
InsertElementInst *VecInsert = Data.InsertElements.front();
return areTwoInsertFromSameBuildVector(
VU, VecInsert, [this](InsertElementInst *II) -> Value * {
Value *Op0 = II->getOperand(0);
if (isVectorized(II) && !isVectorized(Op0))
return nullptr;
return Op0;
});
});
int VecId = -1;
if (It == ShuffledInserts.end()) {
auto &Data = ShuffledInserts.emplace_back();
Data.InsertElements.emplace_back(VU);
DemandedElts.push_back(APInt::getZero(FTy->getNumElements()));
VecId = ShuffledInserts.size() - 1;
auto It = MinBWs.find(ScalarTE);
if (It != MinBWs.end() &&
VectorCasts
.insert(std::make_pair(ScalarTE, FTy->getElementType()))
.second) {
unsigned BWSz = It->second.first;
unsigned DstBWSz = DL->getTypeSizeInBits(FTy->getElementType());
unsigned VecOpcode;
if (DstBWSz < BWSz)
VecOpcode = Instruction::Trunc;
else
VecOpcode =
It->second.second ? Instruction::SExt : Instruction::ZExt;
InstructionCost C = TTI->getCastInstrCost(
VecOpcode, FTy,
getWidenedType(IntegerType::get(FTy->getContext(), BWSz),
FTy->getNumElements()),
getCastContextHint(*ScalarTE), CostKind);
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for extending externally used vector with "
"non-equal minimum bitwidth.\n");
Cost += C;
}
} else {
if (isFirstInsertElement(VU, It->InsertElements.front()))
It->InsertElements.front() = VU;
VecId = std::distance(ShuffledInserts.begin(), It);
}
int InIdx = *InsertIdx;
SmallVectorImpl<int> &Mask =
ShuffledInserts[VecId].ValueMasks[ScalarTE];
if (Mask.empty())
Mask.assign(FTy->getNumElements(), PoisonMaskElem);
Mask[InIdx] = EU.Lane;
DemandedElts[VecId].setBit(InIdx);
continue;
}
}
}
// If we plan to rewrite the tree in a smaller type, we will need to sign
// extend the extracted value back to the original type. Here, we account
// for the extract and the added cost of the sign extend if needed.
InstructionCost ExtraCost = TTI::TCC_Free;
auto *ScalarTy = EU.Scalar->getType();
const unsigned BundleWidth = EU.E.getVectorFactor();
assert(EU.Lane < BundleWidth && "Extracted lane out of bounds.");
auto *VecTy = getWidenedType(ScalarTy, BundleWidth);
const TreeEntry *Entry = &EU.E;
auto It = MinBWs.find(Entry);
SmallVector<unsigned, 1> Indices;
if (isa<StructType>(VecTy)) {
assert(EU.User && "Expected user for struct extract");
const auto *EV = cast<ExtractValueInst>(EU.User);
Indices.assign(EV->getIndices());
}
// We only add extract cost once for the same scalar and struct field.
auto ExtractKey = std::make_pair(EU.Scalar, Indices);
if (!isa_and_nonnull<InsertElementInst>(EU.User) &&
!ExtractCostCalculated.insert(ExtractKey).second)
continue;
if (It != MinBWs.end()) {
Type *MinTy = IntegerType::get(F->getContext(), It->second.first);
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy))
MinTy = getWidenedType(MinTy, VecTy->getNumElements());
unsigned Extend = isKnownNonNegative(EU.Scalar, SimplifyQuery(*DL))
? Instruction::ZExt
: Instruction::SExt;
VecTy = getWidenedType(MinTy, BundleWidth);
ExtraCost =
getExtractWithExtendCost(*TTI, Opts.slp_revec, Extend, ScalarTy,
cast<VectorType>(VecTy), EU.Lane, CostKind);
LLVM_DEBUG(dbgs() << " ExtractExtend or ExtractSubvec cost: "
<< ExtraCost << "\n");
} else {
Type *ExtractTy = VecTy;
if (auto *ST = dyn_cast<StructType>(VecTy)) {
ExtractTy = ExtractValueInst::getIndexedType(ST, Indices);
}
ExtraCost = getVectorInstrCost(
*TTI, Opts.slp_revec, ScalarTy, Instruction::ExtractElement,
ExtractTy, CostKind, EU.Lane, EU.Scalar, ScalarUserAndIdx);
LLVM_DEBUG(dbgs() << " ExtractElement cost for " << *ScalarTy << " from "
<< *VecTy << ": " << ExtraCost << "\n");
}
// Leave the scalar instructions as is if they are cheaper than extracts.
if (Entry->Idx != 0 || Entry->getOpcode() == Instruction::GetElementPtr ||
Entry->getOpcode() == Instruction::Load) {
// Checks if the user of the external scalar is phi in loop body.
auto IsPhiInLoop = [&](const ExternalUser &U) {
if (auto *Phi = dyn_cast_if_present<PHINode>(U.User)) {
auto *I = cast<Instruction>(U.Scalar);
const Loop *L = LI->getLoopFor(Phi->getParent());
return L && (Phi->getParent() == I->getParent() ||
L == LI->getLoopFor(I->getParent()));
}
return false;
};
if (!ValueToExtUses) {
ValueToExtUses.emplace();
// Ignore phis in loops.
for (const auto &P :
make_filter_range(enumerate(ExternalUses), [&](const auto &P) {
return !IsPhiInLoop(P.value());
}))
ValueToExtUses->try_emplace(P.value().Scalar, P.index());
}
// Can use original instruction, if no operands vectorized or they are
// marked as externally used already.
auto *Inst = cast<Instruction>(EU.Scalar);
InstructionCost ScalarCost = TTI->getInstructionCost(Inst, CostKind);
auto OperandIsScalar = [&](Value *V) {
if (!isVectorized(V)) {
// Some extractelements might be not vectorized, but
// transformed into shuffle and removed from the function,
// consider it here.
if (auto *EE = dyn_cast<ExtractElementInst>(V))
return !EE->hasOneUse() || !MustGather.contains(EE);
return true;
}
if (isa<StructType>(V->getType()))
return false;
return ValueToExtUses->contains(V);
};
bool CanBeUsedAsScalar = all_of(Inst->operands(), OperandIsScalar);
bool CanBeUsedAsScalarCast = false;
if (auto *CI = dyn_cast<CastInst>(Inst); CI && !CanBeUsedAsScalar) {
if (auto *Op = dyn_cast<Instruction>(CI->getOperand(0));
Op && all_of(Op->operands(), OperandIsScalar)) {
InstructionCost OpCost =
(isVectorized(Op) && !ValueToExtUses->contains(Op))
? TTI->getInstructionCost(Op, CostKind)
: 0;
if (ScalarCost + OpCost <= ExtraCost) {
CanBeUsedAsScalar = CanBeUsedAsScalarCast = true;
ScalarCost += OpCost;
}
}
}
if (CanBeUsedAsScalar) {
bool KeepScalar = ScalarCost <= ExtraCost;
// Try to keep original scalar if the user is the phi node from the same
// block as the root phis, currently vectorized. It allows to keep
// better ordering info of PHIs, being vectorized currently.
bool IsProfitablePHIUser =
(KeepScalar || (ScalarCost - ExtraCost <= TTI::TCC_Basic &&
getRootNodeScalars().size() > 2)) &&
getRootNode().hasState() &&
getRootNode().getOpcode() == Instruction::PHI &&
!Inst->hasNUsesOrMore(UsesLimit) &&
none_of(Inst->users(),
[&](User *U) {
auto *PHIUser = dyn_cast<PHINode>(U);
return (!PHIUser ||
PHIUser->getParent() !=
getRootNode().getMainOp()->getParent()) &&
!isVectorized(U);
}) &&
count_if(Entry->Scalars, [&](Value *V) {
return ValueToExtUses->contains(V);
}) <= 2;
if (IsProfitablePHIUser) {
KeepScalar = true;
} else if (KeepScalar && ScalarCost != TTI::TCC_Free &&
ExtraCost - ScalarCost <= TTI::TCC_Basic &&
(!GatheredLoadsEntriesFirst.has_value() ||
Entry->Idx < *GatheredLoadsEntriesFirst)) {
unsigned ScalarUsesCount = count_if(Entry->Scalars, [&](Value *V) {
return ValueToExtUses->contains(V);
});
auto It = ExtractsCount.find(Entry);
if (It != ExtractsCount.end()) {
assert(ScalarUsesCount >= It->getSecond().size() &&
"Expected total number of external uses not less than "
"number of scalar uses.");
ScalarUsesCount -= It->getSecond().size();
}
// Keep original scalar if number of externally used instructions in
// the same entry is not power of 2. It may help to do some extra
// vectorization for now.
KeepScalar = ScalarUsesCount <= 1 || !has_single_bit(ScalarUsesCount);
}
if (KeepScalar) {
ExternalUsesAsOriginalScalar.insert(EU.Scalar);
// Struct operands cannot be rebuilt by the !User extraction
// path (it has no insertvalue chain), so leave their existing
// ExtractValueInst user in place.
for (Value *V : make_filter_range(Inst->operands(), [](Value *V) {
return !isa<StructType>(V->getType());
})) {
auto It = ValueToExtUses->find(V);
if (It != ValueToExtUses->end()) {
// Replace all uses to avoid compiler crash.
ExternalUses[It->second].User = nullptr;
}
}
ExtraCost = ScalarCost;
if (!IsPhiInLoop(EU))
ExtractsCount[Entry].insert(Inst);
if (CanBeUsedAsScalarCast) {
ScalarOpsFromCasts.insert(Inst->getOperand(0));
// Update the users of the operands of the cast operand to avoid
// compiler crash.
if (auto *IOp = dyn_cast<Instruction>(Inst->getOperand(0))) {
for (Value *V : make_filter_range(IOp->operands(), [](Value *V) {
return !isa<StructType>(V->getType());
})) {
auto It = ValueToExtUses->find(V);
if (It != ValueToExtUses->end()) {
// Replace all uses to avoid compiler crash.
ExternalUses[It->second].User = nullptr;
}
}
}
}
}
}
}
// Scale the extract cost by the execution frequency of the block where
// codegen will place the extractelement. That block is the nearest common
// dominator of all effective use sites (precomputed in ScalarToExtractBlock
// above), which is order-independent. For scalars kept as originals the
// existing ScaleCost path (user-block based) remains correct, since the
// scalar instruction executes at its definition site's frequency.
if (!ExternalUsesAsOriginalScalar.contains(EU.Scalar)) {
if (ExtraCost.isValid() && ExtraCost != 0) {
if (!EU.User) {
// No external user instruction is recorded (User == nullptr): the
// scalar stays live in vectorized instructions or is used as an
// extra arg, and is not present in ScalarToExtractBlock (the
// pre-pass only records sites of real users). vectorizeTree() then
// places the extractelement right after the vectorized instruction
// (in the entry's block) and replaces the scalar uses with it, so
// scale by the entry block's execution frequency to match that
// placement.
ExtraCost = ScaleCost(ExtraCost, *Entry, EU.Scalar, /*U=*/nullptr);
} else {
BasicBlock *ExtractBB = ScalarToExtractBlock.lookup(EU.Scalar);
if (const Loop *L = ExtractBB ? LI->getLoopFor(ExtractBB) : nullptr) {
uint64_t Scale = getLoopNestScale(
findInnermostNonInvariantLoop(L, ArrayRef<Value *>(EU.Scalar)));
LLVM_DEBUG(dbgs()
<< "SLP: Extract scale " << Scale << " (NCD block) for "
<< EU.Scalar->getNameOrAsOperand() << "\n");
ExtraCost *= Scale;
}
}
}
} else {
ExtraCost = ScaleCost(ExtraCost, *Entry, EU.Scalar,
cast_if_present<Instruction>(EU.User));
}
ExtractCost += ExtraCost;
}
// Charge the pointer-chain cost difference once when every escaped scalar
// is used only to drive an address computation (see the detection loop
// above). Vectorizing the tree in this pattern forces lane extracts (or a
// vector GEP with unknown stride) to drive the address computation, which is
// typically more expensive than keeping the indices scalar in a unit-stride
// address chain. Add the delta once rather than per external use.
if (AllUsersGEPSWithStoresLoads && !Pointers.empty()) {
const TreeEntry &RootEntry = getRootNode();
const Value *CommonBase = getUnderlyingObject(Pointers.front());
if (all_of(Pointers, [CommonBase](const Value *P) {
return getUnderlyingObject(P) == CommonBase;
})) {
auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
Pointers, CommonBase, TTI::PointersChainInfo::getUnitStride(),
UserScalarTy, CostKind);
InstructionCost VectorGEPCost = TTI->getPointersChainCost(
Pointers, CommonBase, TTI::PointersChainInfo::getUnknownStride(),
VecTy, CostKind);
ExtractCost += ScaleCost(VectorGEPCost - ScalarGEPCost, RootEntry);
}
}
// Insert externals for extract of operands of casts to be emitted as scalars
// instead of extractelement.
for (Value *V : ScalarOpsFromCasts) {
ExternalUsesAsOriginalScalar.insert(V);
if (ArrayRef<TreeEntry *> TEs = getTreeEntries(V); !TEs.empty()) {
const auto *It = find_if_not(TEs, [&](TreeEntry *TE) {
return TransformedToGatherNodes.contains(TE) ||
DeletedNodes.contains(TE);
});
if (It != TEs.end()) {
const TreeEntry *UserTE = *It;
ExternalUses.emplace_back(V, nullptr, *UserTE,
UserTE->findLaneForValue(V));
}
}
}
// Add reduced value cost, if resized.
if (!VectorizedVals.empty()) {
const TreeEntry &Root = getRootNode();
auto BWIt = MinBWs.find(&Root);
if (BWIt != MinBWs.end()) {
Type *DstTy = Root.Scalars.front()->getType();
unsigned OriginalSz = DL->getTypeSizeInBits(DstTy->getScalarType());
unsigned SrcSz =
ReductionBitWidth == 0 ? BWIt->second.first : ReductionBitWidth;
if (OriginalSz != SrcSz) {
unsigned Opcode = Instruction::Trunc;
if (OriginalSz > SrcSz)
Opcode = BWIt->second.second ? Instruction::SExt : Instruction::ZExt;
Type *SrcTy = IntegerType::get(DstTy->getContext(), SrcSz);
if (auto *VecTy = dyn_cast<FixedVectorType>(DstTy)) {
assert(Opts.slp_revec && "Only supported by REVEC.");
SrcTy = getWidenedType(SrcTy, VecTy->getNumElements());
}
InstructionCost CastCost = TTI->getCastInstrCost(
Opcode, DstTy, SrcTy, TTI::CastContextHint::None, CostKind);
CastCost = ScaleCost(CastCost, Root, /*Scalar=*/nullptr, ReductionRoot);
Cost += CastCost;
}
}
}
// Buildvector with externally used scalars, which should remain as scalars,
// should not be vectorized, the compiler may hang.
if (getCostThreshold() < 0 && VectorizableTree.size() > 1 &&
isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
VectorizableTree[1]->hasState() &&
VectorizableTree[1]->State == TreeEntry::Vectorize &&
all_of(VectorizableTree[1]->Scalars, [&](Value *V) {
return ExternalUsesAsOriginalScalar.contains(V);
}))
return InstructionCost::getInvalid();
Cost += ExtractCost;
auto &&ResizeToVF = [this, &Cost](const TreeEntry *TE, ArrayRef<int> Mask,
bool ForSingleMask) {
InstructionCost C = 0;
unsigned VF = Mask.size();
unsigned VecVF = TE->getVectorFactor();
bool HasLargeIndex =
any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); });
if ((VF != VecVF && HasLargeIndex) ||
!ShuffleVectorInst::isIdentityMask(Mask, VF)) {
if (HasLargeIndex) {
SmallVector<int> OrigMask(VecVF, PoisonMaskElem);
std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)),
OrigMask.begin());
C = getShuffleCost(
*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(getWidenedType(TE->getMainOp()->getType(), VecVF)),
CostKind, OrigMask);
LLVM_DEBUG(
dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of insertelement external users.\n";
TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
Cost += C;
return std::make_pair(TE, true);
}
if (!ForSingleMask) {
SmallVector<int> ResizeMask(VF, PoisonMaskElem);
for (unsigned I = 0; I < VF; ++I) {
if (Mask[I] != PoisonMaskElem)
ResizeMask[Mask[I]] = Mask[I];
}
if (!ShuffleVectorInst::isIdentityMask(ResizeMask, VF))
C = getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(getWidenedType(
TE->getMainOp()->getType(), VecVF)),
CostKind, ResizeMask);
LLVM_DEBUG(
dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of insertelement external users.\n";
TE->dump(); dbgs() << "SLP: Current total cost = " << Cost << "\n");
Cost += C;
}
}
return std::make_pair(TE, false);
};
// Calculate the cost of the reshuffled vectors, if any.
for (int I = 0, E = ShuffledInserts.size(); I < E; ++I) {
Value *Base = ShuffledInserts[I].InsertElements.front()->getOperand(0);
auto Vector = ShuffledInserts[I].ValueMasks.takeVector();
unsigned VF = 0;
auto EstimateShufflesCost = [&](ArrayRef<int> Mask,
ArrayRef<const TreeEntry *> TEs) {
assert((TEs.size() == 1 || TEs.size() == 2) &&
"Expected exactly 1 or 2 tree entries.");
if (TEs.size() == 1) {
if (VF == 0)
VF = TEs.front()->getVectorFactor();
auto *FTy = getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
if (!ShuffleVectorInst::isIdentityMask(Mask, VF) &&
!all_of(enumerate(Mask), [=](const auto &Data) {
return Data.value() == PoisonMaskElem ||
(Data.index() < VF &&
static_cast<int>(Data.index()) == Data.value());
})) {
InstructionCost C =
getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(FTy), CostKind, Mask);
C = ScaleCost(C, *TEs.front());
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of insertelement "
"external users.\n";
TEs.front()->dump();
dbgs() << "SLP: Current total cost = " << Cost << "\n");
Cost += C;
}
} else {
if (VF == 0) {
if (TEs.front() &&
TEs.front()->getVectorFactor() == TEs.back()->getVectorFactor())
VF = TEs.front()->getVectorFactor();
else
VF = Mask.size();
}
auto *FTy = getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
InstructionCost C = getShuffleCost(
*TTI, TTI::SK_PermuteTwoSrc, cast<VectorType>(FTy), CostKind, Mask);
C = ScaleCost(C, *TEs.back());
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
<< " for final shuffle of vector node and external "
"insertelement users.\n";
if (TEs.front()) { TEs.front()->dump(); } TEs.back()->dump();
dbgs() << "SLP: Current total cost = " << Cost << "\n");
Cost += C;
}
VF = Mask.size();
return TEs.back();
};
(void)performExtractsShuffleAction<const TreeEntry>(
MutableArrayRef(Vector.data(), Vector.size()), Base,
[](const TreeEntry *E) { return E->getVectorFactor(); }, ResizeToVF,
EstimateShufflesCost);
InstructionCost InsertCost = TTI->getScalarizationOverhead(
cast<FixedVectorType>(
ShuffledInserts[I].InsertElements.front()->getType()),
DemandedElts[I],
/*Insert*/ true, /*Extract*/ false, CostKind);
Cost -= InsertCost;
}
// Add the cost for reduced value resize (if required).
if (ReductionBitWidth != 0) {
assert(UserIgnoreList && "Expected reduction tree.");
const TreeEntry &E = getRootNode();
auto It = MinBWs.find(&E);
if (It != MinBWs.end() && It->second.first != ReductionBitWidth) {
unsigned SrcSize = It->second.first;
unsigned DstSize = ReductionBitWidth;
unsigned Opcode = Instruction::Trunc;
if (SrcSize < DstSize) {
bool IsArithmeticExtendedReduction =
all_of(*UserIgnoreList, [](Value *V) {
auto *I = cast<Instruction>(V);
return is_contained({Instruction::Add, Instruction::FAdd,
Instruction::Mul, Instruction::FMul,
Instruction::And, Instruction::Or,
Instruction::Xor},
I->getOpcode());
});
if (IsArithmeticExtendedReduction)
Opcode =
Instruction::BitCast; // Handle it by getExtendedReductionCost
else
Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
}
if (Opcode != Instruction::BitCast) {
auto *SrcVecTy =
getWidenedType(Builder.getIntNTy(SrcSize), E.getVectorFactor());
auto *DstVecTy =
getWidenedType(Builder.getIntNTy(DstSize), E.getVectorFactor());
TTI::CastContextHint CCH = getCastContextHint(E);
switch (E.getOpcode()) {
case Instruction::SExt:
case Instruction::ZExt:
case Instruction::Trunc: {
// Split and gather roots have no operand entry to take the context
// hint from.
if (E.isGather() || E.State == TreeEntry::SplitVectorize)
break;
const TreeEntry *OpTE = getOperandEntry(&E, 0);
CCH = getCastContextHint(*OpTE);
break;
}
default:
break;
}
InstructionCost CastCost =
TTI->getCastInstrCost(Opcode, DstVecTy, SrcVecTy, CCH, CostKind);
CastCost = ScaleCost(CastCost, getRootNode(), /*Scalar=*/nullptr,
ReductionRoot);
Cost += CastCost;
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << CastCost
<< " for final resize for reduction from " << SrcVecTy
<< " to " << DstVecTy << "\n";
dbgs() << "SLP: Current total cost = " << Cost << "\n");
}
}
}
std::optional<InstructionCost> SpillCost;
if (Cost < -getCostThreshold()) {
SpillCost = getSpillCost();
Cost += *SpillCost;
}
#ifndef NDEBUG
SmallString<256> Str;
{
raw_svector_ostream OS(Str);
OS << "SLP: Spill Cost = ";
if (SpillCost)
OS << *SpillCost;
else
OS << "<skipped>";
OS << ".\nSLP: Extract Cost = " << ExtractCost << ".\n";
if (ReductionRoot)
OS << "SLP: Reduction Cost = " << ReductionCost << ".\n";
OS << "SLP: Total Cost = " << Cost << ".\n";
}
LLVM_DEBUG(dbgs() << Str);
if (Opts.view_slp_tree)
ViewGraph(this, "SLP" + F->getName(), false, Str);
#endif
return Cost;
}
/// Tries to find extractelement instructions with constant indices from fixed
/// vector type and gather such instructions into a bunch, which highly likely
/// might be detected as a shuffle of 1 or 2 input vectors. If this attempt was
/// successful, the matched scalars are replaced by poison values in \p VL for
/// future analysis.
std::optional<TTI::ShuffleKind>
BoUpSLP::tryToGatherSingleRegisterExtractElements(
MutableArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) const {
// Scan list of gathered scalars for extractelements that can be represented
// as shuffles.
MapVector<Value *, SmallVector<int>> VectorOpToIdx;
SmallVector<int> UndefVectorExtracts;
for (int I = 0, E = VL.size(); I < E; ++I) {
auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
if (!EI) {
if (isa<UndefValue>(VL[I]))
UndefVectorExtracts.push_back(I);
continue;
}
auto *VecTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
if (!VecTy || !isa<ConstantInt, UndefValue>(EI->getIndexOperand()))
continue;
std::optional<unsigned> Idx = getExtractIndex(EI);
// Undefined index.
if (!Idx) {
UndefVectorExtracts.push_back(I);
continue;
}
if (Idx >= VecTy->getNumElements()) {
UndefVectorExtracts.push_back(I);
continue;
}
SmallBitVector ExtractMask(VecTy->getNumElements(), true);
ExtractMask.reset(*Idx);
if (isUndefVector</*IsPoisonOnly=*/true>(EI->getVectorOperand(),
ExtractMask)
.all()) {
UndefVectorExtracts.push_back(I);
continue;
}
VectorOpToIdx[EI->getVectorOperand()].push_back(I);
}
// Sort the vector operands by the maximum number of uses in extractelements.
SmallVector<std::pair<Value *, SmallVector<int>>> Vectors =
VectorOpToIdx.takeVector();
stable_sort(Vectors, [](const auto &P1, const auto &P2) {
return P1.second.size() > P2.second.size();
});
// Find the best pair of the vectors or a single vector.
const int UndefSz = UndefVectorExtracts.size();
unsigned SingleMax = 0;
unsigned PairMax = 0;
if (!Vectors.empty()) {
SingleMax = Vectors.front().second.size() + UndefSz;
if (Vectors.size() > 1) {
auto *ItNext = std::next(Vectors.begin());
PairMax = SingleMax + ItNext->second.size();
}
}
if (SingleMax == 0 && PairMax == 0 && UndefSz == 0)
return std::nullopt;
// Check if better to perform a shuffle of 2 vectors or just of a single
// vector.
SmallVector<Value *> SavedVL(VL.begin(), VL.end());
SmallVector<Value *> GatheredExtracts(
VL.size(), PoisonValue::get(VL.front()->getType()));
if (SingleMax >= PairMax && SingleMax) {
for (int Idx : Vectors.front().second)
std::swap(GatheredExtracts[Idx], VL[Idx]);
} else if (!Vectors.empty()) {
for (unsigned Idx : {0, 1})
for (int Idx : Vectors[Idx].second)
std::swap(GatheredExtracts[Idx], VL[Idx]);
}
// Add extracts from undefs too.
for (int Idx : UndefVectorExtracts)
std::swap(GatheredExtracts[Idx], VL[Idx]);
// Check that gather of extractelements can be represented as just a
// shuffle of a single/two vectors the scalars are extracted from.
std::optional<TTI::ShuffleKind> Res =
isFixedVectorShuffle(GatheredExtracts, Mask, AC, Cache);
if (!Res || all_of(Mask, equal_to(PoisonMaskElem))) {
// TODO: try to check other subsets if possible.
// Restore the original VL and mask if attempt was not successful.
copy(SavedVL, VL.begin());
Mask.assign(VL.size(), PoisonMaskElem);
return std::nullopt;
}
// Restore unused scalars from mask, if some of the extractelements were not
// selected for shuffle.
for (int I = 0, E = GatheredExtracts.size(); I < E; ++I) {
if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]) &&
isa<UndefValue>(GatheredExtracts[I])) {
std::swap(VL[I], GatheredExtracts[I]);
continue;
}
auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
if (!EI || !isa<FixedVectorType>(EI->getVectorOperandType()) ||
!isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
is_contained(UndefVectorExtracts, I))
continue;
}
return Res;
}
/// Tries to find extractelement instructions with constant indices from fixed
/// vector type and gather such instructions into a bunch, which highly likely
/// might be detected as a shuffle of 1 or 2 input vectors. If this attempt was
/// successful, the matched scalars are replaced by poison values in \p VL for
/// future analysis.
SmallVector<std::optional<TTI::ShuffleKind>>
BoUpSLP::tryToGatherExtractElements(SmallVectorImpl<Value *> &VL,
SmallVectorImpl<int> &Mask,
unsigned NumParts) const {
assert(NumParts > 0 && "NumParts expected be greater than or equal to 1.");
SmallVector<std::optional<TTI::ShuffleKind>> ShufflesRes(NumParts);
Mask.assign(VL.size(), PoisonMaskElem);
unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
for (unsigned Part : seq<unsigned>(NumParts)) {
// Scan list of gathered scalars for extractelements that can be represented
// as shuffles.
const unsigned PartOffset = Part * SliceSize;
const unsigned PartSize = getNumElems(VL.size(), SliceSize, Part);
// It may happen in case of revec, need to check no access out of bounds.
if (PartOffset + PartSize > VL.size())
break;
MutableArrayRef<Value *> SubVL =
MutableArrayRef(VL).slice(PartOffset, PartSize);
SmallVector<int> SubMask;
std::optional<TTI::ShuffleKind> Res =
tryToGatherSingleRegisterExtractElements(SubVL, SubMask);
ShufflesRes[Part] = Res;
copy(SubMask, std::next(Mask.begin(), Part * SliceSize));
if (SubVL.size() != SliceSize)
break;
}
if (none_of(ShufflesRes, [](const std::optional<TTI::ShuffleKind> &Res) {
return Res.has_value();
}))
ShufflesRes.clear();
return ShufflesRes;
}
std::optional<TargetTransformInfo::ShuffleKind>
BoUpSLP::isGatherShuffledSingleRegisterEntry(
const TreeEntry *TE, ArrayRef<Value *> VL, MutableArrayRef<int> Mask,
SmallVectorImpl<const TreeEntry *> &Entries, unsigned Part, bool ForOrder,
unsigned SliceSize) {
Entries.clear();
if (TE->Idx == 0)
return std::nullopt;
const unsigned MaskBase = Part * SliceSize;
// TODO: currently checking only for Scalars in the tree entry, need to count
// reused elements too for better cost estimation.
auto GetUserEntry = [&](const TreeEntry *TE) {
while (TE->UserTreeIndex && TE->UserTreeIndex.EdgeIdx == UINT_MAX)
TE = TE->UserTreeIndex.UserTE;
if (TE == &getRootNode())
return EdgeInfo(const_cast<TreeEntry *>(TE), 0);
return TE->UserTreeIndex;
};
auto HasGatherUser = [&](const TreeEntry *TE) {
while (TE->Idx != 0 && TE->UserTreeIndex) {
if (TE->UserTreeIndex.EdgeIdx == UINT_MAX)
return true;
TE = TE->UserTreeIndex.UserTE;
}
return false;
};
const EdgeInfo TEUseEI = GetUserEntry(TE);
if (!TEUseEI || (TEUseEI.UserTE->Idx == 0 && TEUseEI.UserTE->isGather() &&
!TEUseEI.UserTE->hasState()))
return std::nullopt;
const Instruction *TEInsertPt = &getLastInstructionInBundle(TEUseEI.UserTE);
const BasicBlock *TEInsertBlock = nullptr;
// Main node of PHI entries keeps the correct order of operands/incoming
// blocks.
if (auto *PHI = TEUseEI.UserTE->hasState()
? dyn_cast<PHINode>(TEUseEI.UserTE->getMainOp())
: nullptr;
PHI && TEUseEI.UserTE->State != TreeEntry::SplitVectorize) {
TEInsertBlock = PHI->getIncomingBlock(TEUseEI.EdgeIdx);
TEInsertPt = TEInsertBlock->getTerminator();
} else {
TEInsertBlock = TEInsertPt->getParent();
}
if (!DT->isReachableFromEntry(TEInsertBlock))
return std::nullopt;
auto *NodeUI = DT->getNode(TEInsertBlock);
assert(NodeUI && "Should only process reachable instructions");
SmallPtrSet<Value *, 4> GatheredScalars(llvm::from_range, VL);
auto CheckOrdering = [&](const Instruction *InsertPt) {
// Argument InsertPt is an instruction where vector code for some other
// tree entry (one that shares one or more scalars with TE) is going to be
// generated. This lambda returns true if insertion point of vector code
// for the TE dominates that point (otherwise dependency is the other way
// around). The other node is not limited to be of a gather kind. Gather
// nodes are not scheduled and their vector code is inserted before their
// first user. If user is PHI, that is supposed to be at the end of a
// predecessor block. Otherwise it is the last instruction among scalars of
// the user node. So, instead of checking dependency between instructions
// themselves, we check dependency between their insertion points for vector
// code (since each scalar instruction ends up as a lane of a vector
// instruction).
const BasicBlock *InsertBlock = InsertPt->getParent();
auto *NodeEUI = DT->getNode(InsertBlock);
if (!NodeEUI)
return false;
assert((NodeUI == NodeEUI) ==
(NodeUI->getDFSNumIn() == NodeEUI->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
// Check the order of the gather nodes users.
if (TEInsertPt->getParent() != InsertBlock &&
(DT->dominates(NodeUI, NodeEUI) || !DT->dominates(NodeEUI, NodeUI)))
return false;
if (TEInsertPt->getParent() == InsertBlock &&
TEInsertPt->comesBefore(InsertPt))
return false;
return true;
};
// Find all tree entries used by the gathered values. If no common entries
// found - not a shuffle.
// Here we build a set of tree nodes for each gathered value and trying to
// find the intersection between these sets. If we have at least one common
// tree node for each gathered value - we have just a permutation of the
// single vector. If we have 2 different sets, we're in situation where we
// have a permutation of 2 input vectors.
SmallVector<SmallPtrSet<const TreeEntry *, 4>> UsedTEs;
SmallDenseMap<Value *, int> UsedValuesEntry;
SmallPtrSet<const Value *, 16> VisitedValue;
bool IsReusedNodeFound = false;
auto CheckAndUseSameNode = [&](const TreeEntry *TEPtr) {
// The node is reused - exit.
if (IsReusedNodeFound)
return false;
if ((TEPtr->getVectorFactor() != VL.size() &&
TEPtr->Scalars.size() != VL.size()) ||
(!TEPtr->isSame(VL) && !TEPtr->isSame(TE->Scalars)))
return false;
IsReusedNodeFound =
equal(TE->Scalars, TEPtr->Scalars) &&
equal(TE->ReorderIndices, TEPtr->ReorderIndices) &&
equal(TE->ReuseShuffleIndices, TEPtr->ReuseShuffleIndices);
UsedTEs.clear();
UsedTEs.emplace_back().insert(TEPtr);
auto IsNotConstant = [](Value *V) { return !isConstant(V); };
for (Value *V : make_filter_range(VL, IsNotConstant))
UsedValuesEntry.try_emplace(V, 0);
return true;
};
auto CheckParentNodes = [&](const TreeEntry *User1, const TreeEntry *User2,
unsigned EdgeIdx) {
const TreeEntry *Ptr1 = User1;
const TreeEntry *Ptr2 = User2;
SmallDenseMap<const TreeEntry *, unsigned> PtrToIdx;
while (Ptr2) {
PtrToIdx.try_emplace(Ptr2, EdgeIdx);
EdgeIdx = Ptr2->UserTreeIndex.EdgeIdx;
Ptr2 = Ptr2->UserTreeIndex.UserTE;
}
while (Ptr1) {
unsigned Idx = Ptr1->UserTreeIndex.EdgeIdx;
Ptr1 = Ptr1->UserTreeIndex.UserTE;
if (auto It = PtrToIdx.find(Ptr1); It != PtrToIdx.end())
return Idx < It->second;
}
return false;
};
// Cache `isUsedOutsideBlock(TEInsertPt)` - TEInsertPt is loop-invariant and
// the function walks the instruction's user list.
std::optional<bool> TEInsertPtUsedOutsideBlock;
auto IsTEInsertPtUsedOutsideBlock = [&] {
if (!TEInsertPtUsedOutsideBlock)
TEInsertPtUsedOutsideBlock =
isUsedOutsideBlock(const_cast<Instruction *>(TEInsertPt));
return *TEInsertPtUsedOutsideBlock;
};
// Cache the TEUseEI/TEInsertPt-only prefix of the per-call lambda predicate
// below - all of these depend only on outer-scope state, not the lambda's
// arguments.
const bool TEUseEIInsertPtUsedOutside =
TEUseEI && TEUseEI.UserTE && TEUseEI.UserTE->hasCopyableElements() &&
!TEUseEI.UserTE->isCopyableElement(
const_cast<Instruction *>(TEInsertPt)) &&
IsTEInsertPtUsedOutsideBlock();
auto CheckNonSchedulableOrdering = [&](const TreeEntry *E,
Instruction *InsertPt) {
return TEUseEIInsertPtUsedOutside &&
InsertPt->getNextNode() == TEInsertPt &&
(!E->hasCopyableElements() || !E->isCopyableElement(InsertPt) ||
!isUsedOutsideBlock(InsertPt));
};
// Cache the TEUseEI.UserTE-dependent predicate - it is invariant across the
// double loop below. all_of with isUsedOutsideBlock walks each scalar's
// users and is the expensive component.
const bool TEUserNeedsEmitFirst =
TEUseEI.UserTE->State == TreeEntry::Vectorize &&
TEUseEI.UserTE->hasState() &&
(TEUseEI.UserTE->getOpcode() != Instruction::PHI ||
TEUseEI.UserTE->isAltShuffle()) &&
all_of(TEUseEI.UserTE->Scalars, isUsedOutsideBlock);
// Cache `all_of(UserTE->Scalars, isUsedOutsideBlock)` per UserTE - the
// same UserTE may be encountered for many TEPtr values inside the loop.
SmallDenseMap<const TreeEntry *, bool> ScalarsUsedOutsideBlockCache;
auto AllScalarsUsedOutsideBlock = [&](const TreeEntry *UserTE) {
auto [It, Inserted] = ScalarsUsedOutsideBlockCache.try_emplace(UserTE);
if (!Inserted)
return It->second;
bool Res = all_of(UserTE->Scalars, isUsedOutsideBlock);
It->second = Res;
return Res;
};
for (Value *V : VL) {
if (isConstant(V) || !VisitedValue.insert(V).second)
continue;
// Build a list of tree entries where V is used.
SmallPtrSet<const TreeEntry *, 4> VToTEs;
SmallVector<const TreeEntry *> GatherNodes(
ValueToGatherNodes.lookup(V).takeVector());
if (TransformedToGatherNodes.contains(TE)) {
auto IsLiveUserGather = [&](TreeEntry *E) {
return TE != E && TransformedToGatherNodes.contains(E) &&
E->UserTreeIndex && !E->UserTreeIndex.UserTE->isGather();
};
for (TreeEntry *E :
make_filter_range(getSplitTreeEntries(V), IsLiveUserGather))
GatherNodes.push_back(E);
for (TreeEntry *E :
make_filter_range(getTreeEntries(V), IsLiveUserGather))
GatherNodes.push_back(E);
} else if (const TreeEntry *E = getSameValuesTreeEntry(V, TE->Scalars);
E && !TEUserNeedsEmitFirst &&
TransformedToGatherNodes.contains(E) && E->UserTreeIndex &&
E->UserTreeIndex.UserTE == TE->UserTreeIndex.UserTE &&
!E->UserTreeIndex.UserTE->isGather()) {
// Regular gathers reuse only perfectly matched transformed nodes of the
// same user. Gathers, emitted before their user, cannot reuse transformed
// nodes, which are emitted with the user.
GatherNodes.push_back(E);
}
for (const TreeEntry *TEPtr :
make_filter_range(GatherNodes, [&](const TreeEntry *TEPtr) {
return TEPtr != TE && TEPtr->Idx != 0 &&
!DeletedNodes.contains(TEPtr);
})) {
assert(any_of(TEPtr->Scalars,
[&](Value *V) { return GatheredScalars.contains(V); }) &&
"Must contain at least single gathered value.");
assert(TEPtr->UserTreeIndex &&
"Expected only single user of a gather node.");
if (any_of(TEPtr->CombinedEntriesWithIndices,
[&](const auto &P) { return P.first == TE->Idx; }))
continue;
const EdgeInfo &UseEI = TEPtr->UserTreeIndex;
PHINode *UserPHI = (UseEI.UserTE->State != TreeEntry::SplitVectorize &&
UseEI.UserTE->hasState())
? dyn_cast<PHINode>(UseEI.UserTE->getMainOp())
: nullptr;
Instruction *InsertPt =
UserPHI ? UserPHI->getIncomingBlock(UseEI.EdgeIdx)->getTerminator()
: &getLastInstructionInBundle(UseEI.UserTE);
if (TEInsertPt == InsertPt) {
// Check nodes, which might be emitted first.
if (TEUserNeedsEmitFirst) {
if (UseEI.UserTE->State != TreeEntry::Vectorize ||
(UseEI.UserTE->hasState() &&
UseEI.UserTE->getOpcode() == Instruction::PHI &&
!UseEI.UserTE->isAltShuffle()) ||
!AllScalarsUsedOutsideBlock(UseEI.UserTE))
continue;
}
// If the schedulable insertion point is used in multiple entries - just
// exit, no known ordering at this point, available only after real
// scheduling.
if (!doesNotNeedToBeScheduled(InsertPt) &&
(TEUseEI.UserTE != UseEI.UserTE || TEUseEI.EdgeIdx < UseEI.EdgeIdx))
continue;
// If the users are the PHI nodes with the same incoming blocks - skip.
if (TEUseEI.UserTE->State == TreeEntry::Vectorize &&
TEUseEI.UserTE->getOpcode() == Instruction::PHI &&
UseEI.UserTE->State == TreeEntry::Vectorize &&
UseEI.UserTE->getOpcode() == Instruction::PHI &&
TEUseEI.UserTE != UseEI.UserTE)
continue;
// If 2 gathers are operands of the same entry (regardless of whether
// user is PHI or else), compare operands indices, use the earlier one
// as the base.
if (TEUseEI.UserTE == UseEI.UserTE && TEUseEI.EdgeIdx < UseEI.EdgeIdx)
continue;
// If the user instruction is used for some reason in different
// vectorized nodes - make it depend on index.
if (TEUseEI.UserTE != UseEI.UserTE &&
(TEUseEI.UserTE->Idx < UseEI.UserTE->Idx ||
HasGatherUser(TEUseEI.UserTE)))
continue;
// If the user node is the operand of the other user node - skip.
if (CheckParentNodes(TEUseEI.UserTE, UseEI.UserTE, UseEI.EdgeIdx))
continue;
}
if (!TEUseEI.UserTE->isGather() && !UserPHI &&
TEUseEI.UserTE->doesNotNeedToSchedule() !=
UseEI.UserTE->doesNotNeedToSchedule() &&
is_contained(UseEI.UserTE->Scalars, TEInsertPt))
continue;
// Check if the user node of the TE comes after user node of TEPtr,
// otherwise TEPtr depends on TE.
if ((TEInsertBlock != InsertPt->getParent() ||
TEUseEI.EdgeIdx < UseEI.EdgeIdx || TEUseEI.UserTE != UseEI.UserTE) &&
(!CheckOrdering(InsertPt) ||
(UseEI.UserTE->hasCopyableElements() &&
IsTEInsertPtUsedOutsideBlock() &&
is_contained(UseEI.UserTE->Scalars, TEInsertPt))))
continue;
// The node is reused - exit.
if (CheckAndUseSameNode(TEPtr))
break;
// The parent node is copyable with last inst used outside? And the last
// inst is the next inst for the lastinst of TEPtr? Exit, if yes, to
// preserve def-use chain.
if (CheckNonSchedulableOrdering(UseEI.UserTE, InsertPt))
continue;
VToTEs.insert(TEPtr);
}
if (ArrayRef<TreeEntry *> VTEs = getSplitTreeEntries(V); !VTEs.empty()) {
const auto *It = find_if(VTEs, [&](const TreeEntry *MTE) {
return MTE != TE && MTE != TEUseEI.UserTE &&
!DeletedNodes.contains(MTE) &&
!TransformedToGatherNodes.contains(MTE);
});
if (It != VTEs.end()) {
const TreeEntry *VTE = *It;
if (none_of(TE->CombinedEntriesWithIndices,
[&](const auto &P) { return P.first == VTE->Idx; })) {
Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
continue;
}
// The node is reused - exit.
if (CheckAndUseSameNode(VTE))
break;
VToTEs.insert(VTE);
}
}
if (ArrayRef<TreeEntry *> VTEs = getTreeEntries(V); !VTEs.empty()) {
const auto *It = find_if(VTEs, [&, MainTE = TE](const TreeEntry *TE) {
return TE != MainTE && !DeletedNodes.contains(TE) &&
!TransformedToGatherNodes.contains(TE);
});
if (It != VTEs.end()) {
const TreeEntry *VTE = *It;
if (ForOrder && VTE->Idx < GatheredLoadsEntriesFirst.value_or(0) &&
VTEs.size() > 1 && VTE->State != TreeEntry::Vectorize) {
VTEs = VTEs.drop_front();
// Iterate through all vectorized nodes.
const auto *MIt = find_if(VTEs, [](const TreeEntry *MTE) {
return MTE->State == TreeEntry::Vectorize;
});
if (MIt == VTEs.end())
continue;
VTE = *MIt;
}
if (none_of(TE->CombinedEntriesWithIndices,
[&](const auto &P) { return P.first == VTE->Idx; })) {
Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
if (&LastBundleInst == TEInsertPt ||
!CheckOrdering(&LastBundleInst) ||
CheckNonSchedulableOrdering(VTE, &LastBundleInst))
continue;
}
// The node is reused - exit.
if (CheckAndUseSameNode(VTE))
break;
VToTEs.insert(VTE);
}
}
if (IsReusedNodeFound)
break;
if (VToTEs.empty())
continue;
if (UsedTEs.empty()) {
// The first iteration, just insert the list of nodes to vector.
UsedTEs.push_back(VToTEs);
UsedValuesEntry.try_emplace(V, 0);
} else {
// Need to check if there are any previously used tree nodes which use V.
// If there are no such nodes, consider that we have another one input
// vector.
SmallPtrSet<const TreeEntry *, 4> SavedVToTEs(VToTEs);
unsigned Idx = 0;
for (SmallPtrSet<const TreeEntry *, 4> &Set : UsedTEs) {
// Do we have a non-empty intersection of previously listed tree entries
// and tree entries using current V?
set_intersect(VToTEs, Set);
if (!VToTEs.empty()) {
// Yes, write the new subset and continue analysis for the next
// scalar.
Set.swap(VToTEs);
break;
}
VToTEs = SavedVToTEs;
++Idx;
}
// No non-empty intersection found - need to add a second set of possible
// source vectors.
if (Idx == UsedTEs.size()) {
// If the number of input vectors is greater than 2 - not a permutation,
// fallback to the regular gather.
// TODO: support multiple reshuffled nodes.
if (UsedTEs.size() == 2)
continue;
UsedTEs.push_back(SavedVToTEs);
Idx = UsedTEs.size() - 1;
}
UsedValuesEntry.try_emplace(V, Idx);
}
}
if (UsedTEs.empty()) {
Entries.clear();
return std::nullopt;
}
unsigned VF = 0;
if (UsedTEs.size() == 1) {
// Keep the order to avoid non-determinism.
SmallVector<const TreeEntry *> FirstEntries(UsedTEs.front().begin(),
UsedTEs.front().end());
sort(FirstEntries, [](const TreeEntry *TE1, const TreeEntry *TE2) {
return TE1->Idx < TE2->Idx;
});
// Try to find the perfect match in another gather node at first.
auto *It = find_if(FirstEntries, [=](const TreeEntry *EntryPtr) {
return (EntryPtr->getVectorFactor() == TE->Scalars.size() &&
EntryPtr->isSame(TE->Scalars)) ||
EntryPtr->isSame(VL);
});
if (It != FirstEntries.end() &&
(IsReusedNodeFound || (*It)->getVectorFactor() == VL.size() ||
((*It)->getVectorFactor() == TE->Scalars.size() &&
TE->ReuseShuffleIndices.size() == VL.size() &&
(*It)->isSame(TE->Scalars)))) {
Entries.push_back(*It);
if (IsReusedNodeFound || (*It)->getVectorFactor() == VL.size()) {
std::iota(std::next(Mask.begin(), MaskBase),
std::next(Mask.begin(), MaskBase + VL.size()), 0);
} else {
SmallVector<int> CommonMask = TE->getCommonMask();
copy(CommonMask, Mask.begin());
}
// Clear undef scalars.
for (unsigned I : seq<unsigned>(VL.size()))
if (isa<PoisonValue>(VL[I]))
Mask[MaskBase + I] = PoisonMaskElem;
return TargetTransformInfo::SK_PermuteSingleSrc;
}
// No perfect match, just shuffle, so choose the first tree node from the
// tree.
Entries.push_back(FirstEntries.front());
// Update mapping between values and corresponding tree entries.
for (auto &P : UsedValuesEntry)
P.second = 0;
VF = FirstEntries.front()->getVectorFactor();
} else {
// Try to find nodes with the same vector factor.
assert(UsedTEs.size() == 2 && "Expected at max 2 permuted entries.");
// Keep the order of tree nodes to avoid non-determinism.
DenseMap<int, const TreeEntry *> VFToTE;
for (const TreeEntry *TE : UsedTEs.front()) {
unsigned VF = TE->getVectorFactor();
auto It = VFToTE.find(VF);
if (It != VFToTE.end()) {
if (It->second->Idx > TE->Idx)
It->getSecond() = TE;
continue;
}
VFToTE.try_emplace(VF, TE);
}
// Same, keep the order to avoid non-determinism.
SmallVector<const TreeEntry *> SecondEntries(UsedTEs.back().begin(),
UsedTEs.back().end());
sort(SecondEntries, [](const TreeEntry *TE1, const TreeEntry *TE2) {
return TE1->Idx < TE2->Idx;
});
for (const TreeEntry *TE : SecondEntries) {
auto It = VFToTE.find(TE->getVectorFactor());
if (It != VFToTE.end()) {
VF = It->first;
Entries.push_back(It->second);
Entries.push_back(TE);
break;
}
}
// No 2 source vectors with the same vector factor - just choose 2 with max
// index.
if (Entries.empty()) {
Entries.push_back(*llvm::max_element(
UsedTEs.front(), [](const TreeEntry *TE1, const TreeEntry *TE2) {
return TE1->Idx < TE2->Idx;
}));
Entries.push_back(SecondEntries.front());
VF = std::max(Entries.front()->getVectorFactor(),
Entries.back()->getVectorFactor());
} else {
VF = Entries.front()->getVectorFactor();
}
SmallVector<SmallPtrSet<Value *, 8>> ValuesToEntries;
for (const TreeEntry *E : Entries)
ValuesToEntries.emplace_back().insert(E->Scalars.begin(),
E->Scalars.end());
// Update mapping between values and corresponding tree entries.
for (auto &P : UsedValuesEntry) {
for (unsigned Idx : seq<unsigned>(ValuesToEntries.size()))
if (ValuesToEntries[Idx].contains(P.first)) {
P.second = Idx;
break;
}
}
}
bool IsSplatOrUndefs = isSplat(VL) || all_of(VL, IsaPred<UndefValue>);
// Checks if the 2 PHIs are compatible in terms of high possibility to be
// vectorized.
auto AreCompatiblePHIs = [&](Value *V, Value *V1) {
auto *PHI = cast<PHINode>(V);
auto *PHI1 = cast<PHINode>(V1);
// Check that all incoming values are compatible/from same parent (if they
// are instructions).
// The incoming values are compatible if they all are constants, or
// instruction with the same/alternate opcodes from the same basic block.
for (int I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) {
Value *In = PHI->getIncomingValue(I);
Value *In1 = PHI1->getIncomingValue(I);
if (isConstant(In) && isConstant(In1))
continue;
if (!getSameOpcode({In, In1}, *TLI))
return false;
if (cast<Instruction>(In)->getParent() !=
cast<Instruction>(In1)->getParent())
return false;
}
return true;
};
// Check if the value can be ignored during analysis for shuffled gathers.
// We suppose it is better to ignore instruction, which do not form splats,
// are not vectorized/not extractelements (these instructions will be handled
// by extractelements processing) or may form vector node in future.
// Cache results - each V in VL is queried up to 3 times (direct +
// NeighborMightBeIgnored from both neighbors), and areAllUsersVectorized
// walks each instruction's user list.
SmallDenseMap<Value *, bool> MightBeIgnoredCache;
auto MightBeIgnored = [=, &MightBeIgnoredCache](Value *V) {
auto [It, Inserted] = MightBeIgnoredCache.try_emplace(V);
if (!Inserted)
return It->second;
auto *I = dyn_cast<Instruction>(V);
bool Res = I && !IsSplatOrUndefs && !isVectorized(I) &&
!isVectorLikeInstWithConstOps(I) &&
!areAllUsersVectorized(I, UserIgnoreList) && isSimple(I);
It->second = Res;
return Res;
};
// Check that the neighbor instruction may form a full vector node with the
// current instruction V. It is possible, if they have same/alternate opcode
// and same parent basic block.
auto NeighborMightBeIgnored = [&](Value *V, int Idx) {
Value *V1 = VL[Idx];
bool UsedInSameVTE = false;
auto It = UsedValuesEntry.find(V1);
if (It != UsedValuesEntry.end())
UsedInSameVTE = It->second == UsedValuesEntry.find(V)->second;
return V != V1 && MightBeIgnored(V1) && !UsedInSameVTE &&
getSameOpcode({V, V1}, *TLI) &&
cast<Instruction>(V)->getParent() ==
cast<Instruction>(V1)->getParent() &&
(!isa<PHINode>(V1) || AreCompatiblePHIs(V, V1));
};
// Build a shuffle mask for better cost estimation and vector emission.
SmallBitVector UsedIdxs(Entries.size());
SmallVector<std::pair<unsigned, int>> EntryLanes;
for (int I = 0, E = VL.size(); I < E; ++I) {
Value *V = VL[I];
auto It = UsedValuesEntry.find(V);
if (It == UsedValuesEntry.end())
continue;
// Do not try to shuffle scalars, if they are constants, or instructions
// that can be vectorized as a result of the following vector build
// vectorization.
if (isConstant(V) || (MightBeIgnored(V) &&
((I > 0 && NeighborMightBeIgnored(V, I - 1)) ||
(I != E - 1 && NeighborMightBeIgnored(V, I + 1)))))
continue;
unsigned Idx = It->second;
EntryLanes.emplace_back(Idx, I);
UsedIdxs.set(Idx);
}
// Iterate through all shuffled scalars and select entries, which can be used
// for final shuffle.
SmallVector<const TreeEntry *> TempEntries;
for (unsigned I = 0, Sz = Entries.size(); I < Sz; ++I) {
if (!UsedIdxs.test(I))
continue;
// Fix the entry number for the given scalar. If it is the first entry, set
// Pair.first to 0, otherwise to 1 (currently select at max 2 nodes).
// These indices are used when calculating final shuffle mask as the vector
// offset.
for (std::pair<unsigned, int> &Pair : EntryLanes)
if (Pair.first == I)
Pair.first = TempEntries.size();
TempEntries.push_back(Entries[I]);
}
Entries.swap(TempEntries);
if (EntryLanes.size() == Entries.size() &&
!VL.equals(ArrayRef(TE->Scalars)
.slice(MaskBase, getNumElems(TE->Scalars.size(), SliceSize,
Part)))) {
// We may have here 1 or 2 entries only. If the number of scalars is equal
// to the number of entries, no need to do the analysis, it is not very
// profitable. Since VL is not the same as TE->Scalars, it means we already
// have some shuffles before. Cut off not profitable case.
Entries.clear();
return std::nullopt;
}
// Build the final mask, check for the identity shuffle, if possible.
bool IsIdentity = Entries.size() == 1;
// Pair.first is the offset to the vector, while Pair.second is the index of
// scalar in the list.
for (const std::pair<unsigned, int> &Pair : EntryLanes) {
unsigned Idx = MaskBase + Pair.second;
Mask[Idx] =
Pair.first * VF +
(ForOrder ? std::distance(
Entries[Pair.first]->Scalars.begin(),
find(Entries[Pair.first]->Scalars, VL[Pair.second]))
: Entries[Pair.first]->findLaneForValue(VL[Pair.second]));
IsIdentity &= Mask[Idx] == Pair.second;
}
if (ForOrder || IsIdentity || Entries.empty()) {
switch (Entries.size()) {
case 1:
if (IsIdentity || EntryLanes.size() > 1 || VL.size() <= 2)
return TargetTransformInfo::SK_PermuteSingleSrc;
break;
case 2:
if (EntryLanes.size() > 2 || VL.size() <= 2)
return TargetTransformInfo::SK_PermuteTwoSrc;
break;
default:
break;
}
} else if (!isa<VectorType>(VL.front()->getType()) &&
(EntryLanes.size() > Entries.size() || VL.size() <= 2)) {
// Do the cost estimation if shuffle beneficial than buildvector.
SmallVector<int> SubMask(std::next(Mask.begin(), MaskBase),
std::next(Mask.begin(), MaskBase + VL.size()));
int MinElement = SubMask.front(), MaxElement = SubMask.front();
for (int Idx : make_filter_range(
SubMask, [](int Idx) { return Idx != PoisonMaskElem; })) {
if (MinElement == PoisonMaskElem || MinElement % VF > Idx % VF)
MinElement = Idx;
if (MaxElement == PoisonMaskElem || MaxElement % VF < Idx % VF)
MaxElement = Idx;
}
assert(MaxElement >= 0 && MinElement >= 0 &&
MaxElement % VF >= MinElement % VF &&
"Expected at least single element.");
// If the leading [0, MinIdx) range sits in its own register part(s),
// skip those whole parts when sizing the destination - everything below
// the register-aligned floor is unused and never indexed.
unsigned Offset = 0;
unsigned MinIdx = MinElement % VF;
if (MinIdx > 1) {
unsigned RegFloor = getFloorFullVectorNumberOfElements(
*TTI, VL.front()->getType(), MinIdx, Opts.slp_revec);
auto *RegFloorTy = getWidenedType(VL.front()->getType(), RegFloor);
unsigned RegFloorParts =
getNumberOfParts(RegFloorTy, VL.front()->getType(), RegFloor);
if (RegFloorParts > 1)
Offset = RegFloor;
}
unsigned NewVF =
std::max<unsigned>(VL.size(), (MaxElement % VF) - Offset + 1);
if (NewVF < VF) {
for (int &Idx : make_filter_range(
SubMask, [](int Idx) { return Idx != PoisonMaskElem; }))
Idx = (Idx % VF) - Offset + (Idx >= static_cast<int>(VF) ? NewVF : 0);
} else {
NewVF = VF;
}
auto *VecTy =
cast<VectorType>(getWidenedType(VL.front()->getType(), NewVF));
auto *MaskVecTy =
cast<VectorType>(getWidenedType(VL.front()->getType(), SubMask.size()));
auto GetShuffleCost = [&,
&TTI = *TTI](ArrayRef<int> Mask,
ArrayRef<const TreeEntry *> Entries,
VectorType *VecTy) -> InstructionCost {
if (Entries.size() == 1 && Entries.front()->getInterleaveFactor() > 0 &&
ShuffleVectorInst::isDeInterleaveMaskOfFactor(
Mask, Entries.front()->getInterleaveFactor()))
return TTI::TCC_Free;
return getShuffleCost(TTI,
Entries.size() > 1 ? TTI::SK_PermuteTwoSrc
: TTI::SK_PermuteSingleSrc,
VecTy, CostKind, Mask);
};
InstructionCost ShuffleCost = GetShuffleCost(SubMask, Entries, VecTy);
InstructionCost FirstShuffleCost = 0;
SmallVector<int> FirstMask(SubMask.begin(), SubMask.end());
if (Entries.size() == 1 || !Entries[0]->isGather()) {
FirstShuffleCost = ShuffleCost;
} else {
// Transform mask to include only first entry.
APInt DemandedElts = APInt::getAllOnes(SubMask.size());
bool IsIdentity = true;
for (auto [I, Idx] : enumerate(FirstMask)) {
if (Idx >= static_cast<int>(NewVF)) {
Idx = PoisonMaskElem;
} else {
DemandedElts.clearBit(I);
if (Idx != PoisonMaskElem)
IsIdentity &= static_cast<int>(I) == Idx;
}
}
if (!IsIdentity)
FirstShuffleCost = GetShuffleCost(FirstMask, Entries.front(), VecTy);
FirstShuffleCost +=
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
InstructionCost SecondShuffleCost = 0;
SmallVector<int> SecondMask(SubMask.begin(), SubMask.end());
if (Entries.size() == 1 || !Entries[1]->isGather()) {
SecondShuffleCost = ShuffleCost;
} else {
// Transform mask to include only first entry.
APInt DemandedElts = APInt::getAllOnes(SubMask.size());
bool IsIdentity = true;
for (auto [I, Idx] : enumerate(SecondMask)) {
if (Idx < static_cast<int>(NewVF) && Idx >= 0) {
Idx = PoisonMaskElem;
} else {
DemandedElts.clearBit(I);
if (Idx != PoisonMaskElem) {
Idx -= NewVF;
IsIdentity &= static_cast<int>(I) == Idx;
}
}
}
if (!IsIdentity)
SecondShuffleCost = GetShuffleCost(SecondMask, Entries[1], VecTy);
SecondShuffleCost +=
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
APInt DemandedElts = APInt::getAllOnes(SubMask.size());
for (auto [I, Idx] : enumerate(SubMask))
if (Idx == PoisonMaskElem)
DemandedElts.clearBit(I);
InstructionCost BuildVectorCost =
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
const TreeEntry *BestEntry = nullptr;
auto MaskSlice = MutableArrayRef(Mask).slice(MaskBase, VL.size());
if (FirstShuffleCost < ShuffleCost) {
for (int &Idx : MaskSlice)
if (Idx >= static_cast<int>(VF))
Idx = PoisonMaskElem;
BestEntry = Entries.front();
ShuffleCost = FirstShuffleCost;
}
if (SecondShuffleCost < ShuffleCost) {
for (int &Idx : MaskSlice) {
if (Idx < static_cast<int>(VF))
Idx = PoisonMaskElem;
else
Idx -= VF;
}
BestEntry = Entries[1];
ShuffleCost = SecondShuffleCost;
}
if (BuildVectorCost >= ShuffleCost) {
if (BestEntry) {
Entries.clear();
Entries.push_back(BestEntry);
}
return Entries.size() > 1 ? TargetTransformInfo::SK_PermuteTwoSrc
: TargetTransformInfo::SK_PermuteSingleSrc;
}
}
Entries.clear();
// Clear the corresponding mask elements.
std::fill(std::next(Mask.begin(), MaskBase),
std::next(Mask.begin(), MaskBase + VL.size()), PoisonMaskElem);
return std::nullopt;
}
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>>
BoUpSLP::isGatherShuffledEntry(
const TreeEntry *TE, ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
SmallVectorImpl<SmallVector<const TreeEntry *>> &Entries, unsigned NumParts,
Type *ScalarTy, bool ForOrder) {
assert(NumParts > 0 && NumParts < VL.size() &&
"Expected positive number of registers.");
Entries.clear();
// No need to check for the topmost gather node.
if (TE == &getRootNode() &&
(!GatheredLoadsEntriesFirst.has_value() ||
none_of(ArrayRef(VectorizableTree).drop_front(),
[](const std::unique_ptr<TreeEntry> &TE) {
return !TE->isGather();
})))
return {};
Mask.assign(VL.size(), PoisonMaskElem);
assert((TE->UserTreeIndex || TE == &getRootNode()) &&
"Expected only single user of the gather node.");
unsigned PWSz =
getFullVectorNumberOfElements(*TTI, ScalarTy, VL.size(), Opts.slp_revec);
if (TE->UserTreeIndex && TE->UserTreeIndex.UserTE->isGather() &&
TE->UserTreeIndex.EdgeIdx == UINT_MAX &&
(TE->Idx == 0 ||
(TE->hasState() && TE->getOpcode() == Instruction::ExtractElement) ||
isSplat(TE->Scalars) ||
(TE->hasState() &&
getSameValuesTreeEntry(TE->getMainOp(), TE->Scalars))))
return {};
unsigned SliceSize = getPartNumElems(PWSz, NumParts);
SmallVector<std::optional<TTI::ShuffleKind>> Res;
for (unsigned Part : seq<unsigned>(NumParts)) {
if (Part * SliceSize >= VL.size())
break;
ArrayRef<Value *> SubVL =
VL.slice(Part * SliceSize, getNumElems(VL.size(), SliceSize, Part));
SmallVectorImpl<const TreeEntry *> &SubEntries = Entries.emplace_back();
std::optional<TTI::ShuffleKind> SubRes =
isGatherShuffledSingleRegisterEntry(TE, SubVL, Mask, SubEntries, Part,
ForOrder, SliceSize);
if (!SubRes)
SubEntries.clear();
Res.push_back(SubRes);
if (SubEntries.size() == 1 && *SubRes == TTI::SK_PermuteSingleSrc &&
SubEntries.front()->getVectorFactor() == VL.size() &&
(SubEntries.front()->isSame(TE->Scalars) ||
SubEntries.front()->isSame(VL))) {
SmallVector<const TreeEntry *> LocalSubEntries;
LocalSubEntries.swap(SubEntries);
Entries.clear();
Res.clear();
std::iota(Mask.begin(), Mask.end(), 0);
// Clear undef scalars.
for (int I = 0, Sz = VL.size(); I < Sz; ++I)
if (isa<PoisonValue>(VL[I]))
Mask[I] = PoisonMaskElem;
Entries.emplace_back(1, LocalSubEntries.front());
Res.push_back(TargetTransformInfo::SK_PermuteSingleSrc);
return Res;
}
}
if (all_of(Res,
[](const std::optional<TTI::ShuffleKind> &SK) { return !SK; })) {
Entries.clear();
return {};
}
return Res;
}
InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
Type *ScalarTy) const {
const unsigned VF = VL.size();
auto *VecTy = getWidenedType(ScalarTy, VF);
// Find the cost of inserting/extracting values from the vector.
// Check if the same elements are inserted several times and count them as
// shuffle candidates.
APInt DemandedElements = APInt::getZero(VF);
InstructionCost Cost;
auto EstimateInsertCost = [&](unsigned I, Value *V) {
DemandedElements.setBit(I);
if (V->getType() != ScalarTy)
Cost += TTI->getCastInstrCost(Instruction::Trunc, ScalarTy, V->getType(),
TTI::CastContextHint::None, CostKind);
};
SmallVector<int> ConstantShuffleMask(VF, PoisonMaskElem);
std::iota(ConstantShuffleMask.begin(), ConstantShuffleMask.end(), 0);
// No need to shuffle duplicates for constants.
for (auto [I, V] : make_filter_range(enumerate(VL), [&](const auto &P) {
return !(ForPoisonSrc && isConstant(P.value())) &&
!isa<UndefValue>(P.value());
})) {
if (isConstant(V)) {
ConstantShuffleMask[I] = I + VF;
continue;
}
EstimateInsertCost(I, V);
}
// FIXME: add a cost for constant vector materialization.
bool IsAnyNonUndefConst =
any_of(VL, [](Value *V) { return !isa<UndefValue>(V) && isConstant(V); });
// 1. Shuffle input source vector and constant vector.
if (!ForPoisonSrc && IsAnyNonUndefConst) {
Cost +=
getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteTwoSrc,
cast<VectorType>(VecTy), CostKind, ConstantShuffleMask);
}
// 2. Insert unique non-constants.
if (!DemandedElements.isZero())
Cost += getScalarizationOverhead(*TTI, Opts.slp_revec, ScalarTy,
cast<VectorType>(VecTy), DemandedElements,
/*Insert=*/true,
/*Extract=*/false, CostKind,
ForPoisonSrc && !IsAnyNonUndefConst, VL);
return Cost;
}
Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
auto It = EntryToLastInstruction.find(E);
if (It != EntryToLastInstruction.end())
return *cast<Instruction>(It->second);
Instruction *Res = nullptr;
// Get the basic block this bundle is in. All instructions in the bundle
// should be in this block (except for extractelement-like instructions with
// constant indices or gathered loads or copyables).
Instruction *Front;
unsigned Opcode;
if (E->hasState()) {
Front = E->getMainOp();
Opcode = E->getOpcode();
} else {
Front = cast<Instruction>(*find_if(E->Scalars, IsaPred<Instruction>));
Opcode = Front->getOpcode();
}
auto *BB = Front->getParent();
assert(
((GatheredLoadsEntriesFirst.has_value() && Opcode == Instruction::Load &&
E->isGather() && E->Idx < *GatheredLoadsEntriesFirst) ||
E->State == TreeEntry::SplitVectorize || E->hasCopyableElements() ||
all_of(E->Scalars,
[=](Value *V) -> bool {
if (Opcode == Instruction::GetElementPtr &&
!isa<GetElementPtrInst>(V))
return true;
auto *I = dyn_cast<Instruction>(V);
return !I || !E->getMatchingMainOpOrAltOp(I) ||
I->getParent() == BB || isVectorLikeInstWithConstOps(I);
})) &&
"Expected gathered loads or GEPs or instructions from same basic "
"block.");
auto FindLastInst = [&]() {
Instruction *LastInst = Front;
for (Instruction *I : make_isa_range<Instruction>(E->Scalars)) {
if (E->isCopyableElement(I))
continue;
if (LastInst->getParent() == I->getParent()) {
if (LastInst->comesBefore(I))
LastInst = I;
continue;
}
assert(((Opcode == Instruction::GetElementPtr &&
!isa<GetElementPtrInst>(I)) ||
E->State == TreeEntry::SplitVectorize ||
(isVectorLikeInstWithConstOps(LastInst) &&
isVectorLikeInstWithConstOps(I)) ||
(GatheredLoadsEntriesFirst.has_value() &&
Opcode == Instruction::Load && E->isGather() &&
E->Idx < *GatheredLoadsEntriesFirst)) &&
"Expected vector-like or non-GEP in GEP node insts only.");
if (!DT->isReachableFromEntry(LastInst->getParent())) {
LastInst = I;
continue;
}
if (!DT->isReachableFromEntry(I->getParent()))
continue;
auto *NodeA = DT->getNode(LastInst->getParent());
auto *NodeB = DT->getNode(I->getParent());
assert(NodeA && "Should only process reachable instructions");
assert(NodeB && "Should only process reachable instructions");
assert((NodeA == NodeB) ==
(NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeA->getDFSNumIn() < NodeB->getDFSNumIn())
LastInst = I;
}
BB = LastInst->getParent();
return LastInst;
};
auto FindFirstInst = [&]() {
Instruction *FirstInst = Front;
for (Instruction *I : make_isa_range<Instruction>(E->Scalars)) {
if (E->isCopyableElement(I))
continue;
if (FirstInst->getParent() == I->getParent()) {
if (I->comesBefore(FirstInst))
FirstInst = I;
continue;
}
assert(((Opcode == Instruction::GetElementPtr &&
!isa<GetElementPtrInst>(I)) ||
(isVectorLikeInstWithConstOps(FirstInst) &&
isVectorLikeInstWithConstOps(I))) &&
"Expected vector-like or non-GEP in GEP node insts only.");
if (!DT->isReachableFromEntry(FirstInst->getParent())) {
FirstInst = I;
continue;
}
if (!DT->isReachableFromEntry(I->getParent()))
continue;
auto *NodeA = DT->getNode(FirstInst->getParent());
auto *NodeB = DT->getNode(I->getParent());
assert(NodeA && "Should only process reachable instructions");
assert(NodeB && "Should only process reachable instructions");
assert((NodeA == NodeB) ==
(NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeA->getDFSNumIn() > NodeB->getDFSNumIn())
FirstInst = I;
}
return FirstInst;
};
if (E->State == TreeEntry::SplitVectorize) {
Res = FindLastInst();
if (ArrayRef<TreeEntry *> Entries = getTreeEntries(Res); !Entries.empty()) {
for (auto *E : Entries) {
auto *I = dyn_cast_if_present<Instruction>(E->VectorizedValue);
if (!I)
I = &getLastInstructionInBundle(E);
if (Res->getParent() == I->getParent() && Res->comesBefore(I))
Res = I;
}
}
EntryToLastInstruction.try_emplace(E, Res);
return *Res;
}
// Set insertpoint for gathered loads to the very first load.
if (GatheredLoadsEntriesFirst.has_value() &&
E->Idx >= *GatheredLoadsEntriesFirst && !E->isGather() &&
Opcode == Instruction::Load) {
Res = FindFirstInst();
EntryToLastInstruction.try_emplace(E, Res);
return *Res;
}
// Set the insert point to the beginning of the basic block if the entry
// should not be scheduled.
auto FindScheduleBundle = [&](const TreeEntry *E) -> const ScheduleBundle * {
if (E->isGather())
return nullptr;
// Found previously that the instruction do not need to be scheduled.
const auto *It = BlocksSchedules.find(BB);
if (It == BlocksSchedules.end())
return nullptr;
for (Instruction *I : make_isa_range<Instruction>(E->Scalars)) {
if (isa<PHINode>(I) ||
(!E->isCopyableElement(I) && doesNotNeedToBeScheduled(I)))
continue;
ArrayRef<ScheduleBundle *> Bundles = It->second->getScheduleBundles(I);
if (Bundles.empty())
continue;
const auto *It = find_if(
Bundles, [&](ScheduleBundle *B) { return B->getTreeEntry() == E; });
if (It != Bundles.end())
return *It;
}
return nullptr;
};
const ScheduleBundle *Bundle = FindScheduleBundle(E);
if (!E->isGather() && !Bundle) {
if ((Opcode == Instruction::GetElementPtr &&
any_of(E->Scalars,
[](Value *V) {
return !isa<GetElementPtrInst>(V) && isa<Instruction>(V);
})) ||
(all_of(E->Scalars,
[&](Value *V) {
return isa<PoisonValue>(V) ||
(E->Idx == 0 &&
isa<InsertElementInst, InsertValueInst>(V)) ||
E->isCopyableElement(V) ||
(!isVectorLikeInstWithConstOps(V) &&
isUsedOutsideBlock(V));
}) &&
(!E->doesNotNeedToSchedule() ||
any_of(make_isa_range<Instruction>(E->Scalars),
[&](Instruction *I) {
if (E->hasCopyableElements() && E->isCopyableElement(I))
return false;
return !areAllOperandsNonInsts(I);
}) ||
none_of(make_isa_range<Instruction>(E->Scalars), [&](Instruction *I) {
if (E->hasCopyableElements() && E->isCopyableElement(I))
return false;
return MustGather.contains(I);
}))))
Res = FindLastInst();
else
Res = FindFirstInst();
EntryToLastInstruction.try_emplace(E, Res);
return *Res;
}
// Find the last instruction. The common case should be that BB has been
// scheduled, and the last instruction is VL.back(). So we start with
// VL.back() and iterate over schedule data until we reach the end of the
// bundle. The end of the bundle is marked by null ScheduleData.
if (Bundle) {
assert(!E->isGather() && "Gathered instructions should not be scheduled");
Res = Bundle->getBundle().back()->getInst();
EntryToLastInstruction.try_emplace(E, Res);
return *Res;
}
// LastInst can still be null at this point if there's either not an entry
// for BB in BlocksSchedules or there's no ScheduleData available for
// VL.back(). This can be the case if buildTreeRec aborts for various
// reasons (e.g., the maximum recursion depth is reached, the maximum region
// size is reached, etc.). ScheduleData is initialized in the scheduling
// "dry-run".
//
// If this happens, we can still find the last instruction by brute force. We
// iterate forwards from Front (inclusive) until we either see all
// instructions in the bundle or reach the end of the block. If Front is the
// last instruction in program order, LastInst will be set to Front, and we
// will visit all the remaining instructions in the block.
//
// One of the reasons we exit early from buildTreeRec is to place an upper
// bound on compile-time. Thus, taking an additional compile-time hit here is
// not ideal. However, this should be exceedingly rare since it requires that
// we both exit early from buildTreeRec and that the bundle be out-of-order
// (causing us to iterate all the way to the end of the block).
if (!Res)
Res = FindLastInst();
assert(Res && "Failed to find last instruction in bundle");
EntryToLastInstruction.try_emplace(E, Res);
return *Res;
}
void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
auto *Front = E->getMainOp();
Instruction *LastInst = &getLastInstructionInBundle(E);
// Struct-call extractvalue entries may have scalars in different basic
// blocks. The vector extractvalue depends only on the vectorized call, so
// use the call bundle to set the insert point and dominate all uses.
const bool UseCallBundleInsertPoint =
E->getOpcode() == Instruction::ExtractValue &&
!E->StructEVIndices.empty() &&
any_of(E->Scalars, [BB = Front->getParent()](Value *V) {
return cast<Instruction>(V)->getParent() != BB;
});
if (UseCallBundleInsertPoint)
LastInst = &getLastInstructionInBundle(getOperandEntry(E, 0));
assert(LastInst && "Failed to find last instruction in bundle");
BasicBlock::iterator LastInstIt = LastInst->getIterator();
// If the instruction is PHI, set the insert point after all the PHIs.
bool IsPHI = isa<PHINode>(LastInst);
if (IsPHI) {
LastInstIt = LastInst->getParent()->getFirstNonPHIIt();
if (LastInstIt != LastInst->getParent()->end() &&
LastInstIt->getParent()->isLandingPad())
LastInstIt = std::next(LastInstIt);
}
if (IsPHI ||
(!E->isGather() && E->State != TreeEntry::SplitVectorize &&
!UseCallBundleInsertPoint &&
(E->doesNotNeedToSchedule() ||
(E->hasCopyableElements() && !E->isCopyableElement(LastInst) &&
isUsedOutsideBlock(LastInst)))) ||
(GatheredLoadsEntriesFirst.has_value() &&
E->Idx >= *GatheredLoadsEntriesFirst && !E->isGather() &&
E->getOpcode() == Instruction::Load)) {
Builder.SetInsertPoint(LastInstIt);
} else {
// Set the insertion point after the last instruction in the bundle. Set the
// debug location to Front.
Builder.SetInsertPoint(LastInst->getNextNode()->getIterator());
if (Instruction *Res = LastInstructionToPos.lookup(LastInst)) {
Builder.SetInsertPoint(Res->getIterator());
} else {
Res = Builder.CreateAlignedLoad(Builder.getPtrTy(),
PoisonValue::get(Builder.getPtrTy()),
MaybeAlign());
Builder.SetInsertPoint(Res->getIterator());
eraseInstruction(Res);
if (E->State != TreeEntry::SplitVectorize)
LastInstructionToPos.try_emplace(LastInst, Res);
}
}
Builder.SetCurrentDebugLocation(Front->getDebugLoc());
}
Value *BoUpSLP::gather(
ArrayRef<Value *> VL, Value *Root, Type *ScalarTy,
function_ref<Value *(Value *, Value *, ArrayRef<int>)> CreateShuffle) {
// List of instructions/lanes from current block and/or the blocks which are
// part of the current loop. These instructions will be inserted at the end to
// make it possible to optimize loops and hoist invariant instructions out of
// the loops body with better chances for success.
SmallVector<std::pair<Value *, unsigned>, 4> PostponedInsts;
SmallSet<int, 4> PostponedIndices;
Loop *L = LI->getLoopFor(Builder.GetInsertBlock());
auto &&CheckPredecessor = [](BasicBlock *InstBB, BasicBlock *InsertBB) {
SmallPtrSet<BasicBlock *, 4> Visited;
while (InsertBB && InsertBB != InstBB && Visited.insert(InsertBB).second)
InsertBB = InsertBB->getSinglePredecessor();
return InsertBB && InsertBB == InstBB;
};
for (int I = 0, E = VL.size(); I < E; ++I) {
if (auto *Inst = dyn_cast<Instruction>(VL[I]))
if ((CheckPredecessor(Inst->getParent(), Builder.GetInsertBlock()) ||
isVectorized(Inst) ||
(L && (!Root || L->isLoopInvariant(Root)) && L->contains(Inst))) &&
PostponedIndices.insert(I).second)
PostponedInsts.emplace_back(Inst, I);
}
auto &&CreateInsertElement = [this](Value *Vec, Value *V, unsigned Pos,
Type *Ty) {
Value *Scalar = V;
// Drop NUW from trunc to avoid incorrect codegen.
Value *Trunced;
if (match(Scalar, m_NUWTrunc(m_Value(Trunced))))
cast<TruncInst>(Scalar)->setHasNoUnsignedWrap(/*B=*/false);
if (Scalar->getType() != Ty) {
assert(Scalar->getType()->isIntOrIntVectorTy() &&
Ty->isIntOrIntVectorTy() && "Expected integer types only.");
Value *V = Scalar;
if (auto *CI = dyn_cast<CastInst>(Scalar);
isa_and_nonnull<SExtInst, ZExtInst>(CI)) {
Value *Op = CI->getOperand(0);
if (auto *IOp = dyn_cast<Instruction>(Op);
!IOp || !(isDeleted(IOp) || isVectorized(IOp)))
V = Op;
}
Scalar = Builder.CreateIntCast(
V, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
}
Instruction *InsElt;
if (auto *VecTy = dyn_cast<FixedVectorType>(Scalar->getType())) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
Vec =
createInsertVector(Builder, Vec, Scalar, Pos * getNumElements(VecTy));
auto *II = dyn_cast<Instruction>(Vec);
if (!II)
return Vec;
InsElt = II;
} else {
Vec = Builder.CreateInsertElement(Vec, Scalar, Pos);
InsElt = dyn_cast<InsertElementInst>(Vec);
if (!InsElt)
return Vec;
}
GatherShuffleExtractSeq.insert(InsElt);
CSEBlocks.insert(InsElt->getParent());
// Add to our 'need-to-extract' list.
if (isa<Instruction>(V)) {
ArrayRef<TreeEntry *> Entries = getTreeEntries(V);
const auto *It = find_if(Entries, [&](const TreeEntry *E) {
return !TransformedToGatherNodes.contains(E) &&
!DeletedNodes.contains(E);
});
if (It != Entries.end()) {
// Find which lane we need to extract.
User *UserOp = nullptr;
if (Scalar != V) {
if (auto *SI = dyn_cast<Instruction>(Scalar))
UserOp = SI;
} else {
if (V->getType()->isVectorTy()) {
if (auto *SV = dyn_cast<ShuffleVectorInst>(InsElt);
SV && SV->getOperand(0) != V && SV->getOperand(1) != V) {
// Find shufflevector, caused by resize.
auto FindOperand = [](Value *Vec, Value *V) -> Instruction * {
if (auto *SV = dyn_cast<ShuffleVectorInst>(Vec)) {
if (SV->getOperand(0) == V)
return SV;
if (SV->getOperand(1) == V)
return SV;
}
return nullptr;
};
InsElt = nullptr;
if (Instruction *User = FindOperand(SV->getOperand(0), V))
InsElt = User;
else if (Instruction *User = FindOperand(SV->getOperand(1), V))
InsElt = User;
assert(InsElt &&
"Failed to find shufflevector, caused by resize.");
} else if (Opts.slp_revec && isa<ShuffleVectorInst>(InsElt)) {
// ReVec gather used V directly as a shufflevector operand.
// Register a nullptr-User external use so all remaining
// in-IR uses of V get rewritten via replaceAllUsesWith,
// and track V in ExternalUsesWithNonUsers to match the
// bookkeeping done by buildExternalUses.
unsigned FoundLane = (*It)->findLaneForValue(V);
ExternalUses.emplace_back(V, nullptr, **It, FoundLane);
ExternalUsesWithNonUsers.insert(V);
}
}
UserOp = InsElt;
}
if (UserOp) {
unsigned FoundLane = (*It)->findLaneForValue(V);
ExternalUses.emplace_back(V, UserOp, **It, FoundLane);
}
}
}
return Vec;
};
auto *VecTy = getWidenedType(ScalarTy, VL.size());
Value *Vec = PoisonValue::get(VecTy);
SmallVector<int> NonConsts;
SmallVector<int> Mask(VL.size());
std::iota(Mask.begin(), Mask.end(), 0);
Value *OriginalRoot = Root;
if (auto *SV = dyn_cast_if_present<ShuffleVectorInst>(Root);
SV && isa<PoisonValue>(SV->getOperand(1)) &&
SV->getOperand(0)->getType() == VecTy) {
Root = SV->getOperand(0);
Mask.assign(SV->getShuffleMask().begin(), SV->getShuffleMask().end());
}
// Insert constant values at first.
for (int I = 0, E = VL.size(); I < E; ++I) {
if (PostponedIndices.contains(I))
continue;
if (!isConstant(VL[I])) {
NonConsts.push_back(I);
continue;
}
if (isa<PoisonValue>(VL[I]))
continue;
Vec = CreateInsertElement(Vec, VL[I], I, ScalarTy);
Mask[I] = I + E;
}
if (Root) {
if (isa<PoisonValue>(Vec)) {
Vec = OriginalRoot;
} else {
Vec = CreateShuffle(Root, Vec, Mask);
if (auto *OI = dyn_cast<Instruction>(OriginalRoot);
OI && OI->use_empty() &&
none_of(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->VectorizedValue == OI;
}))
eraseInstruction(OI);
}
}
// Insert non-constant values.
for (int I : NonConsts)
Vec = CreateInsertElement(Vec, VL[I], I, ScalarTy);
// Append instructions, which are/may be part of the loop, in the end to make
// it possible to hoist non-loop-based instructions.
for (const std::pair<Value *, unsigned> &Pair : PostponedInsts)
Vec = CreateInsertElement(Vec, Pair.first, Pair.second, ScalarTy);
return Vec;
}
/// Merges shuffle masks and emits final shuffle instruction, if required. It
/// supports shuffling of 2 input vectors. It implements lazy shuffles emission,
/// when the actual shuffle instruction is generated only if this is actually
/// required. Otherwise, the shuffle instruction emission is delayed till the
/// end of the process, to reduce the number of emitted instructions and further
/// analysis/transformations.
/// The class also will look through the previously emitted shuffle instructions
/// and properly mark indices in mask as undef.
/// For example, given the code
/// \code
/// %s1 = shufflevector <2 x ty> %0, poison, <1, 0>
/// %s2 = shufflevector <2 x ty> %1, poison, <1, 0>
/// \endcode
/// and if need to emit shuffle of %s1 and %s2 with mask <1, 0, 3, 2>, it will
/// look through %s1 and %s2 and emit
/// \code
/// %res = shufflevector <2 x ty> %0, %1, <0, 1, 2, 3>
/// \endcode
/// instead.
/// If 2 operands are of different size, the smallest one will be resized and
/// the mask recalculated properly.
/// For example, given the code
/// \code
/// %s1 = shufflevector <2 x ty> %0, poison, <1, 0, 1, 0>
/// %s2 = shufflevector <2 x ty> %1, poison, <1, 0, 1, 0>
/// \endcode
/// and if need to emit shuffle of %s1 and %s2 with mask <1, 0, 5, 4>, it will
/// look through %s1 and %s2 and emit
/// \code
/// %res = shufflevector <2 x ty> %0, %1, <0, 1, 2, 3>
/// \endcode
/// instead.
class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
bool IsFinalized = false;
/// Combined mask for all applied operands and masks. It is built during
/// analysis and actual emission of shuffle vector instructions.
SmallVector<int> CommonMask;
/// List of operands for the shuffle vector instruction. It hold at max 2
/// operands, if the 3rd is going to be added, the first 2 are combined into
/// shuffle with \p CommonMask mask, the first operand sets to be the
/// resulting shuffle and the second operand sets to be the newly added
/// operand. The \p CommonMask is transformed in the proper way after that.
SmallVector<Value *, 2> InVectors;
IRBuilderBase &Builder;
BoUpSLP &R;
class ShuffleIRBuilder {
IRBuilderBase &Builder;
/// Holds all of the instructions that we gathered.
SetVector<Instruction *> &GatherShuffleExtractSeq;
/// A list of blocks that we are going to CSE.
DenseSet<BasicBlock *> &CSEBlocks;
/// Data layout.
const DataLayout &DL;
public:
ShuffleIRBuilder(IRBuilderBase &Builder,
SetVector<Instruction *> &GatherShuffleExtractSeq,
DenseSet<BasicBlock *> &CSEBlocks, const DataLayout &DL)
: Builder(Builder), GatherShuffleExtractSeq(GatherShuffleExtractSeq),
CSEBlocks(CSEBlocks), DL(DL) {}
~ShuffleIRBuilder() = default;
/// Creates shufflevector for the 2 operands with the given mask.
Value *createShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask) {
if (V1->getType() != V2->getType()) {
assert(V1->getType()->isIntOrIntVectorTy() &&
V1->getType()->isIntOrIntVectorTy() &&
"Expected integer vector types only.");
if (V1->getType() != V2->getType()) {
if (cast<VectorType>(V2->getType())
->getElementType()
->getIntegerBitWidth() < cast<VectorType>(V1->getType())
->getElementType()
->getIntegerBitWidth())
V2 = Builder.CreateIntCast(
V2, V1->getType(), !isKnownNonNegative(V2, SimplifyQuery(DL)));
else
V1 = Builder.CreateIntCast(
V1, V2->getType(), !isKnownNonNegative(V1, SimplifyQuery(DL)));
}
}
Value *Vec = Builder.CreateShuffleVector(V1, V2, Mask);
if (auto *I = dyn_cast<Instruction>(Vec)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
return Vec;
}
/// Creates permutation of the single vector operand with the given mask, if
/// it is not identity mask.
Value *createShuffleVector(Value *V1, ArrayRef<int> Mask) {
if (Mask.empty())
return V1;
unsigned VF = Mask.size();
unsigned LocalVF = cast<FixedVectorType>(V1->getType())->getNumElements();
if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF))
return V1;
Value *Vec = Builder.CreateShuffleVector(V1, Mask);
if (auto *I = dyn_cast<Instruction>(Vec)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
return Vec;
}
Value *createIdentity(Value *V) { return V; }
Value *createPoison(Type *Ty, unsigned VF) {
return PoisonValue::get(getWidenedType(Ty, VF));
}
/// Resizes 2 input vector to match the sizes, if the they are not equal
/// yet. The smallest vector is resized to the size of the larger vector.
void resizeToMatch(Value *&V1, Value *&V2) {
if (V1->getType() == V2->getType())
return;
int V1VF = cast<FixedVectorType>(V1->getType())->getNumElements();
int V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
int VF = std::max(V1VF, V2VF);
int MinVF = std::min(V1VF, V2VF);
SmallVector<int> IdentityMask(VF, PoisonMaskElem);
std::iota(IdentityMask.begin(), std::next(IdentityMask.begin(), MinVF),
0);
Value *&Op = MinVF == V1VF ? V1 : V2;
Op = Builder.CreateShuffleVector(Op, IdentityMask);
if (auto *I = dyn_cast<Instruction>(Op)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
if (MinVF == V1VF)
V1 = Op;
else
V2 = Op;
}
};
/// Smart shuffle instruction emission, walks through shuffles trees and
/// tries to find the best matching vector for the actual shuffle
/// instruction.
Value *createShuffle(Value *V1, Value *V2, ArrayRef<int> Mask) {
assert(V1 && "Expected at least one vector value.");
ShuffleIRBuilder ShuffleBuilder(Builder, R.GatherShuffleExtractSeq,
R.CSEBlocks, *R.DL);
return BaseShuffleAnalysis::createShuffle<Value *>(
V1, V2, Mask, ShuffleBuilder, ScalarTy, R.Opts.slp_revec);
}
/// Cast value \p V to the vector type with the same number of elements, but
/// the base type \p ScalarTy.
Value *castToScalarTyElem(Value *V,
std::optional<bool> IsSigned = std::nullopt) {
auto *VecTy = cast<VectorType>(V->getType());
assert(getNumElements(VecTy) % getNumElements(ScalarTy) == 0);
if (VecTy->getElementType() == ScalarTy->getScalarType())
return V;
return Builder.CreateIntCast(
V, VectorType::get(ScalarTy->getScalarType(), VecTy->getElementCount()),
IsSigned.value_or(!isKnownNonNegative(V, SimplifyQuery(*R.DL))));
}
Value *getVectorizedValue(const TreeEntry &E) {
Value *Vec = E.VectorizedValue;
if (!Vec->getType()->isIntOrIntVectorTy())
return Vec;
return castToScalarTyElem(Vec, any_of(E.Scalars, [&](Value *V) {
return !isa<PoisonValue>(V) &&
!isKnownNonNegative(
V, SimplifyQuery(*R.DL));
}));
}
public:
ShuffleInstructionBuilder(Type *ScalarTy, IRBuilderBase &Builder, BoUpSLP &R)
: BaseShuffleAnalysis(ScalarTy), Builder(Builder), R(R) {}
/// Adjusts extractelements after reusing them.
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
unsigned NumParts, bool &UseVecBaseAsInput) {
UseVecBaseAsInput = false;
SmallPtrSet<Value *, 4> UniqueBases;
Value *VecBase = nullptr;
SmallVector<Value *> VL(E->Scalars.begin(), E->Scalars.end());
if (!E->ReorderIndices.empty()) {
SmallVector<int> ReorderMask(E->ReorderIndices.begin(),
E->ReorderIndices.end());
reorderScalars(VL, ReorderMask);
}
for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
int Idx = Mask[I];
if (Idx == PoisonMaskElem)
continue;
auto *EI = cast<ExtractElementInst>(VL[I]);
VecBase = EI->getVectorOperand();
if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecBase); !TEs.empty())
VecBase = TEs.front()->VectorizedValue;
assert(VecBase && "Expected vectorized value.");
UniqueBases.insert(VecBase);
// If the only one use is vectorized - can delete the extractelement
// itself.
if (!EI->hasOneUse() || R.ExternalUsesAsOriginalScalar.contains(EI) ||
(E->UserTreeIndex && E->UserTreeIndex.EdgeIdx == UINT_MAX &&
!R.isVectorized(EI) &&
count_if(E->Scalars, [&](Value *V) { return V == EI; }) !=
count_if(E->UserTreeIndex.UserTE->Scalars,
[&](Value *V) { return V == EI; })) ||
(NumParts != 1 && count(VL, EI) > 1) ||
any_of(EI->users(), [&](User *U) {
ArrayRef<TreeEntry *> UTEs = R.getTreeEntries(U);
return UTEs.empty() || UTEs.size() > 1 ||
any_of(UTEs,
[&](const TreeEntry *TE) {
return R.DeletedNodes.contains(TE) ||
R.TransformedToGatherNodes.contains(TE);
}) ||
(isa<GetElementPtrInst>(U) &&
!R.areAllUsersVectorized(cast<Instruction>(U))) ||
(!UTEs.empty() &&
count_if(R.VectorizableTree,
[&](const std::unique_ptr<TreeEntry> &TE) {
return TE->UserTreeIndex.UserTE ==
UTEs.front() &&
is_contained(VL, EI);
}) != 1);
}))
continue;
R.eraseInstruction(EI);
}
if (NumParts == 1 || UniqueBases.size() == 1) {
assert(VecBase && "Expected vectorized value.");
return castToScalarTyElem(VecBase);
}
UseVecBaseAsInput = true;
auto TransformToIdentity = [](MutableArrayRef<int> Mask) {
for (auto [I, Idx] : enumerate(Mask))
if (Idx != PoisonMaskElem)
Idx = I;
};
// Perform multi-register vector shuffle, joining them into a single virtual
// long vector.
// Need to shuffle each part independently and then insert all this parts
// into a long virtual vector register, forming the original vector.
Value *Vec = nullptr;
SmallVector<int> VecMask(Mask.size(), PoisonMaskElem);
unsigned SliceSize = getPartNumElems(VL.size(), NumParts);
for (unsigned Part : seq<unsigned>(NumParts)) {
unsigned Limit = getNumElems(VL.size(), SliceSize, Part);
ArrayRef<Value *> SubVL = ArrayRef(VL).slice(Part * SliceSize, Limit);
MutableArrayRef<int> SubMask = Mask.slice(Part * SliceSize, Limit);
constexpr int MaxBases = 2;
SmallVector<Value *, MaxBases> Bases(MaxBases);
auto VLMask = zip(SubVL, SubMask);
const unsigned VF =
accumulate(VLMask, 0U, [&](unsigned S, const auto &D) {
if (std::get<1>(D) == PoisonMaskElem)
return S;
Value *VecOp =
cast<ExtractElementInst>(std::get<0>(D))->getVectorOperand();
if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp);
!TEs.empty())
VecOp = TEs.front()->VectorizedValue;
assert(VecOp && "Expected vectorized value.");
const unsigned Size =
cast<FixedVectorType>(VecOp->getType())->getNumElements();
return std::max(S, Size);
});
for (const auto [V, I] : make_filter_range(VLMask, [](const auto &P) {
return std::get<1>(P) != PoisonMaskElem;
})) {
Value *VecOp = cast<ExtractElementInst>(V)->getVectorOperand();
if (ArrayRef<TreeEntry *> TEs = R.getTreeEntries(VecOp); !TEs.empty())
VecOp = TEs.front()->VectorizedValue;
assert(VecOp && "Expected vectorized value.");
VecOp = castToScalarTyElem(VecOp);
Bases[I / VF] = VecOp;
}
if (!Bases.front())
continue;
Value *SubVec;
if (Bases.back()) {
SubVec = createShuffle(Bases.front(), Bases.back(), SubMask);
TransformToIdentity(SubMask);
} else {
SubVec = Bases.front();
}
if (!Vec) {
Vec = SubVec;
assert((Part == 0 || all_of(seq<unsigned>(0, Part),
[&](unsigned P) {
ArrayRef<int> SubMask =
Mask.slice(P * SliceSize,
getNumElems(Mask.size(),
SliceSize, P));
return all_of(SubMask, [](int Idx) {
return Idx == PoisonMaskElem;
});
})) &&
"Expected first part or all previous parts masked.");
copy(SubMask, std::next(VecMask.begin(), Part * SliceSize));
} else {
unsigned NewVF =
cast<FixedVectorType>(Vec->getType())->getNumElements();
if (Vec->getType() != SubVec->getType()) {
unsigned SubVecVF =
cast<FixedVectorType>(SubVec->getType())->getNumElements();
NewVF = std::max(NewVF, SubVecVF);
}
// Adjust SubMask.
for (int &Idx : SubMask)
if (Idx != PoisonMaskElem)
Idx += NewVF;
copy(SubMask, std::next(VecMask.begin(), Part * SliceSize));
Vec = createShuffle(Vec, SubVec, VecMask);
TransformToIdentity(VecMask);
}
}
copy(VecMask, Mask.begin());
return Vec;
}
/// Checks if the specified entry \p E needs to be delayed because of its
/// dependency nodes.
std::optional<Value *>
needToDelay(const TreeEntry *E,
ArrayRef<SmallVector<const TreeEntry *>> Deps) const {
// No need to delay emission if all deps are ready.
if (all_of(Deps, [](ArrayRef<const TreeEntry *> TEs) {
return all_of(
TEs, [](const TreeEntry *TE) { return TE->VectorizedValue; });
}))
return std::nullopt;
// Postpone gather emission, will be emitted after the end of the
// process to keep correct order.
auto *ResVecTy = getWidenedType(ScalarTy, E->getVectorFactor());
return Builder.CreateAlignedLoad(
ResVecTy,
PoisonValue::get(PointerType::getUnqual(ScalarTy->getContext())),
MaybeAlign());
}
/// Reset the builder to handle perfect diamond match.
void resetForSameNode() {
IsFinalized = false;
CommonMask.clear();
InVectors.clear();
}
/// Adds 2 input vectors (in form of tree entries) and the mask for their
/// shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef<int> Mask) {
Value *V1 = getVectorizedValue(E1);
Value *V2 = getVectorizedValue(E2);
add(V1, V2, Mask);
}
/// Adds single input vector (in form of tree entry) and the mask for its
/// shuffling.
void add(const TreeEntry &E1, ArrayRef<int> Mask) {
Value *V1 = getVectorizedValue(E1);
add(V1, Mask);
}
/// Adds 2 input vectors and the mask for their shuffling.
void add(Value *V1, Value *V2, ArrayRef<int> Mask) {
assert(V1 && V2 && !Mask.empty() && "Expected non-empty input vectors.");
assert(isa<FixedVectorType>(V1->getType()) &&
isa<FixedVectorType>(V2->getType()) &&
"castToScalarTyElem expects V1 and V2 to be FixedVectorType");
V1 = castToScalarTyElem(V1);
V2 = castToScalarTyElem(V2);
if (InVectors.empty()) {
InVectors.push_back(V1);
InVectors.push_back(V2);
CommonMask.assign(Mask.begin(), Mask.end());
return;
}
Value *Vec = InVectors.front();
if (InVectors.size() == 2) {
Vec = createShuffle(Vec, InVectors.back(), CommonMask);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else if (cast<FixedVectorType>(Vec->getType())->getNumElements() !=
Mask.size()) {
Vec = createShuffle(Vec, nullptr, CommonMask);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
V1 = createShuffle(V1, V2, Mask);
unsigned VF = std::max(getVF(V1), getVF(Vec));
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = Idx + VF;
InVectors.front() = Vec;
if (InVectors.size() == 2)
InVectors.back() = V1;
else
InVectors.push_back(V1);
}
/// Adds another one input vector and the mask for the shuffling.
void add(Value *V1, ArrayRef<int> Mask, bool = false,
TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
assert(isa<FixedVectorType>(V1->getType()) &&
"castToScalarTyElem expects V1 to be FixedVectorType");
V1 = castToScalarTyElem(V1);
if (InVectors.empty()) {
InVectors.push_back(V1);
CommonMask.assign(Mask.begin(), Mask.end());
return;
}
const auto *It = find(InVectors, V1);
if (It == InVectors.end()) {
if (InVectors.size() == 2 ||
InVectors.front()->getType() != V1->getType()) {
Value *V = InVectors.front();
if (InVectors.size() == 2) {
V = createShuffle(InVectors.front(), InVectors.back(), CommonMask);
transformMaskAfterShuffle(CommonMask, CommonMask);
} else if (cast<FixedVectorType>(V->getType())->getNumElements() !=
CommonMask.size()) {
V = createShuffle(InVectors.front(), nullptr, CommonMask);
transformMaskAfterShuffle(CommonMask, CommonMask);
}
unsigned VF = std::max(CommonMask.size(), Mask.size());
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (CommonMask[Idx] == PoisonMaskElem && Mask[Idx] != PoisonMaskElem)
CommonMask[Idx] = V->getType() != V1->getType()
? Idx + VF
: Mask[Idx] + getVF(V1);
if (V->getType() != V1->getType())
V1 = createShuffle(V1, nullptr, Mask);
InVectors.front() = V;
if (InVectors.size() == 2)
InVectors.back() = V1;
else
InVectors.push_back(V1);
return;
}
// Check if second vector is required if the used elements are already
// used from the first one.
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem) {
InVectors.push_back(V1);
break;
}
}
unsigned VF = 0;
for (Value *V : InVectors)
VF = std::max(VF, getVF(V));
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF);
}
/// Adds another one input vector and the mask for the shuffling.
void addOrdered(Value *V1, ArrayRef<unsigned> Order) {
SmallVector<int> NewMask;
inversePermutation(Order, NewMask);
add(V1, NewMask);
}
Value *gather(ArrayRef<Value *> VL, unsigned MaskVF = 0,
Value *Root = nullptr,
TTI::VectorInstrContext = TTI::VectorInstrContext::None) {
return R.gather(VL, Root, ScalarTy,
[&](Value *V1, Value *V2, ArrayRef<int> Mask) {
return createShuffle(V1, V2, Mask);
});
}
/// Emits the gather of zero-extended sub-fields of width \p FieldWidth of
/// the same wider integer scalar \p Src, permuted by \p Mask, as a bitcast
/// to the field vector plus a permutation and an extension to the lane type.
Value *createExtractedFieldsVector(Value *Src, unsigned FieldWidth,
ArrayRef<int> Mask, const TreeEntry &) {
auto *FieldTy = IntegerType::get(Src->getContext(), FieldWidth);
unsigned SrcWidth = Src->getType()->getIntegerBitWidth();
assert(SrcWidth % FieldWidth == 0 &&
"Expected the field width to divide the source width.");
Value *Vec = Builder.CreateBitCast(
Src, getWidenedType(FieldTy, SrcWidth / FieldWidth));
if (auto *I = dyn_cast<Instruction>(Vec)) {
R.GatherShuffleExtractSeq.insert(I);
R.CSEBlocks.insert(I->getParent());
// A vectorized source scalar is erased; extract it for the bitcast.
ArrayRef<TreeEntry *> TEs = R.getTreeEntries(Src);
const auto *It = find_if_not(TEs, [&](const TreeEntry *TE) {
return R.TransformedToGatherNodes.contains(TE) ||
R.DeletedNodes.contains(TE);
});
if (It != TEs.end())
R.ExternalUses.emplace_back(Src, I, **It, (*It)->findLaneForValue(Src));
}
Vec = createShuffle(Vec, /*V2=*/nullptr, Mask);
// The extracted fields are unsigned.
Vec = castToScalarTyElem(Vec, /*IsSigned=*/false);
return Vec;
}
Value *createFreeze(Value *V) { return Builder.CreateFreeze(V); }
/// Finalize emission of the shuffles.
/// \param Action the action (if any) to be performed before final applying of
/// the \p ExtMask mask.
Value *finalize(
ArrayRef<int> ExtMask,
ArrayRef<std::pair<const TreeEntry *, unsigned>> SubVectors,
ArrayRef<int> SubVectorsMask, unsigned VF = 0,
function_ref<void(Value *&, SmallVectorImpl<int> &,
function_ref<Value *(Value *, Value *, ArrayRef<int>)>)>
Action = {}) {
IsFinalized = true;
if (Action) {
Value *Vec = InVectors.front();
if (InVectors.size() == 2) {
Vec = createShuffle(Vec, InVectors.back(), CommonMask);
InVectors.pop_back();
} else {
Vec = createShuffle(Vec, nullptr, CommonMask);
}
transformMaskAfterShuffle(CommonMask, CommonMask);
assert(VF > 0 &&
"Expected vector length for the final value before action.");
unsigned VecVF = cast<FixedVectorType>(Vec->getType())->getNumElements();
if (VecVF < VF) {
SmallVector<int> ResizeMask(VF, PoisonMaskElem);
std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0);
Vec = createShuffle(Vec, nullptr, ResizeMask);
}
Action(Vec, CommonMask, [this](Value *V1, Value *V2, ArrayRef<int> Mask) {
return createShuffle(V1, V2, Mask);
});
InVectors.front() = Vec;
}
if (!SubVectors.empty()) {
Value *Vec = InVectors.front();
if (InVectors.size() == 2) {
Vec = createShuffle(Vec, InVectors.back(), CommonMask);
InVectors.pop_back();
} else {
Vec = createShuffle(Vec, nullptr, CommonMask);
}
transformMaskAfterShuffle(CommonMask, CommonMask);
auto CreateSubVectors = [&](Value *Vec,
SmallVectorImpl<int> &CommonMask) {
for (auto [E, Idx] : SubVectors) {
Value *V = getVectorizedValue(*E);
unsigned InsertionIndex = Idx * getNumElements(ScalarTy);
// Use scalar version of the SCalarType to correctly handle shuffles
// for revectorization. The revectorization mode operates by the
// vectors, but here we need to operate on the scalars, because the
// masks were already transformed for the vector elements and we don't
// need doing this transformation again.
Type *OrigScalarTy = ScalarTy;
ScalarTy = ScalarTy->getScalarType();
Vec = createInsertVector(
Builder, Vec, V, InsertionIndex,
std::bind(&ShuffleInstructionBuilder::createShuffle, this, _1, _2,
_3));
ScalarTy = OrigScalarTy;
if (!CommonMask.empty()) {
std::iota(std::next(CommonMask.begin(), Idx),
std::next(CommonMask.begin(), Idx + E->getVectorFactor()),
Idx);
}
}
return Vec;
};
if (SubVectorsMask.empty()) {
Vec = CreateSubVectors(Vec, CommonMask);
} else {
SmallVector<int> SVMask(CommonMask.size(), PoisonMaskElem);
copy(SubVectorsMask, SVMask.begin());
for (auto [I1, I2] : zip(SVMask, CommonMask)) {
if (I2 != PoisonMaskElem) {
assert(I1 == PoisonMaskElem && "Expected unused subvectors mask");
I1 = I2 + CommonMask.size();
}
}
Value *InsertVec =
CreateSubVectors(PoisonValue::get(Vec->getType()), CommonMask);
Vec = createShuffle(InsertVec, Vec, SVMask);
transformMaskAfterShuffle(CommonMask, SVMask);
}
InVectors.front() = Vec;
}
if (!ExtMask.empty()) {
if (CommonMask.empty()) {
CommonMask.assign(ExtMask.begin(), ExtMask.end());
} else {
SmallVector<int> NewMask(ExtMask.size(), PoisonMaskElem);
for (int I = 0, Sz = ExtMask.size(); I < Sz; ++I) {
if (ExtMask[I] == PoisonMaskElem)
continue;
NewMask[I] = CommonMask[ExtMask[I]];
}
CommonMask.swap(NewMask);
}
}
if (CommonMask.empty()) {
assert(InVectors.size() == 1 && "Expected only one vector with no mask");
return InVectors.front();
}
if (InVectors.size() == 2)
return createShuffle(InVectors.front(), InVectors.back(), CommonMask);
return createShuffle(InVectors.front(), nullptr, CommonMask);
}
~ShuffleInstructionBuilder() {
assert((IsFinalized || CommonMask.empty()) &&
"Shuffle construction must be finalized.");
}
};
Value *BoUpSLP::vectorizeOperand(TreeEntry *E, unsigned NodeIdx) {
return vectorizeTree(getOperandEntry(E, NodeIdx));
}
template <typename ResTy, typename BVTy>
std::optional<ResTy> BoUpSLP::processExtractedFieldsGather(
BVTy &ShuffleBuilder, const TreeEntry *E, Type *ScalarTy,
ArrayRef<Value *> VL,
ArrayRef<std::pair<const TreeEntry *, unsigned>> SubVectors) {
if (!SubVectors.empty() || !E->ReuseShuffleIndices.empty() ||
!ScalarTy->isIntegerTy())
return std::nullopt;
std::optional<std::tuple<Value *, unsigned, SmallVector<int>>>
ExtractedFields = matchGatheredExtractedFields(VL, *DL);
if (!ExtractedFields ||
ScalarTy->getIntegerBitWidth() < std::get<1>(*ExtractedFields))
return std::nullopt;
Value *Src = std::get<0>(*ExtractedFields);
unsigned FieldWidth = std::get<1>(*ExtractedFields);
SmallVector<int> &Mask = std::get<2>(*ExtractedFields);
return ShuffleBuilder.createExtractedFieldsVector(Src, FieldWidth, Mask, *E);
}
template <typename BVTy, typename ResTy, typename... Args>
ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
Args &...Params) {
assert((E->isGather() || TransformedToGatherNodes.contains(E)) &&
"Expected gather node.");
unsigned VF = E->getVectorFactor();
bool NeedFreeze = false;
SmallVector<Value *> GatheredScalars(E->Scalars.begin(), E->Scalars.end());
// Do not process split vectorize node, marked to be gathers/buildvectors.
SmallVector<std::pair<const TreeEntry *, unsigned>> SubVectors(
E->CombinedEntriesWithIndices.size());
if (E->State == TreeEntry::SplitVectorize &&
TransformedToGatherNodes.contains(E)) {
SubVectors.clear();
} else {
// Clear values, to be replaced by insertvector instructions.
for (auto [EIdx, Idx] : E->CombinedEntriesWithIndices)
for_each(MutableArrayRef(GatheredScalars)
.slice(Idx, VectorizableTree[EIdx]->getVectorFactor()),
[&](Value *&V) { V = PoisonValue::get(V->getType()); });
transform(
E->CombinedEntriesWithIndices, SubVectors.begin(), [&](const auto &P) {
return std::make_pair(VectorizableTree[P.first].get(), P.second);
});
}
// Build a mask out of the reorder indices and reorder scalars per this
// mask.
SmallVector<int> ReorderMask(E->ReorderIndices.begin(),
E->ReorderIndices.end());
if (!ReorderMask.empty())
reorderScalars(GatheredScalars, ReorderMask);
SmallVector<int> SubVectorsMask;
inversePermutation(E->ReorderIndices, SubVectorsMask);
// Transform non-clustered elements in the mask to poison (-1).
// "Clustered" operations will be reordered using this mask later.
if (!SubVectors.empty() && !SubVectorsMask.empty()) {
for (unsigned I : seq<unsigned>(GatheredScalars.size()))
if (E->Scalars[I] == GatheredScalars[ReorderMask[I]])
SubVectorsMask[ReorderMask[I]] = PoisonMaskElem;
} else {
SubVectorsMask.clear();
}
SmallVector<Value *> StoredGS(GatheredScalars);
auto FindReusedSplat = [&](MutableArrayRef<int> Mask, unsigned InputVF,
unsigned I, unsigned SliceSize,
bool IsNotPoisonous) {
if (!isSplat(E->Scalars) || none_of(E->Scalars, [](Value *V) {
return isa<UndefValue>(V) && !isa<PoisonValue>(V);
}))
return false;
TreeEntry *UserTE = E->UserTreeIndex.UserTE;
unsigned EdgeIdx = E->UserTreeIndex.EdgeIdx;
if (UserTE->getNumOperands() != 2)
return false;
if (!IsNotPoisonous) {
auto *It = find_if(ArrayRef(VectorizableTree).drop_front(UserTE->Idx + 1),
[=](const std::unique_ptr<TreeEntry> &TE) {
return TE->UserTreeIndex.UserTE == UserTE &&
TE->UserTreeIndex.EdgeIdx != EdgeIdx;
});
if (It == VectorizableTree.end())
return false;
SmallVector<Value *> GS((*It)->Scalars.begin(), (*It)->Scalars.end());
if (!(*It)->ReorderIndices.empty()) {
inversePermutation((*It)->ReorderIndices, ReorderMask);
reorderScalars(GS, ReorderMask);
}
if (!all_of(zip(GatheredScalars, GS), [&](const auto &P) {
Value *V0 = std::get<0>(P);
Value *V1 = std::get<1>(P);
return !isa<UndefValue>(V0) || isa<PoisonValue>(V0) ||
(isa<UndefValue>(V0) && !isa<PoisonValue>(V0) &&
is_contained(E->Scalars, V1));
}))
return false;
}
int Idx;
if ((Mask.size() < InputVF &&
ShuffleVectorInst::isExtractSubvectorMask(Mask, InputVF, Idx) &&
Idx == 0) ||
(Mask.size() == InputVF &&
ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))) {
std::iota(
std::next(Mask.begin(), I * SliceSize),
std::next(Mask.begin(),
I * SliceSize + getNumElems(Mask.size(), SliceSize, I)),
0);
} else {
unsigned IVal =
*find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
std::fill(
std::next(Mask.begin(), I * SliceSize),
std::next(Mask.begin(),
I * SliceSize + getNumElems(Mask.size(), SliceSize, I)),
IVal);
}
return true;
};
BVTy ShuffleBuilder(ScalarTy, Params...);
ResTy Res = ResTy();
SmallVector<int> Mask;
SmallVector<int> ExtractMask(GatheredScalars.size(), PoisonMaskElem);
SmallVector<std::optional<TTI::ShuffleKind>> ExtractShuffles;
Value *ExtractVecBase = nullptr;
bool UseVecBaseAsInput = false;
SmallVector<std::optional<TargetTransformInfo::ShuffleKind>> GatherShuffles;
SmallVector<SmallVector<const TreeEntry *>> Entries;
Type *OrigScalarTy = GatheredScalars.front()->getType();
auto *VecTy = getWidenedType(ScalarTy, GatheredScalars.size());
unsigned NumParts = getNumberOfParts(VecTy, ScalarTy, GatheredScalars.size());
if (std::optional<ResTy> Res = processExtractedFieldsGather<ResTy>(
ShuffleBuilder, E, ScalarTy, GatheredScalars, SubVectors))
return *Res;
if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
// Check for gathered extracts.
bool Resized = false;
ExtractShuffles =
tryToGatherExtractElements(GatheredScalars, ExtractMask, NumParts);
if (!ExtractShuffles.empty()) {
SmallVector<const TreeEntry *> ExtractEntries;
for (const auto &P :
make_filter_range(enumerate(ExtractMask), [](const auto &P) {
return P.value() != PoisonMaskElem;
})) {
if (ArrayRef<TreeEntry *> TEs =
getTreeEntries(cast<ExtractElementInst>(StoredGS[P.index()])
->getVectorOperand());
!TEs.empty())
ExtractEntries.append(TEs.begin(), TEs.end());
}
if (std::optional<ResTy> Delayed =
ShuffleBuilder.needToDelay(E, ExtractEntries)) {
// Delay emission of gathers which are not ready yet.
PostponedGathers.insert(E);
// Postpone gather emission, will be emitted after the end of the
// process to keep correct order.
return *Delayed;
}
if (Value *VecBase = ShuffleBuilder.adjustExtracts(
E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput)) {
ExtractVecBase = VecBase;
if (auto *VecBaseTy = dyn_cast<FixedVectorType>(VecBase->getType()))
if (VF == VecBaseTy->getNumElements() &&
GatheredScalars.size() != VF) {
Resized = true;
GatheredScalars.append(VF - GatheredScalars.size(),
PoisonValue::get(OrigScalarTy));
NumParts = getNumberOfParts(getWidenedType(OrigScalarTy, VF),
OrigScalarTy, VF);
}
}
}
// Gather extracts after we check for full matched gathers only.
if (!ExtractShuffles.empty() || !E->hasState() ||
E->getOpcode() != Instruction::Load ||
(((E->hasState() && E->getOpcode() == Instruction::Load) ||
any_of(E->Scalars, IsaPred<LoadInst>)) &&
any_of(make_isa_range<LoadInst>(E->Scalars),
[this](LoadInst *LI) { return isVectorized(LI); })) ||
(E->hasState() && E->isAltShuffle()) ||
all_of(E->Scalars, [this](Value *V) { return isVectorized(V); }) ||
isSplat(E->Scalars) ||
(E->Scalars != GatheredScalars && GatheredScalars.size() <= 2)) {
GatherShuffles = isGatherShuffledEntry(E, GatheredScalars, Mask, Entries,
NumParts, ScalarTy);
}
// The splat gather subtrees are speculative: the keep/drop check may
// delete them and does not track reuse of their internal gathers by other
// gathers.
if (hasSplatSubtreeGather(Entries)) {
GatherShuffles.clear();
Entries.clear();
std::fill(Mask.begin(), Mask.end(), PoisonMaskElem);
}
if (!GatherShuffles.empty()) {
if (std::optional<ResTy> Delayed =
ShuffleBuilder.needToDelay(E, Entries)) {
// Delay emission of gathers which are not ready yet.
PostponedGathers.insert(E);
// Postpone gather emission, will be emitted after the end of the
// process to keep correct order.
return *Delayed;
}
// Match against the expanded vector of the gather node, so that poison
// lanes of the matched entry cannot wildcard-match its real scalars.
auto IsFullVectorMatch = [&](const TreeEntry *FrontTE) {
SmallVector<int> CommonMask = E->getCommonMask();
SmallVector<Value *> Expanded(E->getVectorFactor());
for (unsigned I : seq<unsigned>(E->getVectorFactor()))
Expanded[I] = CommonMask.empty() ? E->Scalars[I]
: CommonMask[I] == PoisonMaskElem
? PoisonValue::get(OrigScalarTy)
: E->Scalars[CommonMask[I]];
return FrontTE->isSame(Expanded);
};
if (GatherShuffles.size() == 1 &&
*GatherShuffles.front() == TTI::SK_PermuteSingleSrc &&
(Entries.front().front()->isSame(E->Scalars) ||
IsFullVectorMatch(Entries.front().front()))) {
// Perfect match in the graph, will reuse the previously vectorized
// node. Cost is 0.
LLVM_DEBUG(dbgs() << "SLP: perfect diamond match for gather bundle "
<< shortBundleName(E->Scalars, E->Idx) << ".\n");
// Restore the mask for previous partially matched values.
Mask.resize(E->Scalars.size());
const TreeEntry *FrontTE = Entries.front().front();
if (FrontTE->ReorderIndices.empty() && E->ReorderIndices.empty() &&
((FrontTE->ReuseShuffleIndices.empty() &&
E->Scalars.size() == FrontTE->Scalars.size()) ||
(E->Scalars.size() == FrontTE->ReuseShuffleIndices.size()))) {
std::iota(Mask.begin(), Mask.end(), 0);
} else {
for (auto [I, V] : enumerate(E->Scalars)) {
if (isa<PoisonValue>(V)) {
Mask[I] = PoisonMaskElem;
continue;
}
Mask[I] = FrontTE->findLaneForValue(V);
}
}
// Reset the builder(s) to correctly handle perfect diamond matched
// nodes.
ShuffleBuilder.resetForSameNode();
// Full matched entry found, no need to insert subvectors.
if ((IsFullVectorMatch(FrontTE) &&
FrontTE->ReuseShuffleIndices.empty() &&
FrontTE->ReorderIndices.empty() &&
E->getVectorFactor() == FrontTE->getVectorFactor()) ||
(equal(E->Scalars, FrontTE->Scalars) &&
equal(E->ReorderIndices, FrontTE->ReorderIndices) &&
equal(E->ReuseShuffleIndices, FrontTE->ReuseShuffleIndices))) {
Mask.resize(FrontTE->getVectorFactor());
std::iota(Mask.begin(), Mask.end(), 0);
ShuffleBuilder.add(*FrontTE, Mask);
Res = ShuffleBuilder.finalize({}, {}, {});
} else {
ShuffleBuilder.add(*FrontTE, Mask);
Res = ShuffleBuilder.finalize(E->getCommonMask(), {}, {});
}
return Res;
}
if (!Resized) {
if (GatheredScalars.size() != VF &&
any_of(Entries, [&](ArrayRef<const TreeEntry *> TEs) {
return any_of(TEs, [&](const TreeEntry *TE) {
return TE->getVectorFactor() == VF;
});
}))
GatheredScalars.append(VF - GatheredScalars.size(),
PoisonValue::get(OrigScalarTy));
}
// Remove shuffled elements from list of gathers.
for (int I = 0, Sz = Mask.size(); I < Sz; ++I) {
if (Mask[I] != PoisonMaskElem)
GatheredScalars[I] = PoisonValue::get(OrigScalarTy);
}
}
}
auto TryPackScalars = [&](SmallVectorImpl<Value *> &Scalars,
SmallVectorImpl<int> &ReuseMask,
bool IsRootPoison) {
// For splats with can emit broadcasts instead of gathers, so try to find
// such sequences.
bool IsSplat = IsRootPoison && isSplat(Scalars) &&
(Scalars.size() > 2 || Scalars.front() == Scalars.back());
Scalars.append(VF - Scalars.size(), PoisonValue::get(OrigScalarTy));
SmallVector<int> UndefPos;
DenseMap<Value *, unsigned> UniquePositions;
// Gather unique non-const values and all constant values.
// For repeated values, just shuffle them.
int NumNonConsts = 0;
int SinglePos = 0;
for (auto [I, V] : enumerate(Scalars)) {
if (isa<UndefValue>(V)) {
if (!isa<PoisonValue>(V)) {
ReuseMask[I] = I;
UndefPos.push_back(I);
}
continue;
}
if (isConstant(V)) {
ReuseMask[I] = I;
continue;
}
++NumNonConsts;
SinglePos = I;
Value *OrigV = V;
Scalars[I] = PoisonValue::get(OrigScalarTy);
if (IsSplat) {
Scalars.front() = OrigV;
ReuseMask[I] = 0;
} else {
const auto Res = UniquePositions.try_emplace(OrigV, I);
Scalars[Res.first->second] = OrigV;
ReuseMask[I] = Res.first->second;
}
}
if (NumNonConsts == 1) {
// Restore single insert element.
if (IsSplat) {
ReuseMask.assign(VF, PoisonMaskElem);
std::swap(Scalars.front(), Scalars[SinglePos]);
if (!UndefPos.empty() && UndefPos.front() == 0)
Scalars.front() = UndefValue::get(OrigScalarTy);
}
ReuseMask[SinglePos] = SinglePos;
} else if (!UndefPos.empty() && IsSplat) {
// For undef values, try to replace them with the simple broadcast.
// We can do it if the broadcasted value is guaranteed to be
// non-poisonous, or by freezing the incoming scalar value first.
auto *It = find_if(Scalars, [this, E](Value *V) {
return !isa<UndefValue>(V) &&
(isVectorized(V) || isGuaranteedNotToBePoison(V, AC) ||
(E->UserTreeIndex && any_of(V->uses(), [E](const Use &U) {
// Check if the value already used in the same operation in
// one of the nodes already.
return E->UserTreeIndex.EdgeIdx != U.getOperandNo() &&
is_contained(E->UserTreeIndex.UserTE->Scalars,
U.getUser());
})));
});
if (It != Scalars.end()) {
// Replace undefs by the non-poisoned scalars and emit broadcast.
int Pos = std::distance(Scalars.begin(), It);
for (int I : UndefPos) {
// Set the undef position to the non-poisoned scalar.
ReuseMask[I] = Pos;
// Replace the undef by the poison, in the mask it is replaced by
// non-poisoned scalar already.
if (I != Pos)
Scalars[I] = PoisonValue::get(OrigScalarTy);
}
} else {
// Replace undefs by the poisons, emit broadcast and then emit
// freeze.
for (int I : UndefPos) {
ReuseMask[I] = PoisonMaskElem;
if (isa<UndefValue>(Scalars[I]))
Scalars[I] = PoisonValue::get(OrigScalarTy);
}
NeedFreeze = true;
}
}
};
if (!ExtractShuffles.empty() || !GatherShuffles.empty()) {
bool IsNonPoisoned = true;
bool IsUsedInExpr = true;
Value *Vec1 = nullptr;
if (!ExtractShuffles.empty()) {
// Gather of extractelements can be represented as just a shuffle of
// a single/two vectors the scalars are extracted from.
// Find input vectors.
Value *Vec2 = nullptr;
for (unsigned I = 0, Sz = ExtractMask.size(); I < Sz; ++I) {
if (!Mask.empty() && Mask[I] != PoisonMaskElem)
ExtractMask[I] = PoisonMaskElem;
}
if (UseVecBaseAsInput) {
Vec1 = ExtractVecBase;
} else {
for (unsigned I = 0, Sz = ExtractMask.size(); I < Sz; ++I) {
if (ExtractMask[I] == PoisonMaskElem)
continue;
if (isa<UndefValue>(StoredGS[I]))
continue;
auto *EI = cast<ExtractElementInst>(StoredGS[I]);
Value *VecOp = EI->getVectorOperand();
if (ArrayRef<TreeEntry *> TEs = getTreeEntries(VecOp);
!TEs.empty() && TEs.front()->VectorizedValue)
VecOp = TEs.front()->VectorizedValue;
if (!Vec1) {
Vec1 = VecOp;
} else if (Vec1 != VecOp) {
assert((!Vec2 || Vec2 == VecOp) &&
"Expected only 1 or 2 vectors shuffle.");
Vec2 = VecOp;
}
}
}
if (Vec2) {
IsUsedInExpr = false;
IsNonPoisoned &= isGuaranteedNotToBePoison(Vec1, AC) &&
isGuaranteedNotToBePoison(Vec2, AC);
ShuffleBuilder.add(Vec1, Vec2, ExtractMask);
} else if (Vec1) {
bool IsNotPoisonedVec = isGuaranteedNotToBePoison(Vec1, AC);
IsUsedInExpr &= FindReusedSplat(
ExtractMask,
cast<FixedVectorType>(Vec1->getType())->getNumElements(), 0,
ExtractMask.size(), IsNotPoisonedVec);
ShuffleBuilder.add(Vec1, ExtractMask, /*ForExtracts=*/true);
IsNonPoisoned &= IsNotPoisonedVec;
} else {
IsUsedInExpr = false;
ShuffleBuilder.add(PoisonValue::get(VecTy), ExtractMask,
/*ForExtracts=*/true);
}
}
if (!GatherShuffles.empty()) {
unsigned SliceSize = getPartNumElems(VF, NumParts);
if (Mask.size() == E->Scalars.size())
SliceSize = getPartNumElems(
E->Scalars.size(),
getNumberOfParts(VecTy, ScalarTy, E->Scalars.size()));
SmallVector<int> VecMask(Mask.size(), PoisonMaskElem);
for (const auto [I, TEs] : enumerate(Entries)) {
if (TEs.empty()) {
assert(!GatherShuffles[I] &&
"No shuffles with empty entries list expected.");
continue;
}
assert((TEs.size() == 1 || TEs.size() == 2) &&
"Expected shuffle of 1 or 2 entries.");
unsigned Limit = getNumElems(Mask.size(), SliceSize, I);
auto SubMask = ArrayRef(Mask).slice(I * SliceSize, Limit);
VecMask.assign(VecMask.size(), PoisonMaskElem);
copy(SubMask, std::next(VecMask.begin(), I * SliceSize));
if (TEs.size() == 1) {
bool IsNotPoisonedVec =
TEs.front()->VectorizedValue
? isGuaranteedNotToBePoison(TEs.front()->VectorizedValue, AC)
: true;
IsUsedInExpr &=
FindReusedSplat(VecMask, TEs.front()->getVectorFactor(), I,
SliceSize, IsNotPoisonedVec);
ShuffleBuilder.add(*TEs.front(), VecMask);
IsNonPoisoned &= IsNotPoisonedVec;
} else {
IsUsedInExpr = false;
ShuffleBuilder.add(*TEs.front(), *TEs.back(), VecMask);
if (TEs.front()->VectorizedValue && TEs.back()->VectorizedValue)
IsNonPoisoned &=
isGuaranteedNotToBePoison(TEs.front()->VectorizedValue, AC) &&
isGuaranteedNotToBePoison(TEs.back()->VectorizedValue, AC);
}
}
}
// Try to figure out best way to combine values: build a shuffle and insert
// elements or just build several shuffles.
// Insert non-constant scalars.
SmallVector<Value *> NonConstants(GatheredScalars);
int EMSz = ExtractMask.size();
int MSz = Mask.size();
// Try to build constant vector and shuffle with it only if currently we
// have a single permutation and more than 1 scalar constants.
bool IsSingleShuffle = ExtractShuffles.empty() || GatherShuffles.empty();
bool IsIdentityShuffle =
((UseVecBaseAsInput ||
all_of(ExtractShuffles,
[](const std::optional<TTI::ShuffleKind> &SK) {
return SK.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc;
})) &&
none_of(ExtractMask, [&](int I) { return I >= EMSz; }) &&
ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) ||
(!GatherShuffles.empty() &&
all_of(GatherShuffles,
[](const std::optional<TTI::ShuffleKind> &SK) {
return SK.value_or(TTI::SK_PermuteTwoSrc) ==
TTI::SK_PermuteSingleSrc;
}) &&
none_of(Mask, [&](int I) { return I >= MSz; }) &&
ShuffleVectorInst::isIdentityMask(Mask, MSz));
bool EnoughConstsForShuffle =
IsSingleShuffle &&
(none_of(GatheredScalars,
[](Value *V) {
return isa<UndefValue>(V) && !isa<PoisonValue>(V);
}) ||
any_of(GatheredScalars,
[](Value *V) {
return isa<Constant>(V) && !isa<UndefValue>(V);
})) &&
(!IsIdentityShuffle ||
(GatheredScalars.size() == 2 &&
any_of(GatheredScalars,
[](Value *V) { return !isa<UndefValue>(V); })) ||
count_if(GatheredScalars, [](Value *V) {
return isa<Constant>(V) && !isa<PoisonValue>(V);
}) > 1);
// NonConstants array contains just non-constant values, GatheredScalars
// contains only constant to build final vector and then shuffle.
for (int I = 0, Sz = GatheredScalars.size(); I < Sz; ++I) {
if (EnoughConstsForShuffle && isa<Constant>(GatheredScalars[I]))
NonConstants[I] = PoisonValue::get(OrigScalarTy);
else
GatheredScalars[I] = PoisonValue::get(OrigScalarTy);
}
// Generate constants for final shuffle and build a mask for them.
if (!all_of(GatheredScalars, IsaPred<PoisonValue>)) {
SmallVector<int> BVMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, BVMask, /*IsRootPoison=*/true);
Value *BV = ShuffleBuilder.gather(GatheredScalars, BVMask.size());
ShuffleBuilder.add(BV, BVMask);
}
if (all_of(NonConstants, [=](Value *V) {
return isa<PoisonValue>(V) ||
(IsSingleShuffle && ((IsIdentityShuffle &&
IsNonPoisoned) || IsUsedInExpr) && isa<UndefValue>(V));
}))
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
SubVectorsMask);
else
Res = ShuffleBuilder.finalize(
E->ReuseShuffleIndices, SubVectors, SubVectorsMask, E->Scalars.size(),
[&](Value *&Vec, SmallVectorImpl<int> &Mask, auto CreateShuffle) {
bool IsSplat = isSplat(NonConstants);
SmallVector<int> BVMask(Mask.size(), PoisonMaskElem);
TryPackScalars(NonConstants, BVMask, /*IsRootPoison=*/false);
auto CheckIfSplatIsProfitable = [&]() {
// Estimate the cost of splatting + shuffle and compare with
// insert + shuffle.
Value *V = *find_if_not(NonConstants, IsaPred<UndefValue>);
if (isa<ExtractElementInst>(V) || isVectorized(V))
return false;
InstructionCost SplatCost = TTI->getVectorInstrCost(
Instruction::InsertElement, VecTy, CostKind, /*Index=*/0,
PoisonValue::get(VecTy), V);
SmallVector<int> NewMask(Mask.begin(), Mask.end());
for (auto [Idx, I] : enumerate(BVMask))
if (I != PoisonMaskElem)
NewMask[Idx] = Mask.size();
SplatCost +=
getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc,
cast<VectorType>(VecTy), CostKind, NewMask);
InstructionCost BVCost = TTI->getVectorInstrCost(
Instruction::InsertElement, VecTy, CostKind,
*find_if(Mask, not_equal_to(PoisonMaskElem)), Vec, V);
// Shuffle required?
if (count(BVMask, PoisonMaskElem) <
static_cast<int>(BVMask.size() - 1)) {
SmallVector<int> NewMask(Mask.begin(), Mask.end());
for (auto [Idx, I] : enumerate(BVMask))
if (I != PoisonMaskElem)
NewMask[Idx] = I;
BVCost +=
getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(VecTy), CostKind, NewMask);
}
return SplatCost <= BVCost;
};
if (!IsSplat || Mask.size() <= 2 || !CheckIfSplatIsProfitable()) {
for (auto [Idx, I] : enumerate(BVMask))
if (I != PoisonMaskElem)
Mask[Idx] = I;
Vec = ShuffleBuilder.gather(NonConstants, Mask.size(), Vec);
} else {
Value *V = *find_if_not(NonConstants, IsaPred<UndefValue>);
SmallVector<Value *> Values(NonConstants.size(),
PoisonValue::get(ScalarTy));
Values[0] = V;
Value *BV = ShuffleBuilder.gather(Values, BVMask.size());
SmallVector<int> SplatMask(BVMask.size(), PoisonMaskElem);
transform(BVMask, SplatMask.begin(), [](int I) {
return I == PoisonMaskElem ? PoisonMaskElem : 0;
});
if (!ShuffleVectorInst::isIdentityMask(SplatMask, VF))
BV = CreateShuffle(BV, nullptr, SplatMask);
for (auto [Idx, I] : enumerate(BVMask))
if (I != PoisonMaskElem)
Mask[Idx] = BVMask.size() + Idx;
Vec = CreateShuffle(Vec, BV, Mask);
for (auto [Idx, I] : enumerate(Mask))
if (I != PoisonMaskElem)
Mask[Idx] = Idx;
}
});
} else if (!allConstant(GatheredScalars)) {
// Gather unique scalars and all constants.
SmallVector<int> ReuseMask(GatheredScalars.size(), PoisonMaskElem);
TryPackScalars(GatheredScalars, ReuseMask, /*IsRootPoison=*/true);
TTI::VectorInstrContext ContextHint = TTI::VectorInstrContext::None;
if constexpr (std::is_same_v<ResTy, InstructionCost>) {
auto GatherUserOps =
[&](SmallVectorImpl<TTI::BuildVectorUseOp> &UserOps) {
UserOps.clear();
return !NeedFreeze && collectGatherUserOps(*E, UserOps);
};
ContextHint =
TTI->getBuildVectorContextHint(ReuseMask, E->Scalars, GatherUserOps);
}
Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.size(),
/*Root*/ nullptr, ContextHint);
ShuffleBuilder.add(BV, ReuseMask, /*ForExtract*/ false, ContextHint);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
SubVectorsMask, /*VF*/ 0, /*Action*/ {});
} else {
// Gather all constants.
SmallVector<int> Mask(GatheredScalars.size(), PoisonMaskElem);
for (auto [I, V] : enumerate(GatheredScalars)) {
if (!isa<PoisonValue>(V))
Mask[I] = I;
}
Value *BV = ShuffleBuilder.gather(GatheredScalars);
ShuffleBuilder.add(BV, Mask);
Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors,
SubVectorsMask);
}
if (NeedFreeze)
Res = ShuffleBuilder.createFreeze(Res);
return Res;
}
Value *BoUpSLP::createBuildVector(const TreeEntry *E, Type *ScalarTy) {
// Do not do this for split vectorize node, marked to be gathers/buildvectors.
if (E->State != TreeEntry::SplitVectorize ||
!TransformedToGatherNodes.contains(E)) {
for (auto [EIdx, _] : E->CombinedEntriesWithIndices)
(void)vectorizeTree(VectorizableTree[EIdx].get());
}
return processBuildVector<ShuffleInstructionBuilder, Value *>(E, ScalarTy,
Builder, *this);
}
/// \returns \p I after propagating metadata from \p VL only for instructions in
/// \p VL.
static Instruction *propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
SmallVector<Value *> Insts(make_isa_range<Instruction>(VL));
return llvm::propagateMetadata(Inst, Insts);
}
Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Value *V = E->Scalars.front();
Type *ScalarTy = getValueType(V, Opts.slp_revec);
auto It = MinBWs.find(E);
if (It != MinBWs.end()) {
auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy);
ScalarTy = IntegerType::get(F->getContext(), It->second.first);
if (VecTy)
ScalarTy = getWidenedType(ScalarTy, VecTy->getNumElements());
}
if (E->VectorizedValue)
return E->VectorizedValue;
auto *VecTy = getWidenedType(ScalarTy, E->Scalars.size());
if (E->isGather() || TransformedToGatherNodes.contains(E)) {
// Set insert point for non-reduction initial nodes.
if (E->hasState() && E->Idx == 0 && !UserIgnoreList)
setInsertPointAfterBundle(E);
Value *Vec = createBuildVector(E, ScalarTy);
E->VectorizedValue = Vec;
return Vec;
}
if (E->State == TreeEntry::SplitVectorize) {
assert(E->CombinedEntriesWithIndices.size() == 2 &&
"Expected exactly 2 combined entries.");
setInsertPointAfterBundle(E);
TreeEntry &OpTE1 =
*VectorizableTree[E->CombinedEntriesWithIndices.front().first];
assert(OpTE1.isSame(
ArrayRef(E->Scalars).take_front(OpTE1.getVectorFactor())) &&
"Expected same first part of scalars.");
Value *Op1 = vectorizeTree(&OpTE1);
TreeEntry &OpTE2 =
*VectorizableTree[E->CombinedEntriesWithIndices.back().first];
assert(
OpTE2.isSame(ArrayRef(E->Scalars).take_back(OpTE2.getVectorFactor())) &&
"Expected same second part of scalars.");
Value *Op2 = vectorizeTree(&OpTE2);
auto GetOperandSignedness = [&](const TreeEntry *OpE) {
bool IsSigned = false;
auto It = MinBWs.find(OpE);
if (It != MinBWs.end())
IsSigned = It->second.second;
else
IsSigned = any_of(OpE->Scalars, [&](Value *R) {
if (isa<PoisonValue>(V))
return false;
return !isKnownNonNegative(R, SimplifyQuery(*DL));
});
return IsSigned;
};
if (cast<VectorType>(Op1->getType())->getElementType() !=
ScalarTy->getScalarType()) {
assert(ScalarTy->isIntegerTy() && "Expected item in MinBWs.");
Op1 = Builder.CreateIntCast(
Op1,
getWidenedType(
ScalarTy,
cast<FixedVectorType>(Op1->getType())->getNumElements()),
GetOperandSignedness(&OpTE1));
}
if (cast<VectorType>(Op2->getType())->getElementType() !=
ScalarTy->getScalarType()) {
assert(ScalarTy->isIntegerTy() && "Expected item in MinBWs.");
Op2 = Builder.CreateIntCast(
Op2,
getWidenedType(
ScalarTy,
cast<FixedVectorType>(Op2->getType())->getNumElements()),
GetOperandSignedness(&OpTE2));
}
if (E->ReorderIndices.empty()) {
SmallVector<int> Mask(E->getVectorFactor(), PoisonMaskElem);
std::iota(
Mask.begin(),
std::next(Mask.begin(), E->CombinedEntriesWithIndices.back().second),
0);
unsigned ScalarTyNumElements = getNumElements(ScalarTy);
if (ScalarTyNumElements != 1) {
assert(Opts.slp_revec && "Only supported by REVEC.");
transformScalarShuffleIndiciesToVector(ScalarTyNumElements, Mask);
}
Value *Vec = Builder.CreateShuffleVector(Op1, Mask);
Vec = createInsertVector(Builder, Vec, Op2,
E->CombinedEntriesWithIndices.back().second *
ScalarTyNumElements);
E->VectorizedValue = Vec;
return Vec;
}
unsigned CommonVF =
std::max(OpTE1.getVectorFactor(), OpTE2.getVectorFactor());
const unsigned Scale = getNumElements(ScalarTy);
CommonVF *= Scale;
if (getNumElements(Op1->getType()) != CommonVF) {
SmallVector<int> Mask(CommonVF, PoisonMaskElem);
copy(createReplicatedMask(Scale, OpTE1.getVectorFactor() * Scale),
Mask.begin());
Op1 = Builder.CreateShuffleVector(Op1, Mask);
}
if (getNumElements(Op2->getType()) != CommonVF) {
SmallVector<int> Mask(CommonVF, PoisonMaskElem);
copy(createReplicatedMask(Scale, OpTE2.getVectorFactor() * Scale),
Mask.begin());
Op2 = Builder.CreateShuffleVector(Op2, Mask);
}
Value *Vec = Builder.CreateShuffleVector(Op1, Op2, E->getSplitMask());
E->VectorizedValue = Vec;
return Vec;
}
bool IsReverseOrder = E->isReverse() && !isWidenedStridedLoad(E);
auto FinalShuffle = [&](Value *V, const TreeEntry *E) {
if (isa<StructType>(ScalarTy)) {
// TODO: Reordering of struct types is not supported.
assert(E->ReorderIndices.empty() &&
"Expected no reordering for struct types.");
assert(E->ReuseShuffleIndices.empty() &&
"Expected no reuse shuffle indices for struct types.");
return V;
}
ShuffleInstructionBuilder ShuffleBuilder(ScalarTy, Builder, *this);
if ((E->State == TreeEntry::StridedVectorize && IsReverseOrder) ||
E->State == TreeEntry::CompressVectorize) {
ShuffleBuilder.addOrdered(V, {});
} else if (E->getOpcode() == Instruction::Store &&
(E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize)) {
ArrayRef<int> Mask =
ArrayRef(reinterpret_cast<const int *>(E->ReorderIndices.begin()),
E->ReorderIndices.size());
ShuffleBuilder.add(V, Mask);
} else {
ShuffleBuilder.addOrdered(V, E->ReorderIndices);
}
SmallVector<std::pair<const TreeEntry *, unsigned>> SubVectors(
E->CombinedEntriesWithIndices.size());
transform(
E->CombinedEntriesWithIndices, SubVectors.begin(), [&](const auto &P) {
return std::make_pair(VectorizableTree[P.first].get(), P.second);
});
assert(
(E->CombinedEntriesWithIndices.empty() || E->ReorderIndices.empty()) &&
"Expected either combined subnodes or reordering");
return ShuffleBuilder.finalize(E->ReuseShuffleIndices, SubVectors, {});
};
assert(!E->isGather() && "Unhandled state");
unsigned ShuffleOrOp =
E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
if (!E->isAltShuffle()) {
switch (E->CombinedOp) {
case TreeEntry::ReducedBitcast:
case TreeEntry::ReducedBitcastBSwap:
case TreeEntry::ReducedBitcastLoads:
case TreeEntry::ReducedBitcastBSwapLoads:
case TreeEntry::ReducedBitPack:
case TreeEntry::ReducedCmpBitcast:
ShuffleOrOp = E->CombinedOp;
break;
default:
break;
}
}
Instruction *VL0 = E->getMainOp();
auto GetOperandSignedness = [&](unsigned Idx) {
const TreeEntry *OpE = getOperandEntry(E, Idx);
bool IsSigned = false;
auto It = MinBWs.find(OpE);
if (It != MinBWs.end())
IsSigned = It->second.second;
else
IsSigned = any_of(OpE->Scalars, [&](Value *R) {
if (isa<PoisonValue>(V))
return false;
return !isKnownNonNegative(R, SimplifyQuery(*DL));
});
return IsSigned;
};
auto PropagateIRFlags = [&](Value *V, unsigned Opcode = 0,
ArrayRef<Value *> VL = {}) -> Value * {
ArrayRef<Value *> Scalars = VL.empty() ? E->Scalars : VL;
SmallSetVector<Value *, 4> UniqueInsts;
for (Instruction *I : make_isa_range<Instruction>(Scalars)) {
if (E->hasCopyableElements() && E->isCopyableElement(I))
continue;
UniqueInsts.insert(I);
}
if (!Opcode)
Opcode = E->getOpcode();
propagateIRFlags(V, UniqueInsts.getArrayRef(), nullptr, It == MinBWs.end());
auto *I = dyn_cast<Instruction>(V);
if (!I)
return V;
// Drop nneg propagated from the original cast.
Value *Op;
if (auto *CI = dyn_cast<CastInst>(VL0);
CI && match(I, m_NNegZExt(m_Value(Op))) &&
Op->getType()->getScalarType() != CI->getSrcTy())
cast<ZExtInst>(I)->setNonNeg(false);
// A lane may be emitted in negated form (sub C, x as add x, -C, or an
// add/sub lane with swapped operands) when all its uses are
// sign-insensitive (icmp eq/ne 0 or abs). The value stays correct for
// such uses, but nsw/nuw of the original instruction do not cover the
// negated overflow conditions.
if (E->State == TreeEntry::Vectorize && E->getNumOperands() == 2 &&
I->getType()->isIntOrIntVectorTy() &&
(Opcode == Instruction::Add || Opcode == Instruction::Sub)) {
auto IsNegatedLane = [&](Instruction *LaneI, unsigned Lane) {
unsigned LaneOpcode = LaneI->getOpcode();
if (LaneOpcode != Instruction::Add && LaneOpcode != Instruction::Sub)
return false;
Value *L = E->getOperand(0)[Lane];
Value *R = E->getOperand(1)[Lane];
if (LaneOpcode == Opcode)
// Only a sub lane emitted with swapped operands is negated.
return Opcode == Instruction::Sub && L == LaneI->getOperand(1) &&
R == LaneI->getOperand(0) && L != R;
if (Opcode == Instruction::Add)
// sub C, x emitted as x + -C.
return isa<ConstantInt>(LaneI->getOperand(0)) &&
!isa<ConstantInt>(LaneI->getOperand(1));
// add x, C emitted as -C - x.
return isa<ConstantInt>(L) && !isa<ConstantInt>(R);
};
if (any_of(enumerate(E->Scalars), [&](const auto &P) {
auto *LaneI = dyn_cast<Instruction>(P.value());
return LaneI &&
(!E->hasCopyableElements() ||
!E->isCopyableElement(LaneI)) &&
(VL.empty() || is_contained(VL, LaneI)) &&
IsNegatedLane(LaneI, P.index());
}))
I->dropPoisonGeneratingFlags();
}
I = ::propagateMetadata(I, UniqueInsts.getArrayRef());
// A copyable lane is emitted as a synthesized operation - an absorbed
// fmul becomes fmuladd(a, b, -0.0), other scalars are combined with a
// binop identity value - so every fast-math flag must be justified by
// each copyable scalar: an FP operator justifies only its own flags, a
// constant FP the flags its value cannot violate, and any other value
// only the transform permissions, which cannot change an identity
// lane. An unjustified flag poisons NaN/Inf inputs, flushes a -0.0
// result, or changes the rounding of an absorbed lane.
if (E->hasCopyableElements() && isa<FPMathOperator>(I)) {
FastMathFlags CopyableFMF;
CopyableFMF.set();
auto IsCopyable = [&](Value *V) { return E->isCopyableElement(V); };
for (Value *Scalar : make_filter_range(Scalars, IsCopyable)) {
if (auto *FPMO = dyn_cast<FPMathOperator>(Scalar)) {
CopyableFMF &= FPMO->getFastMathFlags();
continue;
}
auto *CFP = dyn_cast<ConstantFP>(Scalar);
if (!CFP || CFP->isNaN())
CopyableFMF.setNoNaNs(false);
if (!CFP || CFP->isInfinity())
CopyableFMF.setNoInfs(false);
if (!CFP || CFP->isNegZero())
CopyableFMF.setNoSignedZeros(false);
}
I->copyFastMathFlags(I->getFastMathFlags() & CopyableFMF);
}
// Copyable lanes modeled as op(V, V) or with an undef or poison operand do
// not preserve the disjoint flag: or disjoint(V, V) is poison unless V is
// zero, and such an operand may be any value. Matched by value, so operand
// reordering does not affect the detection.
auto *PDI = dyn_cast<PossiblyDisjointInst>(I);
if (E->hasCopyableElements() && PDI &&
any_of(enumerate(E->Scalars), [&](const auto &P) {
Value *Scalar = P.value();
auto IsScalarOrUndef = [Scalar](Value *Op) {
return Op == Scalar || isa<UndefValue>(Op);
};
return E->isCopyableElement(Scalar) && !match(Scalar, m_Zero()) &&
IsScalarOrUndef(E->getOperand(0)[P.index()]) &&
IsScalarOrUndef(E->getOperand(1)[P.index()]);
}))
PDI->setIsDisjoint(false);
// The undef copyable lanes are modeled as op(undef, undef), which must not
// overflow into poison.
if (E->hasCopyableElements() && isUndefTolerantBinOp(I) &&
any_of(E->Scalars, [&](Value *Scalar) {
return isa<UndefValue>(Scalar) && !isa<PoisonValue>(Scalar) &&
E->isCopyableElement(Scalar);
}))
I->dropPoisonGeneratingFlags();
// Drop nuw flags for abs(sub(commutative), true).
if (!MinBWs.contains(E) && Opcode == Instruction::Sub &&
(E->hasCopyableElements() || any_of(Scalars, [](Value *Scalar) {
auto *SI = dyn_cast<Instruction>(Scalar);
return !SI || isCommutative(SI);
})))
I->setHasNoUnsignedWrap(/*b=*/false);
// A sub feeding icmp eq/ne 0 may have its operands swapped; nsw does not
// survive a - b -> b - a (a - b can be INT_MIN while b - a overflows).
if (!MinBWs.contains(E) && Opcode == Instruction::Sub &&
any_of(make_isa_range<Instruction>(Scalars), [](Instruction *SI) {
if (SI->getOpcode() != Instruction::Sub || !isCommutative(SI))
return false;
return any_of(SI->uses(), [](const Use &U) {
CmpPredicate Pred;
return match(U.getUser(),
m_ICmp(Pred, m_Specific(U.get()), m_Zero())) &&
ICmpInst::isEquality(Pred);
});
}))
I->setHasNoSignedWrap(/*b=*/false);
// Interchanging add/sub negates the constant: nsw only survives if the
// constant isn't INT_MIN (negating it would overflow); nuw never
// survives a nonzero constant, since that flips the valid range from
// X >= C to X < C.
if (!MinBWs.contains(E) &&
(Opcode == Instruction::Add || Opcode == Instruction::Sub)) {
for (Value *V : UniqueInsts) {
auto *SI = cast<Instruction>(V);
if (SI->getOpcode() == Opcode ||
!is_contained({Instruction::Add, Instruction::Sub},
SI->getOpcode()))
continue;
for (const ConstantInt *CI :
make_isa_range<ConstantInt>(SI->operands())) {
if (CI->isZero())
continue;
I->setHasNoUnsignedWrap(/*b=*/false);
if (CI->getValue().isMinSignedValue())
I->setHasNoSignedWrap(/*b=*/false);
}
}
}
// mul nsw X, INT_MIN is not equivalent to shl nsw X, BW-1, because shl nsw
// poisons when the sign bit is shifted out of a positive value. When a mul
// lane is converted to shl with shift amount BW-1, nsw must be dropped.
if (!MinBWs.contains(E) && Opcode == Instruction::Shl &&
any_of(UniqueInsts, [&](Value *V) {
auto *SI = cast<Instruction>(V);
if (SI->getOpcode() != Instruction::Mul)
return false;
return any_of(make_isa_range<ConstantInt>(SI->operands()),
[](const ConstantInt *CI) {
return CI->getValue().isMinSignedValue();
});
}))
I->setHasNoSignedWrap(/*b=*/false);
// shl nsw X, BW-1 is not equivalent to mul nsw X, INT_MIN: shl nsw never
// poisons at X = -1 (shifted-out bits match the sign bit), but mul nsw
// does. A shl lane converted to mul this way must drop nsw.
if (!MinBWs.contains(E) && Opcode == Instruction::Mul &&
any_of(UniqueInsts, [&](Value *V) {
auto *SI = cast<Instruction>(V);
if (SI->getOpcode() != Instruction::Shl)
return false;
const auto *CI = dyn_cast<ConstantInt>(SI->getOperand(1));
return CI && CI->getValue() == CI->getBitWidth() - 1;
}))
I->setHasNoSignedWrap(/*b=*/false);
// Keep the intersected samesign unless narrowing changed the sign of some
// compared operand (it neither sign-extends from the narrow type nor has
// a known-zero narrow sign bit), or a converted lane's adjusted constant
// flips the sign.
if (auto *ICmp = dyn_cast<ICmpInst>(I); ICmp && It == MinBWs.end()) {
bool SignChange =
ICmp->getOperand(0)->getType()->getScalarType() !=
VL0->getOperand(0)->getType()->getScalarType() &&
((!GetOperandSignedness(0) &&
!isKnownNonNegative(ICmp->getOperand(0), SimplifyQuery(*DL))) ||
(!GetOperandSignedness(1) &&
!isKnownNonNegative(ICmp->getOperand(1), SimplifyQuery(*DL))));
CmpInst::Predicate P0 = cast<CmpInst>(E->getMainOp())->getPredicate();
bool SignFlip =
!SignChange &&
any_of(make_isa_range<ICmpInst>(E->Scalars), [&](ICmpInst *LaneCI) {
const APInt *OrigC, *AdjC;
Constant *Adj =
CmpSamePredicateHelper::getAdjustedConstant(LaneCI, P0);
return Adj && match(LaneCI->getOperand(1), m_APInt(OrigC)) &&
match(Adj, m_APInt(AdjC)) &&
AdjC->isNegative() != OrigC->isNegative();
});
if (SignChange || SignFlip)
ICmp->setSameSign(/*B=*/false);
}
return I;
};
switch (ShuffleOrOp) {
case Instruction::PHI: {
assert((E->ReorderIndices.empty() || !E->ReuseShuffleIndices.empty() ||
E != &getRootNode() || E->UserTreeIndex) &&
"PHI reordering is free.");
auto *PH = cast<PHINode>(VL0);
Builder.SetInsertPoint(PH->getParent()->getFirstNonPHIIt());
Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
Value *V = NewPhi;
// Adjust insertion point once all PHI's have been generated.
Builder.SetInsertPoint(PH->getParent()->getFirstInsertionPt());
Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
V = FinalShuffle(V, E);
E->VectorizedValue = V;
// If phi node is fully emitted - exit.
if (NewPhi->getNumIncomingValues() != 0)
return NewPhi;
// PHINodes may have multiple entries from the same block. We want to
// visit every block once.
SmallDenseMap<BasicBlock *, unsigned, 4> VisitedBBs;
for (unsigned I : seq<unsigned>(PH->getNumIncomingValues())) {
BasicBlock *IBB = PH->getIncomingBlock(I);
// Stop emission if all incoming values are generated.
if (NewPhi->getNumIncomingValues() == PH->getNumIncomingValues()) {
LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
return NewPhi;
}
Builder.SetInsertPoint(IBB->getTerminator());
Builder.SetCurrentDebugLocation(getDebugLocFromPHI(*PH));
auto Res = VisitedBBs.try_emplace(IBB, I);
if (!Res.second) {
TreeEntry *OpTE = getOperandEntry(E, I);
if (OpTE->isGather() || DeletedNodes.contains(OpTE) ||
TransformedToGatherNodes.contains(OpTE)) {
Value *VecOp = NewPhi->getIncomingValue(Res.first->getSecond());
NewPhi->addIncoming(VecOp, IBB);
assert(!OpTE->VectorizedValue && "Expected no vectorized value.");
OpTE->VectorizedValue = VecOp;
// The combined nodes of the reused gather node still must be
// emitted.
if (OpTE->isGather() && !DeletedNodes.contains(OpTE))
for (auto [EIdx, _] : OpTE->CombinedEntriesWithIndices)
(void)vectorizeTree(VectorizableTree[EIdx].get());
continue;
}
}
Value *Vec = vectorizeOperand(E, I);
if (VecTy != Vec->getType()) {
assert((It != MinBWs.end() || getOperandEntry(E, I)->isGather() ||
MinBWs.contains(getOperandEntry(E, I))) &&
"Expected item in MinBWs.");
Vec = Builder.CreateIntCast(Vec, VecTy, GetOperandSignedness(I));
}
NewPhi->addIncoming(Vec, IBB);
}
assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
"Invalid number of incoming values");
assert(E->VectorizedValue && "Expected vectorized value.");
return E->VectorizedValue;
}
case Instruction::ExtractElement: {
Value *V = E->getSingleOperand(0);
setInsertPointAfterBundle(E);
V = FinalShuffle(V, E);
// Insert the copyable lanes (non-extract scalars modeled on the extract
// main op) into the reused source vector. The identity extract order and
// absence of a reuse shuffle are guaranteed when the node is created, so
// a lane maps to its own index.
if (E->hasCopyableElements()) {
assert(E->ReorderIndices.empty() && E->ReuseShuffleIndices.empty() &&
"Copyable extract lanes require identity order and no reuse.");
for (auto [Idx, Scalar] : enumerate(E->Scalars))
if (E->isCopyableElement(Scalar))
V = Builder.CreateInsertElement(V, Scalar, Builder.getInt32(Idx));
}
E->VectorizedValue = V;
return V;
}
case Instruction::ExtractValue: {
if (!E->StructEVIndices.empty()) {
setInsertPointAfterBundle(E);
Value *V = vectorizeOperand(E, 0);
V = Builder.CreateExtractValue(V, E->StructEVIndices);
if (auto *I = dyn_cast<Instruction>(V))
V = ::propagateMetadata(I, E->Scalars);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
return V;
}
auto *LI = cast<LoadInst>(E->getSingleOperand(0));
Builder.SetInsertPoint(LI);
Value *Ptr = LI->getPointerOperand();
LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign());
Value *NewV = PropagateIRFlags(V);
NewV = FinalShuffle(NewV, E);
E->VectorizedValue = NewV;
return NewV;
}
case Instruction::InsertValue:
case Instruction::InsertElement: {
assert(E->ReuseShuffleIndices.empty() && "All inserts should be unique");
if (const TreeEntry *OpE = getOperandEntry(E, 1);
OpE && !OpE->isGather() && OpE->hasState() &&
!OpE->hasCopyableElements())
Builder.SetInsertPoint(cast<Instruction>(E->Scalars.back()));
else
setInsertPointAfterBundle(E);
Value *V = vectorizeOperand(E, 1);
ArrayRef<Value *> Op = E->getOperand(1);
Type *ScalarTy = Op.front()->getType();
if (cast<VectorType>(V->getType())->getElementType() != ScalarTy) {
assert(ScalarTy->isIntegerTy() && "Expected item in MinBWs.");
std::pair<unsigned, bool> Res = MinBWs.lookup(getOperandEntry(E, 1));
assert(Res.first > 0 && "Expected item in MinBWs.");
V = Builder.CreateIntCast(
V,
getWidenedType(
ScalarTy,
cast<FixedVectorType>(V->getType())->getNumElements()),
Res.second);
}
// Create InsertVector shuffle if necessary
auto *FirstInsert = cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
}));
const unsigned NumScalars = E->Scalars.size();
FixedVectorType *SrcVecTy = getInsertBuildVectorSrcTy(E);
const unsigned NumElts = getNumElements(SrcVecTy);
unsigned Offset = *getElementIndex(VL0);
assert(Offset < NumElts && "Failed to find vector index offset");
// Create shuffle to resize vector
SmallVector<int> Mask;
if (!E->ReorderIndices.empty()) {
inversePermutation(E->ReorderIndices, Mask);
Mask.append(NumElts - NumScalars, PoisonMaskElem);
} else {
Mask.assign(NumElts, PoisonMaskElem);
std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0);
}
// Create InsertVector shuffle if necessary
bool IsIdentity = true;
SmallVector<int> PrevMask(NumElts, PoisonMaskElem);
Mask.swap(PrevMask);
for (unsigned I = 0; I < NumScalars; ++I) {
Value *Scalar = E->Scalars[PrevMask[I]];
unsigned InsertIdx = *getElementIndex(Scalar);
IsIdentity &= InsertIdx - Offset == I;
Mask[InsertIdx - Offset] = I;
}
if (!IsIdentity || NumElts != NumScalars) {
Value *V2 = nullptr;
bool IsVNonPoisonous =
!isConstant(V) && isGuaranteedNotToBePoison(V, AC);
SmallVector<int> InsertMask(Mask);
if (NumElts != NumScalars && Offset == 0) {
// Follow all insert element instructions from the current buildvector
// sequence.
Instruction *Ins = VL0;
Instruction *Op;
do {
std::optional<unsigned> InsertIdx = getElementIndex(Ins);
if (!InsertIdx)
break;
if (InsertMask[*InsertIdx] == PoisonMaskElem)
InsertMask[*InsertIdx] = *InsertIdx;
if (!Ins->hasOneUse())
break;
Op = Ins;
Ins = dyn_cast_if_present<Instruction>(
Ins->getUniqueUndroppableUser());
} while (Ins && Ins->getOperand(0) == Op);
SmallBitVector UseMask =
buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask);
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
SmallBitVector IsFirstUndef =
isUndefVector(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
unsigned Idx = 0;
for (unsigned I = 0; I < NumElts; I++) {
if (InsertMask[I] == PoisonMaskElem && !IsFirstPoison.test(I) &&
IsFirstUndef.test(I)) {
if (IsVNonPoisonous) {
InsertMask[I] = I < NumScalars ? I : 0;
continue;
}
if (!V2)
V2 = UndefValue::get(V->getType());
if (Idx >= NumScalars)
Idx = NumScalars - 1;
InsertMask[I] = NumScalars + Idx;
++Idx;
} else if (InsertMask[I] != PoisonMaskElem &&
Mask[I] == PoisonMaskElem) {
InsertMask[I] = PoisonMaskElem;
}
}
} else {
InsertMask = Mask;
}
}
if (!V2)
V2 = PoisonValue::get(V->getType());
V = Builder.CreateShuffleVector(V, V2, InsertMask);
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
}
SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
for (unsigned I = 0; I < NumElts; I++) {
if (Mask[I] != PoisonMaskElem)
InsertMask[Offset + I] = I;
}
SmallBitVector UseMask =
buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask);
SmallBitVector IsFirstUndef =
isUndefVector(FirstInsert->getOperand(0), UseMask);
if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) &&
NumElts != NumScalars) {
if (IsFirstUndef.all()) {
if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
if (!IsFirstPoison.all()) {
for (unsigned I = 0; I < NumElts; I++) {
if (InsertMask[I] == PoisonMaskElem && !IsFirstPoison.test(I))
InsertMask[I] = I + NumElts;
}
}
V = Builder.CreateShuffleVector(
V,
IsFirstPoison.all() ? PoisonValue::get(V->getType())
: FirstInsert->getOperand(0),
InsertMask, cast<Instruction>(E->Scalars.back())->getName());
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
}
} else {
SmallBitVector IsFirstPoison =
isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
for (unsigned I = 0; I < NumElts; I++) {
if (InsertMask[I] == PoisonMaskElem)
InsertMask[I] = IsFirstPoison.test(I) ? PoisonMaskElem : I;
else
InsertMask[I] += NumElts;
}
V = Builder.CreateShuffleVector(
FirstInsert->getOperand(0), V, InsertMask,
cast<Instruction>(E->Scalars.back())->getName());
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
}
}
++NumVectorInstructions;
if (ShuffleOrOp == Instruction::InsertValue &&
!canVectorStoreInsertValue(E)) {
Type *AggTy = E->getMainOp()->getType();
Align SlotAlign = std::max(DL->getPrefTypeAlign(V->getType()),
DL->getPrefTypeAlign(AggTy));
AllocaInst *Slot;
{
// Keep the alloca in the function entry block so it is not executed
// (and does not grow the stack) on every iteration of a loop.
IRBuilderBase::InsertPointGuard Guard(Builder);
Builder.SetInsertPointPastAllocas(F);
Slot = Builder.CreateAlloca(AggTy, /*ArraySize=*/nullptr,
"vec2struct.slot");
Slot->setAlignment(SlotAlign);
}
(void)Builder.CreateAlignedStore(V, Slot, SlotAlign);
V = Builder.CreateAlignedLoad(AggTy, Slot, SlotAlign, "vec2struct");
NumVectorInstructions += 2;
}
E->VectorizedValue = V;
return V;
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::PtrToAddr:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
setInsertPointAfterBundle(E);
Value *InVec = vectorizeOperand(E, 0);
auto *CI = cast<CastInst>(VL0);
Instruction::CastOps VecOpcode = CI->getOpcode();
Type *SrcScalarTy = cast<VectorType>(InVec->getType())->getElementType();
auto SrcIt = MinBWs.find(getOperandEntry(E, 0));
if (!ScalarTy->isFPOrFPVectorTy() && !SrcScalarTy->isFPOrFPVectorTy() &&
(SrcIt != MinBWs.end() || It != MinBWs.end() ||
SrcScalarTy != CI->getOperand(0)->getType()->getScalarType())) {
// Check if the values are candidates to demote.
unsigned SrcBWSz = DL->getTypeSizeInBits(SrcScalarTy);
if (SrcIt != MinBWs.end())
SrcBWSz = SrcIt->second.first;
unsigned BWSz = DL->getTypeSizeInBits(ScalarTy->getScalarType());
if (BWSz == SrcBWSz) {
VecOpcode = Instruction::BitCast;
} else if (BWSz < SrcBWSz) {
VecOpcode = Instruction::Trunc;
} else if (It != MinBWs.end()) {
assert(BWSz > SrcBWSz && "Invalid cast!");
VecOpcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
} else if (SrcIt != MinBWs.end()) {
assert(BWSz > SrcBWSz && "Invalid cast!");
VecOpcode =
SrcIt->second.second ? Instruction::SExt : Instruction::ZExt;
}
} else if (VecOpcode == Instruction::SIToFP && SrcIt != MinBWs.end() &&
!SrcIt->second.second) {
VecOpcode = Instruction::UIToFP;
} else if (VecOpcode == Instruction::BitCast && SrcIt != MinBWs.end() &&
ScalarTy->isFPOrFPVectorTy()) {
Type *OrigSrcScalarTy = CI->getSrcTy();
auto *OrigSrcVectorTy =
getWidenedType(OrigSrcScalarTy, E->Scalars.size());
InVec =
Builder.CreateIntCast(InVec, OrigSrcVectorTy, SrcIt->second.second);
} else if (VecOpcode == Instruction::UIToFP && SrcIt != MinBWs.end() &&
SrcIt->second.second) {
auto *OrigSrcVectorTy =
getWidenedType(CI->getSrcTy(), E->Scalars.size());
InVec = Builder.CreateSExt(InVec, OrigSrcVectorTy);
}
Value *V = (VecOpcode != ShuffleOrOp && VecOpcode == Instruction::BitCast)
? InVec
: Builder.CreateCast(VecOpcode, InVec, VecTy);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::FCmp:
case Instruction::ICmp: {
setInsertPointAfterBundle(E);
Value *L = vectorizeOperand(E, 0);
Value *R = vectorizeOperand(E, 1);
if (L->getType() != R->getType()) {
assert((getOperandEntry(E, 0)->isGather() ||
getOperandEntry(E, 1)->isGather() ||
MinBWs.contains(getOperandEntry(E, 0)) ||
MinBWs.contains(getOperandEntry(E, 1))) &&
"Expected item in MinBWs.");
const unsigned LBW = cast<VectorType>(L->getType())
->getElementType()
->getIntegerBitWidth();
const unsigned RBW = cast<VectorType>(R->getType())
->getElementType()
->getIntegerBitWidth();
// A signed comparison needs the sign bit getActiveBits() ignores.
const bool Signed = cast<CmpInst>(VL0)->isSigned();
auto GetRequiredBits = [Signed](const APInt &V) {
return Signed ? V.getSignificantBits() : V.getActiveBits();
};
if ((LBW < RBW && (!allConstant(E->getOperand(1)) ||
any_of(
E->getOperand(1),
[&](Value *V) {
auto *CI = dyn_cast<ConstantInt>(V);
return !CI ||
GetRequiredBits(CI->getValue()) > LBW;
}))) ||
(LBW > RBW && allConstant(E->getOperand(0)) &&
all_of(E->getOperand(1), [&](Value *V) {
auto *CI = dyn_cast<ConstantInt>(V);
return CI && GetRequiredBits(CI->getValue()) <= RBW;
}))) {
Type *CastTy = R->getType();
L = Builder.CreateIntCast(L, CastTy, GetOperandSignedness(0));
} else {
Type *CastTy = L->getType();
R = Builder.CreateIntCast(R, CastTy, GetOperandSignedness(1));
}
}
CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
// The bit tests compare and L, R with 0, the eq bit tests compare the
// trunc X to i1 lanes with 1, exactly in the narrowed type.
const bool IsBitTest = any_of(E->Scalars, IsaPred<TruncInst>);
if (IsBitTest) {
L = Builder.CreateAnd(L, R);
const bool IsEq =
CmpSamePredicateHelper::canConvertTo(VL0, CmpInst::ICMP_EQ);
P0 = IsEq ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
Type *EltTy = L->getType()->getScalarType();
SmallVector<Constant *> Consts;
for (Value *V : E->Scalars)
Consts.append(getNumElements(VL0->getType()),
ConstantInt::get(EltTy, IsEq && isa<TruncInst>(V)));
R = ConstantVector::get(Consts);
}
// A compare with both operands narrowed to the same type preserves the
// result only if the predicate matches the operand signedness: signed
// predicates need both operands sign-extended from the narrow type,
// unsigned predicates need both zero-extended, equality needs the same
// extension on both sides; operands with a known-zero narrow sign bit
// match any predicate. Otherwise, extend the operands back to the
// original compared type.
auto *LTy = cast<FixedVectorType>(L->getType());
auto *OrigOpTy =
dyn_cast<IntegerType>(VL0->getOperand(0)->getType()->getScalarType());
if (!IsBitTest && OrigOpTy && L->getType() == R->getType() &&
LTy->getElementType()->getIntegerBitWidth() <
OrigOpTy->getBitWidth()) {
bool IsSigned0 = GetOperandSignedness(0);
bool IsSigned1 = GetOperandSignedness(1);
bool NonNeg0 = isKnownNonNegative(L, SimplifyQuery(*DL));
bool NonNeg1 = isKnownNonNegative(R, SimplifyQuery(*DL));
bool KeepNarrowTy;
if (ICmpInst::isSigned(P0))
KeepNarrowTy = (IsSigned0 || NonNeg0) && (IsSigned1 || NonNeg1);
else if (ICmpInst::isUnsigned(P0))
KeepNarrowTy = (!IsSigned0 || NonNeg0) && (!IsSigned1 || NonNeg1);
else
KeepNarrowTy = IsSigned0 == IsSigned1 || NonNeg0 || NonNeg1;
if (!KeepNarrowTy) {
Type *CastTy = getWidenedType(OrigOpTy, LTy->getNumElements());
L = Builder.CreateIntCast(L, CastTy, IsSigned0);
R = Builder.CreateIntCast(R, CastTy, IsSigned1);
}
}
// A booleanized zero-test leaf emits no comparison: its value is the
// wide operand vector, and the i1 lanes are restored by the trunc at
// the booleanized user node. (Trunc leaves are no-ops after the generic
// MinBWs cast processing.)
bool IsBooleanized = isBooleanizedNode(E);
Value *V = IsBooleanized
? (all_of(E->getOperand(0),
[](Value *V) { return match(V, m_Zero()); })
? R
: L)
: Builder.CreateCmp(P0, L, R);
if (!IsBooleanized)
V = PropagateIRFlags(V);
// Do not cast for cmps.
VecTy = cast<FixedVectorType>(V->getType());
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Select: {
setInsertPointAfterBundle(E);
Value *Cond = vectorizeOperand(E, 0);
Value *True = vectorizeOperand(E, 1);
Value *False = vectorizeOperand(E, 2);
if (True->getType() != VecTy || False->getType() != VecTy) {
assert((It != MinBWs.end() || getOperandEntry(E, 1)->isGather() ||
getOperandEntry(E, 2)->isGather() ||
MinBWs.contains(getOperandEntry(E, 1)) ||
MinBWs.contains(getOperandEntry(E, 2))) &&
"Expected item in MinBWs.");
if (True->getType() != VecTy)
True = Builder.CreateIntCast(True, VecTy, GetOperandSignedness(1));
if (False->getType() != VecTy)
False = Builder.CreateIntCast(False, VecTy, GetOperandSignedness(2));
}
unsigned CondNumElements = getNumElements(Cond->getType());
unsigned TrueNumElements = getNumElements(True->getType());
assert(TrueNumElements >= CondNumElements &&
TrueNumElements % CondNumElements == 0 &&
"Cannot vectorize Instruction::Select");
assert(TrueNumElements == getNumElements(False->getType()) &&
"Cannot vectorize Instruction::Select");
if (CondNumElements != TrueNumElements) {
// When the return type is i1 but the source is fixed vector type, we
// need to duplicate the condition value.
Cond = Builder.CreateShuffleVector(
Cond, createReplicatedMask(TrueNumElements / CondNumElements,
CondNumElements));
}
assert(getNumElements(Cond->getType()) == TrueNumElements &&
"Cannot vectorize Instruction::Select");
Value *V =
Builder.CreateSelectWithUnknownProfile(Cond, True, False, DEBUG_TYPE);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::FNeg: {
setInsertPointAfterBundle(E);
Value *Op = vectorizeOperand(E, 0);
Value *V = Builder.CreateUnOp(
static_cast<Instruction::UnaryOps>(E->getOpcode()), Op);
V = PropagateIRFlags(V);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Freeze: {
setInsertPointAfterBundle(E);
Value *Op = vectorizeOperand(E, 0);
if (Op->getType() != VecTy) {
assert((It != MinBWs.end() || getOperandEntry(E, 0)->isGather() ||
MinBWs.contains(getOperandEntry(E, 0))) &&
"Expected item in MinBWs.");
Op = Builder.CreateIntCast(Op, VecTy, GetOperandSignedness(0));
}
Value *V = Builder.CreateFreeze(Op);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::FDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
setInsertPointAfterBundle(E);
// The lanes with the absorbing constant use poison as the other operand.
// Freeze the operand, if it may be poison, to keep these lanes equal to
// the constant.
auto FreezePlaceholders = [&](Value *Op, unsigned OpIdx) {
if (!E->hasCopyableElements() ||
none_of(zip(E->Scalars, E->getOperand(OpIdx)),
[&](const auto &P) {
auto [Scalar, OpScalar] = P;
return E->isCopyableElement(Scalar) &&
!isa<UndefValue>(Scalar) &&
isa<UndefValue>(OpScalar);
}) ||
isGuaranteedNotToBePoison(Op, AC))
return Op;
return Builder.CreateFreeze(Op);
};
if (E->hasReassocScalars()) {
// Vectorize operand columns, then combine pairwise in a balanced
// tree; negated columns (flattened subtracts) are subtracted from
// the positive total at the end.
SmallVector<Value *> CombinedScalars = to_vector(E->Scalars);
append_range(CombinedScalars, E->getReassocScalars());
// Reuse flags when a combine exactly matches a subsumed scalar pair.
// Commutative scalars are keyed operand-order independently,
// subtracts keep their operand order.
auto MakeOperandKey = [](Value *A, Value *B, unsigned Opcode) {
return Instruction::isCommutative(Opcode) && B < A
? std::make_pair(B, A)
: std::make_pair(A, B);
};
SmallDenseMap<std::pair<std::pair<Value *, Value *>, unsigned>,
SmallVector<Instruction *, 1>>
ByOperands;
// Copyable leaves are used as-is, they are never combine targets.
for (Value *V : make_filter_range(CombinedScalars, [&](Value *V) {
return !E->isCopyableElement(V);
})) {
auto *I = cast<Instruction>(V);
const unsigned Opcode = I->getOpcode();
ByOperands[{MakeOperandKey(I->getOperand(0), I->getOperand(1),
Opcode),
Opcode}]
.push_back(I);
}
const unsigned CombineOpcode = getReassocCombineOpcode(E->getOpcode());
// Cast operand columns to VecTy when bit-width demotion changed types.
auto GetCastOperand = [&](unsigned Idx, Value *Op) {
if (Op->getType() == VecTy)
return Op;
return Builder.CreateIntCast(Op, VecTy, GetOperandSignedness(Idx));
};
SmallVector<Value *> Ops;
SmallVector<Value *> NegOps;
// Track which scalar each partial result still represents for flag
// reuse.
SmallVector<SmallVector<Value *>> ScalarOps;
SmallVector<SmallVector<Value *>> NegScalarOps;
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
// Identity-only columns combine to a no-op; skip them rather than
// emitting a real binop against an identity vector.
if (all_of(E->getOperand(Idx), [&](Value *V) {
return isBinOpIdentityConstant(V, CombineOpcode);
}))
continue;
const bool Negated = E->isReassocNegatedOp(Idx);
(Negated ? NegOps : Ops)
.push_back(GetCastOperand(
Idx, FreezePlaceholders(vectorizeOperand(E, Idx), Idx)));
(Negated ? NegScalarOps : ScalarOps)
.emplace_back(E->getOperand(Idx).begin(),
E->getOperand(Idx).end());
}
const bool HasNegatedOps = !NegOps.empty();
// Check whether combining the partial results of \p LHSScalars and
// \p RHSScalars reproduces a subsumed scalar pair lane by lane, so
// the matched scalars' flags can be reused. Fills per-lane
// representatives for the next combine round on an exact match.
auto TryExactCombine =
[&](Value *Combined, unsigned Opcode, ArrayRef<Value *> LHSScalars,
ArrayRef<Value *> RHSScalars,
SmallVectorImpl<Value *> &Representatives) -> Value * {
// Either side may be empty after a non-exact combine.
if (LHSScalars.empty() || LHSScalars.size() != RHSScalars.size())
return nullptr;
SmallVector<Value *> ExactMatches;
Representatives.resize(LHSScalars.size());
bool IsExact =
all_of(seq<unsigned>(LHSScalars.size()), [&](unsigned Lane) {
if (!LHSScalars[Lane] || !RHSScalars[Lane])
return false;
auto It = ByOperands.find(
{MakeOperandKey(LHSScalars[Lane], RHSScalars[Lane], Opcode),
Opcode});
if (It == ByOperands.end())
return false;
append_range(ExactMatches, It->second);
Representatives[Lane] = It->second.front();
return true;
});
if (!IsExact)
return nullptr;
return PropagateIRFlags(Combined, Opcode, ExactMatches);
};
auto CombineFallback = [&](Value *Combined, unsigned Opcode) {
Value *V = PropagateIRFlags(Combined, Opcode, CombinedScalars);
// Drop overflow/fast-math flags not proven exact; regrouping can
// invalidate nsw/nuw and nnan/ninf even when each step was safe.
// nuw is kept for pure add trees only: a negated leaf can make a
// re-paired partial sum exceed the original running total.
auto *CombinedI = dyn_cast<Instruction>(V);
if (!CombinedI)
return V;
if (isa<FPBinaryOperator>(CombinedI)) {
CombinedI->setHasNoNaNs(false);
CombinedI->setHasNoInfs(false);
} else if (isa<OverflowingBinaryOperator>(CombinedI)) {
CombinedI->setHasNoSignedWrap(false);
if (Opcode != Instruction::Add || HasNegatedOps)
CombinedI->setHasNoUnsignedWrap(false);
}
return V;
};
// Combine the group columns pairwise in a balanced tree and return
// the total with the per-lane scalars it still represents.
auto CombineColumns =
[&](SmallVector<Value *> GroupOps,
SmallVector<SmallVector<Value *>> GroupScalarOps,
unsigned Opcode) -> std::pair<Value *, SmallVector<Value *>> {
while (GroupOps.size() > 1) {
SmallVector<Value *> NextOps((GroupOps.size() + 1) / 2);
SmallVector<SmallVector<Value *>> NextScalarOps(NextOps.size());
for (unsigned Idx : seq<unsigned>(NextOps.size())) {
if (2 * Idx + 1 == GroupOps.size()) {
// Odd one out this round: carry it over unpaired.
NextOps[Idx] = GroupOps[2 * Idx];
NextScalarOps[Idx] = std::move(GroupScalarOps[2 * Idx]);
continue;
}
Value *Combined = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(Opcode),
GroupOps[2 * Idx], GroupOps[2 * Idx + 1]);
// Check whether this combine exactly matches a subsumed
// scalar.
SmallVector<Value *> Representatives;
if (Value *Exact = TryExactCombine(
Combined, Opcode, GroupScalarOps[2 * Idx],
GroupScalarOps[2 * Idx + 1], Representatives)) {
NextOps[Idx] = Exact;
NextScalarOps[Idx] = std::move(Representatives);
continue;
}
NextOps[Idx] = CombineFallback(Combined, Opcode);
// No scalar match: deeper combines from this result cannot be
// exact.
}
GroupOps = std::move(NextOps);
GroupScalarOps = std::move(NextScalarOps);
}
return {GroupOps.front(), GroupScalarOps.front()};
};
// -(a+b) differs from -a + -b in the sign of a zero result, so the
// negated columns are summed only for integers or with nsz on every
// link; without it each is subtracted in turn.
const bool CanSumNegated =
CombineOpcode != Instruction::FAdd ||
all_of(make_isa_range<Instruction>(CombinedScalars),
[&](Instruction *I) {
return E->isCopyableElement(I) || I->hasNoSignedZeros();
});
const unsigned SubOpcode = CombineOpcode == Instruction::FAdd
? Instruction::FSub
: Instruction::Sub;
Value *V;
SmallVector<Value *> PosReps;
if (Ops.empty()) {
// Every positive column was an identity: the total is the identity.
V = ConstantExpr::getBinOpIdentity(CombineOpcode, VecTy);
} else {
std::tie(V, PosReps) = CombineColumns(
std::move(Ops), std::move(ScalarOps), CombineOpcode);
}
if (CanSumNegated && !NegOps.empty()) {
auto [NegTotal, NegReps] = CombineColumns(
std::move(NegOps), std::move(NegScalarOps), CombineOpcode);
NegOps = {NegTotal};
NegScalarOps.clear();
NegScalarOps.push_back(std::move(NegReps));
}
for (auto [NegOp, NegScalars] : zip(NegOps, NegScalarOps)) {
Value *Combined = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(SubOpcode), V, NegOp);
SmallVector<Value *> NextReps;
if (Value *Exact = TryExactCombine(Combined, SubOpcode, PosReps,
NegScalars, NextReps)) {
V = Exact;
PosReps = std::move(NextReps);
continue;
}
V = CombineFallback(Combined, SubOpcode);
PosReps.clear();
}
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
Value *LHS = vectorizeOperand(E, 0);
Value *RHS = vectorizeOperand(E, 1);
if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
assert(E->getNumOperands() == 2 && "Expected exactly 2 operands.");
for (unsigned I : seq<unsigned>(0, E->getNumOperands())) {
ArrayRef<Value *> Ops = E->getOperand(I);
if (all_of(Ops, [&](Value *Op) {
auto *CI = dyn_cast<ConstantInt>(Op);
return CI && CI->getValue().countr_one() >= It->second.first;
})) {
V = FinalShuffle(I == 0 ? RHS : LHS, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
}
}
LHS = FreezePlaceholders(LHS, 0);
RHS = FreezePlaceholders(RHS, 1);
if (LHS->getType() != VecTy || RHS->getType() != VecTy) {
assert((It != MinBWs.end() || getOperandEntry(E, 0)->isGather() ||
getOperandEntry(E, 1)->isGather() ||
MinBWs.contains(getOperandEntry(E, 0)) ||
MinBWs.contains(getOperandEntry(E, 1))) &&
"Expected item in MinBWs.");
if (LHS->getType() != VecTy)
LHS = Builder.CreateIntCast(LHS, VecTy, GetOperandSignedness(0));
if (RHS->getType() != VecTy)
RHS = Builder.CreateIntCast(RHS, VecTy, GetOperandSignedness(1));
}
// Structural padding does not imply profitability; re-check the cost
// here so codegen matches the earlier cost-based decision.
Value *V = nullptr;
unsigned NumElts = E->Scalars.size();
FixedVectorType *PaddedVecTy = nullptr;
if (getMaskedDivRemCost(*TTI, Opts.slp_revec, ShuffleOrOp, ScalarTy,
NumElts, CostKind, &PaddedVecTy)
.isValid()) {
assert(PaddedVecTy && "Expected padded type for masked div/rem.");
// Scale the lane count up to elements for REVEC, where each lane
// is itself a vector.
unsigned NumActiveElts = NumElts * getNumElements(ScalarTy);
Value *WidenedLHS =
createInsertVector(Builder, PoisonValue::get(PaddedVecTy), LHS, 0);
Value *WidenedRHS =
createInsertVector(Builder, PoisonValue::get(PaddedVecTy), RHS, 0);
SmallVector<Constant *> MaskValues(
PaddedVecTy->getNumElements(),
ConstantInt::getFalse(Builder.getContext()));
std::fill_n(MaskValues.begin(), NumActiveElts,
ConstantInt::getTrue(Builder.getContext()));
Value *DivRemMask = ConstantVector::get(MaskValues);
Value *MaskedV = Builder.CreateIntrinsic(
getMaskedDivRemIntrinsic(ShuffleOrOp), {PaddedVecTy},
{WidenedLHS, WidenedRHS, DivRemMask});
V = createExtractVector(Builder, MaskedV, NumActiveElts, 0);
}
if (!V)
V = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS);
V = PropagateIRFlags(V);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Load: {
// Loads are inserted at the head of the tree because we don't want to
// sink them all the way down past store instructions.
setInsertPointAfterBundle(E);
LoadInst *LI = cast<LoadInst>(VL0);
Instruction *NewLI;
FixedVectorType *StridedLoadTy = nullptr;
Value *PO = LI->getPointerOperand();
if (E->State == TreeEntry::Vectorize) {
const uint64_t Shift = getVectorLoadShift(E->Scalars);
if (Shift)
PO = Builder.CreatePtrAdd(
PO, ConstantInt::getSigned(DL->getIndexType(PO->getType()),
-static_cast<int64_t>(Shift)));
NewLI = Builder.CreateAlignedLoad(
VecTy, PO, commonAlignment(LI->getAlign(), Shift));
} else if (E->State == TreeEntry::CompressVectorize) {
auto [CompressMask, LoadVecTy, InterleaveFactor, IsMasked] =
CompressEntryToData.at(E);
Align CommonAlignment = LI->getAlign();
if (IsMasked) {
unsigned VF = getNumElements(LoadVecTy);
SmallVector<Constant *> MaskValues(
VF / getNumElements(LI->getType()),
ConstantInt::getFalse(VecTy->getContext()));
for (int I : CompressMask)
MaskValues[I] = ConstantInt::getTrue(VecTy->getContext());
if (auto *VecTy = dyn_cast<FixedVectorType>(LI->getType())) {
assert(Opts.slp_revec && "Only supported by REVEC.");
MaskValues = replicateMask(MaskValues, VecTy->getNumElements());
}
Constant *MaskValue = ConstantVector::get(MaskValues);
NewLI = Builder.CreateMaskedLoad(LoadVecTy, PO, CommonAlignment,
MaskValue);
} else {
NewLI = Builder.CreateAlignedLoad(LoadVecTy, PO, CommonAlignment);
}
NewLI = cast<Instruction>(PropagateIRFlags(NewLI));
// TODO: include this cost into CommonCost.
if (auto *VecTy = dyn_cast<FixedVectorType>(LI->getType())) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
CompressMask);
}
NewLI =
cast<Instruction>(Builder.CreateShuffleVector(NewLI, CompressMask));
} else if (E->State == TreeEntry::StridedVectorize) {
Value *Ptr0 = cast<LoadInst>(E->Scalars.front())->getPointerOperand();
Value *PtrN = cast<LoadInst>(E->Scalars.back())->getPointerOperand();
PO = IsReverseOrder ? PtrN : Ptr0;
Type *StrideTy = DL->getIndexType(PO->getType());
Value *StrideVal;
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
StridedLoadTy = SPtrInfo.Ty;
assert(StridedLoadTy && "Missing StridedPointerInfo for tree entry.");
unsigned StridedLoadEC =
StridedLoadTy->getElementCount().getKnownMinValue();
Value *Stride = SPtrInfo.StrideVal;
if (!Stride) {
const SCEV *StrideSCEV = SPtrInfo.StrideSCEV;
assert(StrideSCEV && "Neither StrideVal nor StrideSCEV were set.");
SCEVExpander Expander(*SE, "strided-load-vec");
Stride = Expander.expandCodeFor(StrideSCEV, StrideSCEV->getType(),
&*Builder.GetInsertPoint());
}
Value *NewStride =
Builder.CreateIntCast(Stride, StrideTy, /*isSigned=*/true);
StrideVal = Builder.CreateMul(
NewStride, ConstantInt::getSigned(
StrideTy, (IsReverseOrder ? -1 : 1) *
static_cast<int>(
DL->getTypeAllocSize(ScalarTy))));
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
auto *Inst = Builder.CreateIntrinsicWithoutFolding(
Intrinsic::experimental_vp_strided_load,
{StridedLoadTy, PO->getType(), StrideTy},
{PO, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedLoadEC)),
Builder.getInt32(StridedLoadEC)});
Inst->addParamAttr(
/*ArgNo=*/0,
Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
NewLI = Inst;
} else if (E->State == TreeEntry::BlendedLoadVectorize) {
// Two masked loads from the candidate bases, blended by the vectorized
// per-lane condition operand. E->Scalars stays in natural lane order,
// so the contiguous loads line up with the mask; ReorderIndices, if
// any, is applied below by the shared FinalShuffle.
Value *P0 = cast<LoadInst>(E->Scalars.front())->getPointerOperand();
if (auto *GEP = dyn_cast<GetElementPtrInst>(P0))
P0 = GEP->getPointerOperand();
auto *Sel = cast<SelectInst>(P0);
Align CommonAlignment =
computeBlendedLoadBaseAlignment(E->Scalars, *DL);
Value *BlendMask = vectorizeOperand(E, 0);
// Place the loads at the bundle head, not at the mask, so they are not
// hoisted above an aliasing store.
setInsertPointAfterBundle(E);
Value *NotMask = Builder.CreateNot(BlendMask);
Value *VecPoison = PoisonValue::get(VecTy);
Value *TrueVal = Builder.CreateMaskedLoad(
VecTy, Sel->getTrueValue(), CommonAlignment, BlendMask, VecPoison);
Value *FalseVal = Builder.CreateMaskedLoad(
VecTy, Sel->getFalseValue(), CommonAlignment, NotMask, VecPoison);
NewLI = cast<Instruction>(
Builder.CreateSelect(BlendMask, TrueVal, FalseVal));
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state");
Value *VecPtr = vectorizeOperand(E, 0);
if (isa<FixedVectorType>(ScalarTy)) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
// CreateMaskedGather expects VecTy and VecPtr have same size. We need
// to expand VecPtr if ScalarTy is a vector type.
unsigned ScalarTyNumElements = getNumElements(ScalarTy);
unsigned VecTyNumElements = getNumElements(VecTy);
assert(VecTyNumElements % ScalarTyNumElements == 0 &&
"Cannot expand getelementptr.");
unsigned VF = VecTyNumElements / ScalarTyNumElements;
SmallVector<Constant *> Indices(VecTyNumElements);
transform(seq(VecTyNumElements), Indices.begin(), [=](unsigned I) {
return Builder.getInt64(I % ScalarTyNumElements);
});
VecPtr = Builder.CreateGEP(
toScalarizedTy(VecTy),
Builder.CreateShuffleVector(
VecPtr, createReplicatedMask(ScalarTyNumElements, VF)),
ConstantVector::get(Indices));
}
// Use the minimum alignment of the gathered loads.
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment);
}
// The wide load also reads the memory of the undef and poison lanes, so
// the metadata of the scalar loads does not apply.
const bool HasUndefLanes = any_of(E->Scalars, IsaPred<UndefValue>);
Value *V = (E->State == TreeEntry::CompressVectorize ||
E->State == TreeEntry::BlendedLoadVectorize || HasUndefLanes)
? NewLI
: PropagateIRFlags(NewLI);
if (StridedLoadTy != VecTy)
V = createWidenedStridedCast(Builder, V, VecTy, *DL);
// The memory of the undef and poison lanes may hold poison, but these
// lanes may be used, so they must not be poison.
if (HasUndefLanes)
V = Builder.CreateFreeze(V);
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Store: {
auto *SI = cast<StoreInst>(VL0);
setInsertPointAfterBundle(E);
Value *VecValue = vectorizeOperand(E, 0);
if (VecValue->getType() != VecTy)
VecValue =
Builder.CreateIntCast(VecValue, VecTy, GetOperandSignedness(0));
VecValue = FinalShuffle(VecValue, E);
Value *Ptr = SI->getPointerOperand();
Instruction *ST;
if (E->State == TreeEntry::Vectorize) {
ST = Builder.CreateAlignedStore(VecValue, Ptr, SI->getAlign());
} else if (E->State == TreeEntry::ExpandVectorize) {
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
Align CommonAlignment = computeCommonAlignment<StoreInst>(E->Scalars);
ArrayRef<int> ExpandMask = E->ReuseShuffleIndices;
unsigned VecNumElts = getNumElements(StridedStoreTy);
assert(VecValue->getType() == StridedStoreTy &&
"FinalShuffle must expand the value to the masked store type.");
assert(ExpandMask.size() == VecNumElts &&
"Reuse mask must match the masked store type.");
Value *BasePtr =
cast<StoreInst>(E->Scalars.front())->getPointerOperand();
SmallVector<Constant *> MaskValues(
VecNumElts, ConstantInt::getFalse(VecTy->getContext()));
for (unsigned I : seq<unsigned>(VecNumElts))
if (ExpandMask[I] != PoisonMaskElem)
MaskValues[I] = ConstantInt::getTrue(VecTy->getContext());
ST = Builder.CreateMaskedStore(VecValue, BasePtr, CommonAlignment,
ConstantVector::get(MaskValues));
} else {
assert(E->State == TreeEntry::StridedVectorize &&
"Expected either strided, masked or consecutive stores.");
bool IsReverseOrder = E->isReverse();
if (IsReverseOrder) {
SI = cast<StoreInst>(E->Scalars[E->ReorderIndices.front()]);
Ptr = SI->getPointerOperand();
}
Align CommonAlignment = computeCommonAlignment<StoreInst>(E->Scalars);
Type *StrideTy = DL->getIndexType(SI->getPointerOperandType());
const StridedPtrInfo &SPtrInfo = TreeEntryToStridedPtrInfoMap.at(E);
FixedVectorType *StridedStoreTy = SPtrInfo.Ty;
assert(StridedStoreTy && "Missing StridedPointerInfo for tree entry.");
unsigned StridedStoreEC = getNumElements(StridedStoreTy);
Value *Stride = SPtrInfo.StrideVal;
assert(Stride && "Missing StridedPointerInfo for tree entry.");
Value *StrideVal =
Builder.CreateIntCast(Stride, StrideTy, /*isSigned=*/true);
// vp_strided_store::stride is defined in bytes
StrideVal = Builder.CreateMul(
StrideVal,
ConstantInt::getSigned(
StrideTy, static_cast<int>(DL->getTypeAllocSize(ScalarTy))));
if (StridedStoreTy != VecTy)
VecValue =
createWidenedStridedCast(Builder, VecValue, StridedStoreTy, *DL);
auto *Inst = Builder.CreateIntrinsicWithoutFolding(
Intrinsic::experimental_vp_strided_store,
{StridedStoreTy, Ptr->getType(), StrideTy},
{VecValue, Ptr, StrideVal,
Builder.getAllOnesMask(ElementCount::getFixed(StridedStoreEC)),
Builder.getInt32(StridedStoreEC)});
Inst->addParamAttr(
/*ArgNo=*/1,
Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
ST = Inst;
}
Value *V = PropagateIRFlags(ST);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::GetElementPtr: {
auto *GEP0 = cast<GetElementPtrInst>(VL0);
setInsertPointAfterBundle(E);
Value *Op0 = vectorizeOperand(E, 0);
SmallVector<Value *> OpVecs;
for (int J = 1, N = GEP0->getNumOperands(); J < N; ++J) {
Value *OpVec = vectorizeOperand(E, J);
OpVecs.push_back(OpVec);
}
Value *V = Builder.CreateGEP(GEP0->getSourceElementType(), Op0, OpVecs);
if (Instruction *I = dyn_cast<GetElementPtrInst>(V)) {
SmallVector<Value *> GEPs;
for (Value *V : E->Scalars) {
if (isa<GetElementPtrInst>(V))
GEPs.push_back(V);
}
V = PropagateIRFlags(I);
}
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::Call: {
CallInst *CI = cast<CallInst>(VL0);
setInsertPointAfterBundle(E);
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
SmallVector<Type *> ArgTys = buildIntrinsicArgTypes(
CI, ID, getNumElements(VecTy),
It != MinBWs.end() ? It->second.first : 0, TTI);
auto VecCallCosts =
getVectorCallCosts(CI, VecTy, TTI, TLI, ArgTys, CostKind);
bool UseIntrinsic = ID != Intrinsic::not_intrinsic &&
VecCallCosts.first <= VecCallCosts.second;
Value *ScalarArg = nullptr;
SmallVector<Value *> OpVecs;
SmallVector<Type *, 2> TysForDecl;
// Add return type if intrinsic is overloaded on it.
if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, -1, TTI)) {
ArrayRef<Type *> ContainedTys = getContainedTypes(VecTy);
for (auto [Idx, Ty] : enumerate(ContainedTys)) {
if (isVectorIntrinsicWithStructReturnOverloadAtField(ID, Idx, TTI))
TysForDecl.push_back(Ty);
}
}
auto *CEI = cast<CallInst>(VL0);
for (unsigned I : seq<unsigned>(0, CI->arg_size())) {
// Some intrinsics have scalar arguments. This argument should not be
// vectorized.
if (UseIntrinsic && isVectorIntrinsicWithScalarOpAtArg(ID, I, TTI)) {
ScalarArg = CEI->getArgOperand(I);
// if decided to reduce bitwidth of abs intrinsic, it second argument
// must be set false (do not return poison, if value issigned min).
if (ID == Intrinsic::abs && It != MinBWs.end() &&
It->second.first < DL->getTypeSizeInBits(CEI->getType()))
ScalarArg = Builder.getFalse();
OpVecs.push_back(ScalarArg);
if (isVectorIntrinsicWithOverloadTypeAtArg(ID, I, TTI))
TysForDecl.push_back(ScalarArg->getType());
continue;
}
Value *OpVec = vectorizeOperand(E, I);
ScalarArg = CEI->getArgOperand(I);
if (cast<VectorType>(OpVec->getType())->getElementType() !=
ScalarArg->getType()->getScalarType() &&
It == MinBWs.end()) {
auto *CastTy =
getWidenedType(ScalarArg->getType(), getNumElements(VecTy));
OpVec = Builder.CreateIntCast(OpVec, CastTy, GetOperandSignedness(I));
} else if (It != MinBWs.end()) {
OpVec = Builder.CreateIntCast(OpVec, VecTy, GetOperandSignedness(I));
}
LLVM_DEBUG(dbgs() << "SLP: OpVec[" << I << "]: " << *OpVec << "\n");
OpVecs.push_back(OpVec);
if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, I, TTI))
TysForDecl.push_back(OpVec->getType());
}
Function *CF;
if (!UseIntrinsic) {
VFShape Shape =
VFShape::get(CI->getFunctionType(),
ElementCount::getFixed(getNumElements(VecTy)),
false /*HasGlobalPred*/);
CF = VFDatabase(*CI).getVectorizedFunction(Shape);
} else {
CF = Intrinsic::getOrInsertDeclaration(F->getParent(), ID, TysForDecl);
}
SmallVector<OperandBundleDef, 1> OpBundles;
CI->getOperandBundlesAsDefs(OpBundles);
Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
V = PropagateIRFlags(V);
cast<CallInst>(V)->setCallingConv(CF->getCallingConv());
V = FinalShuffle(V, E);
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case Instruction::ShuffleVector: {
Value *V;
if (Opts.slp_revec && !E->isAltShuffle()) {
setInsertPointAfterBundle(E);
Value *Src = vectorizeOperand(E, 0);
SmallVector<int> ThisMask(calculateShufflevectorMask(E->Scalars));
if (auto *SVSrc = dyn_cast<ShuffleVectorInst>(Src)) {
SmallVector<int> NewMask(ThisMask.size());
transform(ThisMask, NewMask.begin(), [&SVSrc](int Mask) {
return SVSrc->getShuffleMask()[Mask];
});
V = Builder.CreateShuffleVector(SVSrc->getOperand(0),
SVSrc->getOperand(1), NewMask);
} else {
V = Builder.CreateShuffleVector(Src, ThisMask);
}
V = PropagateIRFlags(V);
V = FinalShuffle(V, E);
} else {
assert(E->isAltShuffle() &&
((Instruction::isBinaryOp(E->getOpcode()) &&
Instruction::isBinaryOp(E->getAltOpcode())) ||
(Instruction::isCast(E->getOpcode()) &&
Instruction::isCast(E->getAltOpcode())) ||
(isa<CmpInst>(VL0) && isa<CmpInst>(E->getAltOp()))) &&
"Invalid Shuffle Vector Operand");
// Gather up main and alt scalar ops to propagate IR flags to each
// vector operation and build the lane-select mask.
ValueList OpScalars, AltScalars;
SmallVector<int> Mask;
E->buildAltOpShuffleMask(
[E, this](Instruction *I) {
assert(E->getMatchingMainOpOrAltOp(I) &&
"Unexpected main/alternate opcode");
return isAlternateInstruction(I, E->getMainOp(), E->getAltOp(),
*TLI);
},
Mask, &OpScalars, &AltScalars);
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
transformScalarShuffleIndiciesToVector(VecTy->getNumElements(), Mask);
}
if (E->hasReassocScalars() && Instruction::isBinaryOp(E->getOpcode())) {
// Peeled alternate chains are linearized completely: the operand
// columns fold into one pure main-opcode chain and one pure
// alt-opcode chain, followed by a single lane-select shuffle. Every
// lane keeps one opcode on all levels, so its value is exact in one
// of the chains.
setInsertPointAfterBundle(E);
for (Value *V : E->getReassocScalars()) {
auto *I = cast<Instruction>(V);
(isAlternateInstruction(I, E->getMainOp(), E->getAltOp(), *TLI)
? AltScalars
: OpScalars)
.push_back(I);
}
auto FoldColumns = [&](unsigned Opcode,
const ValueList &FlagScalars) {
Value *R = nullptr;
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
Value *Column = vectorizeOperand(E, Idx);
if (Column->getType() != VecTy)
Column = Builder.CreateIntCast(Column, VecTy,
GetOperandSignedness(Idx));
if (!R) {
R = Column;
continue;
}
R = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(Opcode), R, Column);
PropagateIRFlags(R, Opcode, FlagScalars);
auto *I = dyn_cast<Instruction>(R);
if (!I)
continue;
// Regrouping can invalidate flags even when each original
// step was safe.
I->dropPoisonGeneratingFlags();
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
return R;
};
Value *V0 = FoldColumns(E->getOpcode(), OpScalars);
Value *V1 = FoldColumns(E->getAltOpcode(), AltScalars);
V = Builder.CreateShuffleVector(V0, V1, Mask);
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
Value *LHS = nullptr, *RHS = nullptr;
if (Instruction::isBinaryOp(E->getOpcode()) || isa<CmpInst>(VL0)) {
setInsertPointAfterBundle(E);
LHS = vectorizeOperand(E, 0);
RHS = vectorizeOperand(E, 1);
} else {
setInsertPointAfterBundle(E);
LHS = vectorizeOperand(E, 0);
}
if (LHS && RHS &&
((Instruction::isBinaryOp(E->getOpcode()) &&
(LHS->getType() != VecTy || RHS->getType() != VecTy)) ||
(isa<CmpInst>(VL0) && LHS->getType() != RHS->getType()))) {
assert((It != MinBWs.end() ||
getOperandEntry(E, 0)->State == TreeEntry::NeedToGather ||
getOperandEntry(E, 1)->State == TreeEntry::NeedToGather ||
MinBWs.contains(getOperandEntry(E, 0)) ||
MinBWs.contains(getOperandEntry(E, 1))) &&
"Expected item in MinBWs.");
Type *CastTy = VecTy;
if (isa<CmpInst>(VL0) && LHS->getType() != RHS->getType()) {
if (cast<VectorType>(LHS->getType())
->getElementType()
->getIntegerBitWidth() < cast<VectorType>(RHS->getType())
->getElementType()
->getIntegerBitWidth())
CastTy = RHS->getType();
else
CastTy = LHS->getType();
}
if (LHS->getType() != CastTy)
LHS = Builder.CreateIntCast(LHS, CastTy, GetOperandSignedness(0));
if (RHS->getType() != CastTy)
RHS = Builder.CreateIntCast(RHS, CastTy, GetOperandSignedness(1));
}
Value *V0, *V1;
if (Instruction::isBinaryOp(E->getOpcode())) {
V0 = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS);
V1 = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(E->getAltOpcode()), LHS, RHS);
} else if (auto *CI0 = dyn_cast<CmpInst>(VL0)) {
V0 = Builder.CreateCmp(CI0->getPredicate(), LHS, RHS);
auto *AltCI = cast<CmpInst>(E->getAltOp());
CmpInst::Predicate AltPred = AltCI->getPredicate();
V1 = Builder.CreateCmp(AltPred, LHS, RHS);
} else {
if (LHS->getType()->isIntOrIntVectorTy() && ScalarTy->isIntegerTy()) {
unsigned SrcBWSz = DL->getTypeSizeInBits(
cast<VectorType>(LHS->getType())->getElementType());
unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
if (BWSz <= SrcBWSz) {
if (BWSz < SrcBWSz)
LHS = Builder.CreateIntCast(LHS, VecTy, It->second.first);
assert(LHS->getType() == VecTy &&
"Expected same type as operand.");
LHS = PropagateIRFlags(LHS);
LHS = FinalShuffle(LHS, E);
E->VectorizedValue = LHS;
++NumVectorInstructions;
return LHS;
}
}
V0 = Builder.CreateCast(
static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy);
V1 = Builder.CreateCast(
static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy);
}
// Add V0 and V1 to later analysis to try to find and remove matching
// instruction, if any.
for (Value *V : {V0, V1}) {
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
}
PropagateIRFlags(V0, E->getOpcode(), OpScalars);
PropagateIRFlags(V1, E->getAltOpcode(), AltScalars);
// Create shuffle to take alternate operations from the vector.
V = Builder.CreateShuffleVector(V0, V1, Mask);
if (auto *I = dyn_cast<Instruction>(V)) {
GatherShuffleExtractSeq.insert(I);
CSEBlocks.insert(I->getParent());
}
}
E->VectorizedValue = V;
++NumVectorInstructions;
return V;
}
case TreeEntry::ReducedBitcast:
case TreeEntry::ReducedBitcastBSwap: {
assert(UserIgnoreList && "Expected reduction operations only.");
setInsertPointAfterBundle(E);
TreeEntry *ZExt = getOperandEntry(E, /*Idx=*/0);
ZExt->VectorizedValue = PoisonValue::get(getWidenedType(
ZExt->getMainOp()->getType(), ZExt->getVectorFactor()));
TreeEntry *Const = getOperandEntry(E, /*Idx=*/1);
Const->VectorizedValue = PoisonValue::get(getWidenedType(
Const->Scalars.front()->getType(), Const->getVectorFactor()));
Value *Op = vectorizeOperand(ZExt, 0);
auto *SrcType = IntegerType::get(
Op->getContext(),
DL->getTypeSizeInBits(cast<CastInst>(ZExt->getMainOp())->getSrcTy()) *
E->getVectorFactor());
auto *OrigScalarTy = ScalarTy;
// Set the scalar type properly to avoid casting to the extending type.
ScalarTy = cast<CastInst>(ZExt->getMainOp())->getSrcTy();
Op = FinalShuffle(Op, E);
auto *V = Builder.CreateBitCast(Op, SrcType);
++NumVectorInstructions;
if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwap) {
V = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, V);
++NumVectorInstructions;
}
if (SrcType != OrigScalarTy) {
V = Builder.CreateIntCast(V, OrigScalarTy, /*isSigned=*/false);
++NumVectorInstructions;
}
E->VectorizedValue = V;
return V;
}
case TreeEntry::ReducedBitcastLoads:
case TreeEntry::ReducedBitcastBSwapLoads: {
assert(UserIgnoreList && "Expected reduction operations only.");
TreeEntry *ZExt = getOperandEntry(E, /*Idx=*/0);
TreeEntry *Load = getOperandEntry(ZExt, /*Idx=*/0);
setInsertPointAfterBundle(Load);
ZExt->VectorizedValue = PoisonValue::get(getWidenedType(
ZExt->getMainOp()->getType(), ZExt->getVectorFactor()));
TreeEntry *Const = getOperandEntry(E, /*Idx=*/1);
Const->VectorizedValue = PoisonValue::get(getWidenedType(
Const->Scalars.front()->getType(), Const->getVectorFactor()));
Load->VectorizedValue = PoisonValue::get(getWidenedType(
Load->getMainOp()->getType(), Load->getVectorFactor()));
LoadInst *LI = cast<LoadInst>(Load->getMainOp());
Value *PO = LI->getPointerOperand();
auto *SrcTy = IntegerType::get(
ScalarTy->getContext(),
DL->getTypeSizeInBits(cast<CastInst>(ZExt->getMainOp())->getSrcTy()) *
E->getVectorFactor());
auto *OrigScalarTy = ScalarTy;
ScalarTy = ZExt->getMainOp()->getType();
Value *V = Builder.CreateAlignedLoad(SrcTy, PO, LI->getAlign());
++NumVectorInstructions;
if (ShuffleOrOp == TreeEntry::ReducedBitcastBSwapLoads) {
V = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, V);
++NumVectorInstructions;
}
if (SrcTy != OrigScalarTy) {
V = Builder.CreateIntCast(V, OrigScalarTy, /*isSigned=*/false);
++NumVectorInstructions;
}
E->VectorizedValue = V;
return V;
}
case TreeEntry::ReducedBitPack: {
assert(UserIgnoreList && "Expected reduction operations only.");
setInsertPointAfterBundle(E);
TreeEntry *ShlTE = E->getOpcode() == Instruction::And
? getOperandEntry(E, /*Idx=*/0)
: E;
std::optional<BitPackInfo> Info = computeBitPackLayout(*E);
assert(Info && "Expected bit pack info.");
TreeEntry *LhsTE = getOperandEntry(ShlTE, /*Idx=*/0);
const TreeEntry *SrcTE = getBitPackSource(*LhsTE, *Info);
SmallVector<TreeEntry *> CombinedTEs;
if (ShlTE != E)
CombinedTEs.append({ShlTE, getOperandEntry(E, /*Idx=*/1)});
CombinedTEs.push_back(getOperandEntry(ShlTE, /*Idx=*/1));
TreeEntry *OpTE = LhsTE;
for (; OpTE != SrcTE; OpTE = getOperandEntry(OpTE, /*Idx=*/0)) {
CombinedTEs.push_back(OpTE);
for (unsigned Idx : seq<unsigned>(1, OpTE->getNumOperands()))
CombinedTEs.push_back(getOperandEntry(OpTE, Idx));
}
for (TreeEntry *TE : CombinedTEs)
TE->VectorizedValue = PoisonValue::get(getWidenedType(
TE->Scalars.front()->getType(), TE->getVectorFactor()));
Value *X = vectorizeTree(OpTE);
unsigned ShiftWidth;
Type *DstScalarTy =
SrcTE == LhsTE ? X->getType()->getScalarType() : ScalarTy;
[[maybe_unused]] InstructionCost PackCost = getBitPackCost(
*TTI, cast<FixedVectorType>(X->getType()), DstScalarTy, *Info,
getZExtSrcWidth(*SrcTE), getCastContextHint(*SrcTE), CostKind, TLI,
E->getMainOp(), ShiftWidth);
assert(PackCost.isValid() && "Expected valid cost.");
unsigned NumInsts;
Value *V =
buildBitPack(Builder, X, *Info, DstScalarTy, ShiftWidth, NumInsts);
NumVectorInstructions += NumInsts;
E->VectorizedValue = V;
return V;
}
case TreeEntry::ReducedCmpBitcast: {
assert(UserIgnoreList && "Expected reduction operations only.");
setInsertPointAfterBundle(E);
TreeEntry *Op1TE = getOperandEntry(E, /*Idx=*/1);
TreeEntry *Op2TE = getOperandEntry(E, /*Idx=*/2);
Op1TE->VectorizedValue =
PoisonValue::get(getWidenedType(ScalarTy, Op1TE->getVectorFactor()));
Op2TE->VectorizedValue =
PoisonValue::get(getWidenedType(ScalarTy, Op2TE->getVectorFactor()));
Value *Cmp = vectorizeOperand(E, /*NodeIdx=*/0);
// Set the scalar type properly to avoid casting to the extending type.
auto *DstTy =
IntegerType::getIntNTy(ScalarTy->getContext(), E->getVectorFactor());
auto *V = Builder.CreateBitCast(Cmp, DstTy);
++NumVectorInstructions;
if (DstTy != ScalarTy) {
V = Builder.CreateIntCast(V, ScalarTy, /*isSigned=*/false);
++NumVectorInstructions;
}
E->VectorizedValue = V;
return V;
}
default:
llvm_unreachable("unknown inst");
}
return nullptr;
}
bool BoUpSLP::isRuntimeCheckableAliasPair(Instruction *Inst1,
Instruction *Inst2) {
// Only simple (non-volatile, non-atomic) accesses may be reordered once
// aliasing is ruled out at runtime; volatile/atomic ordering must be kept.
if (!isSimple(Inst1) || !isSimple(Inst2))
return false;
Value *Ptr1 = getLoadStorePointerOperand(Inst1);
Value *Ptr2 = getLoadStorePointerOperand(Inst2);
// Only simple load/store accesses have a single pointer operand whose
// accessed range can be bounded and compared at runtime.
if (!Ptr1 || !Ptr2)
return false;
if (Ptr1->getType()->getPointerAddressSpace() !=
Ptr2->getType()->getPointerAddressSpace())
return false;
const Value *Base1 = getUnderlyingObject(Ptr1);
const Value *Base2 = getUnderlyingObject(Ptr2);
// A runtime range check is only meaningful between two distinct, identifiable
// objects: two accesses to the same base differ by a compile-time offset, so
// they are resolved statically and a base-range check would fold to a
// constant predicate rather than a useful runtime guard.
if (Base1 == Base2 || isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
return false;
return true;
}
bool BoUpSLP::isCoveredByExistingVersionCheck(BasicBlock *BB,
Instruction *Inst1,
Instruction *Inst2) const {
const auto It = VersionedBlockCheckedPairs.find(BB);
if (It == VersionedBlockCheckedPairs.end())
return false;
Value *Ptr1 = getLoadStorePointerOperand(Inst1);
Value *Ptr2 = getLoadStorePointerOperand(Inst2);
if (!Ptr1 || !Ptr2)
return false;
const Value *Base1 = getUnderlyingObject(Ptr1);
const Value *Base2 = getUnderlyingObject(Ptr2);
if (Base1 == Base2)
return false;
// The recorded pairs keep their discovery order (see
// recordRuntimeAliasCheck), so accept either orientation.
return It->second.contains({Base1, Base2}) ||
It->second.contains({Base2, Base1});
}
/// Returns true if \p BB's body already contains vector instructions, e.g.
/// from an earlier SLP vectorization in the same pass.
static bool blockBodyHasVectorInstructions(const VectorizeOptions &Opts,
BasicBlock *BB) {
// A vector-producing instruction (vector load, binop, shuffle, etc.) has a
// vector result type.
return any_of(*BB, [&](Instruction &I) {
return !isa<PHINode>(&I) && !I.isTerminator() &&
getValueType(&I, Opts.slp_revec)->isVectorTy();
});
}
void BoUpSLP::captureRuntimeCheckBodySnapshot() {
RTOrigBodyOrder.clear();
if (!TryRuntimeAliasChecks || !RTChecks.BB)
return;
BasicBlock *BB = RTChecks.BB;
// Allocas are not duplicated, keep them in the header block.
for (Instruction &I : *BB)
if (!isa<PHINode, AllocaInst>(&I) && !I.isTerminator())
RTOrigBodyOrder.push_back(&I);
}
bool BoUpSLP::recordRuntimeAliasCheck(BasicBlock *BB, Instruction *Inst1,
Instruction *Inst2) {
Value *Ptr1 = getLoadStorePointerOperand(Inst1);
Value *Ptr2 = getLoadStorePointerOperand(Inst2);
if (!Ptr1 || !Ptr2)
return false;
const Value *Base1 = getUnderlyingObject(Ptr1);
const Value *Base2 = getUnderlyingObject(Ptr2);
if (Base1 == Base2)
return false;
// Only a single block can be versioned per attempt.
if (RTChecks.BB && RTChecks.BB != BB)
return false;
// A base-object pair is unordered: (Base1, Base2) and (Base2, Base1) are the
// same check. Keep it in discovery (program) order and drop the reverse as a
// duplicate, rather than ordering by pointer value, which would make the
// emitted checks depend on allocation addresses.
auto Pair = std::make_pair(Base1, Base2);
bool Present = RTChecks.BasePairs.contains(Pair) ||
RTChecks.BasePairs.contains({Base2, Base1});
// After the checks have been validated and bounded, do not introduce new
// pairs.
if (RTChecksFinalized)
return Present;
RTChecks.BB = BB;
if (!Present)
RTChecks.BasePairs.insert(Pair);
return true;
}
bool BoUpSLP::canVersionBlockForRuntimeChecks(BasicBlock *BB) const {
assert(BB && "Expected a block to version for runtime checks.");
// Versioning duplicates the block body, increasing code size on the guarded
// path; do not version when the function is optimized for size (-Os/-Oz).
if (F->hasOptSize())
return false;
if (!DT || !LI)
return false;
if (!DT->isReachableFromEntry(BB))
return false;
// Never version a scalar fallback block: it is the safe, original-order copy
// taken when aliasing is detected and must stay scalar.
if (ScalarFallbackBlocks.contains(BB))
return false;
// The scalar fallback must be a faithful copy of the original scalar body.
// Reject blocks that were already partially vectorized earlier in this pass.
if (blockBodyHasVectorInstructions(Opts, BB))
return false;
// Versioning duplicates the original block body into a scalar fallback.
// Calls that cannot be duplicated or whose semantics depend on the call being
// immediately followed by a return cannot be cloned into the diamond.
if (BB->getTerminatingMustTailCall())
return false;
if (any_of(*BB, [](Instruction &I) {
auto *CB = dyn_cast<CallBase>(&I);
return CB && (CB->cannotDuplicate() || CB->isConvergent());
}))
return false;
// Only static allocas at the start of the entry block can stay in the header
// block, any other alloca would have to be duplicated with the body.
if (any_of(make_range(BB->getFirstNonPHIOrAlloca(), BB->end()),
IsaPred<AllocaInst>))
return false;
return true;
}
bool BoUpSLP::canVersionForRuntimeChecks() {
if (RTChecks.BasePairs.empty() || !RTChecks.BB)
return false;
if (RTChecks.BasePairs.size() > Opts.slp_max_runtime_alias_checks)
return false;
BasicBlock *BB = RTChecks.BB;
// Block-level preconditions (loop/optsize/duplicability/...) are the same
// ones used to gate the optimistic retry, so reuse them here.
if (!canVersionBlockForRuntimeChecks(BB))
return false;
// The scalar fallback is a clone of the body. Operands defined inside the
// body are remapped to their clones, but operands defined outside the body
// (header PHIs, dominating definitions) are reused as-is by the clone. If
// such an outside operand is itself vectorized by this tree, vectorizeTree()
// will delete its scalar, leaving the clone with a dangling, out-of-tree use.
// Versioning cannot model that, so bail out.
for (Instruction &I : make_filter_range(*BB, [](Instruction &I) {
return !isa<PHINode>(&I) && !I.isTerminator();
})) {
for (Instruction *OpI : make_isa_range<Instruction>(I.operands())) {
bool DefinedInBody =
OpI->getParent() == BB && !isa<PHINode>(OpI) && !OpI->isTerminator();
if (!DefinedInBody && isVectorized(OpI))
return false;
}
}
SmallPtrSet<const Value *, 8> Bases;
for (const auto &P : RTChecks.BasePairs) {
Bases.insert(P.first);
Bases.insert(P.second);
}
// Every base object must be available in the (PHI-only) header where the
// guard branch is emitted, i.e. be one of its PHIs or dominate it.
if (any_of(make_isa_range<Instruction>(Bases), [&](const Instruction *I) {
return I->getParent() == BB ? !isa<PHINode>(I) : !DT->dominates(I, BB);
}))
return false;
// Compute the accessed address range [Low, High) for every involved base.
RTChecks.Bounds.clear();
SmallMapVector<Value *, std::pair<const SCEV *, const SCEV *>, 4> OffBounds;
unsigned NumMemInsts = 0;
for (Instruction &I : make_filter_range(
*BB, [](Instruction &I) { return I.mayReadOrWriteMemory(); })) {
// The dependency scan truncates beyond MaxMemDepDistance. Only version
// blocks small enough that the scan is never truncated, so every
// conflicting base pair is guaranteed to be recorded.
if (++NumMemInsts > MaxMemDepDistance)
return false;
Value *Ptr = getLoadStorePointerOperand(&I);
// A non-simple memory access (e.g. a call) keeps its dependencies, so its
// relative order with kept-dependency partners is preserved inside the
// duplicated body and it does not need a bound.
if (!Ptr)
continue;
Value *Base = const_cast<Value *>(getUnderlyingObject(Ptr));
if (!Bases.contains(Base))
continue;
TypeSize TS = DL->getTypeStoreSize(getLoadStoreType(&I));
if (TS.isScalable())
return false;
Type *IntTy = DL->getAddressType(Base->getType());
// All bounds are computed and compared in a single address type, so an
// access that reaches its base object through an addrspacecast between
// differently-sized address spaces cannot be bounded and the block
// cannot be versioned.
if (DL->getAddressType(Ptr->getType()) != IntTy)
return false;
const SCEV *PtrSC = SE->getPtrToAddrExpr(SE->getSCEV(Ptr));
const SCEV *BaseSC = SE->getPtrToAddrExpr(SE->getSCEV(Base));
if (isa<SCEVCouldNotCompute>(PtrSC) || isa<SCEVCouldNotCompute>(BaseSC))
return false;
// Byte offset of this access from its base. Require a constant so the bound
// stays base-plus-constant, and non-negative so the unsigned [Low, High)
// range below actually covers the access.
// TODO: Support variable offsets (e.g. base + %idx) which would need
// runtime umin/umax bounds and a proof that the offset range cannot wrap,
// plus a richer cost model.
const auto *Off = dyn_cast<SCEVConstant>(SE->getMinusSCEV(PtrSC, BaseSC));
if (!Off || Off->getAPInt().isNegative())
return false;
const SCEV *EndOff =
SE->getAddExpr(Off, SE->getConstant(IntTy, TS.getFixedValue()));
auto [It, Inserted] = OffBounds.try_emplace(Base, Off, EndOff);
if (!Inserted) {
// Constant operands, so these fold to a constant min/max offset rather
// than emitting a runtime umin/umax.
It->second.first = SE->getUMinExpr(It->second.first, Off);
It->second.second = SE->getUMaxExpr(It->second.second, EndOff);
}
}
// Every involved base must contribute at least one bounded access.
if (OffBounds.size() != Bases.size())
return false;
// Materialize the absolute [Low, High) = base + [minOff, maxEndOff). The
// offsets are constants, so each bound is a base-plus-constant expression
// that expands to a single add off the base address (no runtime umin/umax).
// The base is used as an opaque value, so the bounds only reference the base
// itself, available at the guard, and a loop-variant base (e.g. a header PHI)
// is reused rather than recomputed from its recurrence.
for (const auto &[Base, Off] : OffBounds) {
const SCEV *BaseSC = SE->getPtrToAddrExpr(SE->getUnknown(Base));
RTChecks.Bounds.try_emplace(Base, SE->getAddExpr(BaseSC, Off.first),
SE->getAddExpr(BaseSC, Off.second));
}
// SSA values defined in the body and used outside of it are merged from both
// paths in the continuation block, so they must stay scalar. Uses by the
// terminator (moved into the continuation block) and by the block's PHIs
// (via the loop latch) count as outside.
for (Instruction &I : make_filter_range(*BB, [&](Instruction &I) {
return !isa<PHINode>(&I) && !I.isTerminator() && isVectorized(&I);
})) {
if (any_of(I.users(), [&](User *U) {
auto *UI = dyn_cast<Instruction>(U);
return !UI || UI->getParent() != BB || isa<PHINode>(UI) ||
UI->isTerminator();
}))
return false;
}
// The runtime checks and the guard branch run on both the vector and the
// scalar fallback path. When aliasing is detected at runtime the scalar code
// executes unchanged plus the check overhead, so bound the check cost
// relative to the guarded scalar region to avoid pessimizing that path too
// much. The scalar region cost is the cost of the (current, still scalar)
// block body; no IR is emitted here.
InstructionCost ScalarCost = 0;
for (Instruction &I : make_filter_range(*BB, [](Instruction &I) {
return !isa<PHINode>(&I) && !I.isTerminator();
})) {
// The backend fuses the fmul into its fadd/fsub user, so the pair costs as
// the fmuladd.
auto [U, FMACost] = getFMulFusingUser(&I, /*InTree=*/false);
ScalarCost += U && U->getParent() == BB
? FMACost - TTI->getInstructionCost(U, CostKind)
: TTI->getInstructionCost(&I, CostKind);
}
InstructionCost CheckCost = getRuntimeChecksCost();
if (!ScalarCost.isValid() || !CheckCost.isValid())
return false;
const Loop *L = LI->getLoopFor(BB);
unsigned MaxPercent =
L && getBooleanLoopAttribute(L, "llvm.loop.isvectorized")
? Opts.slp_vec_loop_rt_checks_cost_percent
: Opts.slp_runtime_alias_checks_max_scalar_cost_percent;
if (CheckCost * 100 > ScalarCost * MaxPercent)
return false;
// Freeze the check set so the final scheduleBlock() keeps the same dropped
// dependencies the bounds were just computed for.
RTChecksFinalized = true;
return true;
}
InstructionCost BoUpSLP::getRuntimeChecksCost() const {
if (RTChecks.BasePairs.empty())
return 0;
LLVMContext &Ctx = F->getContext();
Type *IntTy = DL->getIntPtrType(Ctx);
Type *I1Ty = Type::getInt1Ty(Ctx);
unsigned NumBases = RTChecks.Bounds.size();
unsigned NumPairs = RTChecks.BasePairs.size();
InstructionCost Cost = 0;
// Per base: a [Low, High) address pair. canVersionForRuntimeChecks() folds
// each bound to a base-plus-constant offset, so it expands to two integer
// adds (one per bound) with no runtime umin/umax reduction.
Cost += TTI->getArithmeticInstrCost(Instruction::Add, IntTy, CostKind) *
(2 * NumBases);
// Two integer compares and one logical and per checked pair.
InstructionCost CmpCost = TTI->getCmpSelInstrCost(
Instruction::ICmp, IntTy, I1Ty, CmpInst::ICMP_ULT, CostKind);
InstructionCost AndCost =
TTI->getArithmeticInstrCost(Instruction::And, I1Ty, CostKind);
Cost += (CmpCost * 2 + AndCost) * NumPairs;
// Or-reduction of the per-pair conflicts.
if (NumPairs > 1)
Cost += TTI->getArithmeticInstrCost(Instruction::Or, I1Ty, CostKind) *
(NumPairs - 1);
// The guard branch.
Cost += TTI->getCFInstrCost(Instruction::CondBr, CostKind);
return Cost;
}
Value *BoUpSLP::emitRuntimeAliasCheck(IRBuilderBase &Builder,
SCEVExpander &Exp) {
// Materialize the [Low, High) address bounds for every involved base from
// their SCEVs at the builder's insertion point.
SmallDenseMap<const Value *, std::pair<Value *, Value *>, 4> Range;
for (const auto &[Base, Bound] : RTChecks.Bounds) {
Type *IntTy = DL->getAddressType(Base->getType());
Value *Low =
Exp.expandCodeFor(Bound.first, IntTy, Builder.GetInsertPoint());
Value *High =
Exp.expandCodeFor(Bound.second, IntTy, Builder.GetInsertPoint());
Range[Base] = {Low, High};
}
Value *Cond = nullptr;
for (const auto &P : RTChecks.BasePairs) {
std::pair<Value *, Value *> A = Range.lookup(P.first);
std::pair<Value *, Value *> B = Range.lookup(P.second);
assert(A.first && A.second && B.first && B.second &&
"Missing bounds for a checked base object");
// The objects overlap iff (A.Low < B.High) && (B.Low < A.High).
Value *C0 = Builder.CreateICmpULT(A.first, B.second, "rt.bound0");
Value *C1 = Builder.CreateICmpULT(B.first, A.second, "rt.bound1");
Value *Conflict = Builder.CreateAnd(C0, C1, "rt.conflict");
Cond =
Cond ? Builder.CreateOr(Cond, Conflict, "rt.conflict.all") : Conflict;
}
return Builder.CreateFreeze(Cond, "rt.guard");
}
void BoUpSLP::versionBlocksForRuntimeChecks() {
// Only version when this is the optimistic attempt that collected the checks.
if (!TryRuntimeAliasChecks || RTChecks.BasePairs.empty() || !RTChecks.BB ||
RTOrigBodyOrder.empty())
return;
BasicBlock *BB = RTChecks.BB;
Function *Fn = BB->getParent();
LLVMContext &Ctx = BB->getContext();
Instruction *Term = BB->getTerminator();
// Build the diamond:
// BB (header: PHIs + guard) -> {VecBB, ScalarBB} -> Tail (terminator).
BasicBlock *VecBB = BasicBlock::Create(Ctx, BB->getName() + ".rtvec", Fn);
BasicBlock *ScalarBB =
BasicBlock::Create(Ctx, BB->getName() + ".rtscalar", Fn);
BasicBlock *Tail = BasicBlock::Create(Ctx, BB->getName() + ".rtcont", Fn);
// Emit the runtime alias check while the block is still fully connected:
// LCSSA-preserving SCEV expansion rewrites out-of-loop uses of
// loop-defined bases to poison in predecessor-less blocks. Expand at the
// first insertion point, where no body instruction dominates, so the check
// cannot reuse scalars that are moved into the vector block and deleted.
IRBuilder<> ChkBuilder(BB->getFirstInsertionPt());
ChkBuilder.SetCurrentDebugLocation(Term->getDebugLoc());
SCEVExpander Exp(*SE, "slp.rtcheck");
Value *Cond = emitRuntimeAliasCheck(ChkBuilder, Exp);
// Clone the body into the scalar fallback block, in the original program
// order (RTOrigBodyOrder).
SmallDenseMap<Value *, Value *, 16> VMap;
for (Instruction *I : RTOrigBodyOrder) {
assert(I->getParent() == BB &&
"RTOrigBodyOrder entry no longer in the versioned block");
Instruction *C = I->clone();
if (I->hasName())
C->setName(I->getName() + ".scalar");
C->insertInto(ScalarBB, ScalarBB->end());
VMap[I] = C;
}
// Remap the clones to use the cloned definitions; references to values
// defined outside the body (header or function arguments) are unchanged.
for (Instruction &I : *ScalarBB)
for (Use &U : I.operands())
if (Value *V = VMap.lookup(U.get()))
U.set(V);
// Move the original body into the vector block in the current (scheduled)
// order, keeping the emitted check instructions in the header block.
SmallPtrSet<Instruction *, 16> BodyInsts(llvm::from_range, RTOrigBodyOrder);
for (Instruction &I : make_early_inc_range(*BB))
if (BodyInsts.contains(&I))
I.moveBefore(*VecBB, VecBB->end());
// Move the original terminator into the continuation block and replace it
// with the guard branch.
Term->removeFromParent();
Term->insertInto(Tail, Tail->end());
UncondBrInst::Create(Tail, VecBB);
UncondBrInst::Create(Tail, ScalarBB);
ChkBuilder.SetInsertPoint(BB);
CondBrInst *Guard = ChkBuilder.CreateCondBr(Cond, ScalarBB, VecBB);
// The scalar fallback is taken only when the runtime alias check detects a
// conflict, which is expected to be rare, so bias the guard towards the
// vector path.
if (!ProfcheckDisableMetadataFixes)
setBranchWeights(
*Guard,
{MDBuilder::kUnlikelyBranchWeight, MDBuilder::kLikelyBranchWeight},
/*IsExpected=*/false);
// The continuation block is the new predecessor of the original successors.
for (BasicBlock *Succ : successors(Term))
Succ->replacePhiUsesWith(BB, Tail);
// Merge the body values used outside of the versioned region (by the
// terminator, across the loop latch or in other blocks) from both paths. The
// SCEVs cached for the outside users refer to the body values, which no
// longer dominate them.
for (Instruction *I : RTOrigBodyOrder) {
if (!I->isUsedOutsideOfBlock(VecBB))
continue;
SE->forgetValue(I);
PHINode *Merge = PHINode::Create(I->getType(), 2, I->getName() + ".rtmerge",
Tail->getFirstNonPHIIt());
I->replaceUsesOutsideBlock(Merge, VecBB);
Merge->addIncoming(I, VecBB);
Merge->addIncoming(VMap.lookup(I), ScalarBB);
}
// Keep the dominator tree valid for the remainder of the run.
if (DT) {
DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Eager);
SmallVector<DominatorTree::UpdateType> Updates = {
{DominatorTree::Insert, BB, VecBB},
{DominatorTree::Insert, BB, ScalarBB},
{DominatorTree::Insert, VecBB, Tail},
{DominatorTree::Insert, ScalarBB, Tail}};
// Term (e.g. a switch) may have multiple edges to the same successor;
// dedupe so each distinct successor gets exactly one Insert/Delete pair.
SmallPtrSet<BasicBlock *, 4> UniqueSuccs;
for (BasicBlock *Succ : successors(Term)) {
if (!UniqueSuccs.insert(Succ).second)
continue;
Updates.push_back({DominatorTree::Insert, Tail, Succ});
Updates.push_back({DominatorTree::Delete, BB, Succ});
}
DTU.applyUpdates(Updates);
// Refresh DFS numbers: getLastInstructionInBundle() may use them to order
// instructions across blocks for nodes emitted after versioning.
DT->updateDFSNumbers();
}
// Keep the loop info valid for the remainder of the run: the new blocks
// belong to the loop of the versioned block.
if (Loop *L = LI->getLoopFor(BB))
for (BasicBlock *NewBB : {VecBB, ScalarBB, Tail})
L->addBasicBlockToLoop(NewBB, *LI);
// Record the checked base pairs for the fast-path block so that subsequent
// vectorization there can reuse this guard instead of versioning again.
SmallDenseSet<std::pair<const Value *, const Value *>, 4> &Covered =
VersionedBlockCheckedPairs[VecBB];
Covered.insert_range(RTChecks.BasePairs);
// The scalar fallback must never be vectorized or versioned again.
// TODO: check that scalar fallback does not miss vectorization opportunity
// without runtime checks.
ScalarFallbackBlocks.insert(ScalarBB);
// The checks have been emitted; clear so any later vectorizeTree() in the
// same attempt does not version again.
RTChecks.clear();
RTChecksFinalized = false;
RTOrigBodyOrder.clear();
// The CFG was modified; CFG analyses can no longer be preserved.
CFGChanged = true;
}
Value *BoUpSLP::vectorizeTree() {
ExtraValueToDebugLocsMap ExternallyUsedValues;
return vectorizeTree(ExternallyUsedValues);
}
Value *
BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues,
Instruction *ReductionRoot,
ArrayRef<ReductionVectorPart> VectorValuesAndScales) {
// Clean Entry-to-LastInstruction table. It can be affected after scheduling,
// need to rebuild it.
EntryToLastInstruction.clear();
// All blocks must be scheduled before any instructions are inserted.
for (auto &BSIter : BlocksSchedules)
scheduleBlock(*this, BSIter.second.get());
// Cache last instructions for the nodes to avoid side effects, which may
// appear during vectorization, like extra uses, etc.
for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
// Need to generate insertion point for loads nodes of the bitcast/bswap
// ops.
if (TE->isGather() || DeletedNodes.contains(TE.get()) ||
TE->State == TreeEntry::SplitVectorize ||
(TE->State == TreeEntry::CombinedVectorize &&
(TE->CombinedOp == TreeEntry::ReducedBitcast ||
TE->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
TE->CombinedOp == TreeEntry::ReducedBitPack ||
((TE->CombinedOp == TreeEntry::ReducedBitcastLoads ||
TE->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
TE->CombinedOp == TreeEntry::ReducedCmpBitcast) &&
(!TE->hasState() || TE->getOpcode() != Instruction::Load)))))
continue;
(void)getLastInstructionInBundle(TE.get());
}
// If this tree was scheduled by dropping may-alias memory dependencies in
// favor of runtime alias checks, materialize the checks and version the
// affected block before emitting the vector code. Scheduling has already
// run, so moving the body into the fast-path block is safe, and the
// dominator tree is updated for the rest of the run.
versionBlocksForRuntimeChecks();
if (ReductionRoot)
Builder.SetInsertPoint(ReductionRoot->getIterator());
else
Builder.SetInsertPoint(F->getEntryBlock().begin());
// Vectorize gather operands of the nodes with the external uses only.
SmallVector<std::pair<TreeEntry *, Instruction *>> GatherEntries;
// Multiple gather TEs may share the same UserTE - cache the per-UserTE
// all_of-isUsedOutsideBlock result to avoid re-walking each scalar's
// user list.
SmallDenseMap<const TreeEntry *, bool> UserTEScalarsUsedOutsideBlockCache;
for (const std::unique_ptr<TreeEntry> &TE : make_filter_range(
VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return !DeletedNodes.contains(TE.get());
})) {
if (TE->isGather() && !TE->VectorizedValue && TE->UserTreeIndex.UserTE &&
TE->UserTreeIndex.UserTE->hasState() &&
TE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
(TE->UserTreeIndex.UserTE->getOpcode() != Instruction::PHI ||
TE->UserTreeIndex.UserTE->isAltShuffle()) &&
!TE->UserTreeIndex.UserTE->hasCopyableElements()) {
const TreeEntry *UserTE = TE->UserTreeIndex.UserTE;
auto [It, Inserted] =
UserTEScalarsUsedOutsideBlockCache.try_emplace(UserTE);
if (Inserted)
It->second = all_of(UserTE->Scalars,
[](Value *V) { return isUsedOutsideBlock(V); });
if (!It->second)
continue;
Instruction &LastInst = getLastInstructionInBundle(UserTE);
GatherEntries.emplace_back(TE.get(), &LastInst);
}
}
for (auto &Entry : GatherEntries) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Builder.SetInsertPoint(Entry.second);
Builder.SetCurrentDebugLocation(Entry.second->getDebugLoc());
(void)vectorizeTree(Entry.first);
}
// Emit the subtrees built for the splat gather nodes' unique scalars, so
// the splat gathers can be emitted as their broadcasts. They go before the
// gathered loads, which skip entries that already have a vector value.
for (TreeEntry *TE :
make_filter_range(SplatGatheredScalarsRoots, [&](TreeEntry *TE) {
return !DeletedNodes.contains(TE) && !TE->VectorizedValue;
}))
(void)vectorizeTree(TE);
// Emit gathered loads first to emit better code for the users of those
// gathered loads.
for (const std::unique_ptr<TreeEntry> &TE : make_filter_range(
VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return !DeletedNodes.contains(TE.get());
})) {
if (GatheredLoadsEntriesFirst.has_value() &&
TE->Idx >= *GatheredLoadsEntriesFirst && !TE->VectorizedValue &&
(!TE->isGather() || TE->UserTreeIndex)) {
assert((TE->UserTreeIndex ||
(TE->getOpcode() == Instruction::Load && !TE->isGather())) &&
"Expected gathered load node.");
(void)vectorizeTree(TE.get());
}
}
(void)vectorizeTree(&getRootNode());
// Run through the list of postponed gathers and emit them, replacing the temp
// emitted allocas with actual vector instructions.
ArrayRef<const TreeEntry *> PostponedNodes = PostponedGathers.getArrayRef();
DenseMap<Value *, SmallVector<TreeEntry *>> PostponedValues;
for (const TreeEntry *E : PostponedNodes) {
auto *TE = const_cast<TreeEntry *>(E);
auto *PrevVec = cast<Instruction>(TE->VectorizedValue);
TE->VectorizedValue = nullptr;
auto *UserI = cast<Instruction>(TE->UserTreeIndex.UserTE->VectorizedValue);
// If user is a PHI node, its vector code have to be inserted right before
// block terminator. Since the node was delayed, there were some unresolved
// dependencies at the moment when stab instruction was emitted. In a case
// when any of these dependencies turn out an operand of another PHI, coming
// from this same block, position of a stab instruction will become invalid.
// The is because source vector that supposed to feed this gather node was
// inserted at the end of the block [after stab instruction]. So we need
// to adjust insertion point again to the end of block.
if (isa<PHINode>(UserI) ||
(TE->UserTreeIndex.UserTE->hasState() &&
TE->UserTreeIndex.UserTE->State != TreeEntry::SplitVectorize &&
TE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI)) {
// Insert before all users.
Instruction *InsertPt = PrevVec->getParent()->getTerminator();
for (User *U : PrevVec->users()) {
if (U == UserI)
continue;
auto *UI = dyn_cast<Instruction>(U);
if (!UI || isa<PHINode>(UI) || UI->getParent() != InsertPt->getParent())
continue;
if (UI->comesBefore(InsertPt))
InsertPt = UI;
}
Builder.SetInsertPoint(InsertPt);
} else {
Builder.SetInsertPoint(PrevVec);
}
Builder.SetCurrentDebugLocation(UserI->getDebugLoc());
Value *Vec = vectorizeTree(TE);
if (auto *VecI = dyn_cast<Instruction>(Vec);
VecI && VecI->getParent() == Builder.GetInsertBlock() &&
Builder.GetInsertPoint()->comesBefore(VecI))
VecI->moveBeforePreserving(*Builder.GetInsertBlock(),
Builder.GetInsertPoint());
if (Vec->getType() != PrevVec->getType()) {
assert(Vec->getType()->isIntOrIntVectorTy() &&
PrevVec->getType()->isIntOrIntVectorTy() &&
"Expected integer vector types only.");
std::optional<bool> IsSigned;
for (Value *V : TE->Scalars) {
if (isVectorized(V)) {
for (const TreeEntry *MNTE : getTreeEntries(V)) {
auto It = MinBWs.find(MNTE);
if (It != MinBWs.end()) {
IsSigned = IsSigned.value_or(false) || It->second.second;
if (*IsSigned)
break;
}
}
if (IsSigned.value_or(false))
break;
// Scan through gather nodes.
for (const TreeEntry *BVE : ValueToGatherNodes.lookup(V)) {
auto It = MinBWs.find(BVE);
if (It != MinBWs.end()) {
IsSigned = IsSigned.value_or(false) || It->second.second;
if (*IsSigned)
break;
}
}
if (IsSigned.value_or(false))
break;
if (auto *EE = dyn_cast<ExtractElementInst>(V)) {
IsSigned =
IsSigned.value_or(false) ||
!isKnownNonNegative(EE->getVectorOperand(), SimplifyQuery(*DL));
continue;
}
if (IsSigned.value_or(false))
break;
}
}
if (IsSigned.value_or(false)) {
// Final attempt - check user node.
auto It = MinBWs.find(TE->UserTreeIndex.UserTE);
if (It != MinBWs.end())
IsSigned = It->second.second;
}
assert(IsSigned &&
"Expected user node or perfect diamond match in MinBWs.");
Vec = Builder.CreateIntCast(Vec, PrevVec->getType(), *IsSigned);
}
PrevVec->replaceAllUsesWith(Vec);
PostponedValues.try_emplace(Vec).first->second.push_back(TE);
// Replace the stub vector node, if it was used before for one of the
// buildvector nodes already.
auto It = PostponedValues.find(PrevVec);
if (It != PostponedValues.end()) {
for (TreeEntry *VTE : It->getSecond())
VTE->VectorizedValue = Vec;
}
eraseInstruction(PrevVec);
}
LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()
<< " values .\n");
SmallVector<ShuffledInsertData<Value *>> ShuffledInserts;
// Maps vector instruction to original insertelement instruction
DenseMap<Value *, InsertElementInst *> VectorToInsertElement;
// Maps extract Scalar (plus the struct-field index path, when extracting
// from a struct of vectors) to the corresponding extractelement instruction
// in the basic block. Only one extractelement per block should be emitted.
// The index path is stored in an owning SmallVector so the key remains
// valid after the per-lane ExtractValueInst (whose Indices buffer it was
// copied from) is erased later in this loop.
SmallDenseMap<std::pair<Value *, SmallVector<unsigned, 1>>,
DenseMap<BasicBlock *, std::pair<Value *, Value *>>>
ScalarToEEs;
// Maps (struct-of-vectors Vec, field-index path) to the corresponding
// per-block extractvalue, so different external lanes that need the same
// struct field of the same vectorized call share a single extractvalue.
SmallDenseMap<std::pair<Value *, SmallVector<unsigned, 1>>,
DenseMap<BasicBlock *, Value *>>
StructFieldExtracts;
SmallDenseSet<Value *, 4> UsedInserts;
DenseMap<std::pair<Value *, Type *>, Value *> VectorCasts;
SmallDenseSet<Value *, 4> ScalarsWithNullptrUser;
SmallDenseSet<ExtractElementInst *, 4> IgnoredExtracts;
// Extract all of the elements with the external uses.
for (const auto &ExternalUse : ExternalUses) {
Value *Scalar = ExternalUse.Scalar;
llvm::User *User = ExternalUse.User;
// Skip users that we already RAUW. This happens when one instruction
// has multiple uses of the same value.
if (User && !is_contained(Scalar->users(), User))
continue;
const TreeEntry *E = &ExternalUse.E;
assert(E && "Invalid scalar");
assert(!E->isGather() && "Extracting from a gather list");
// Non-instruction pointers are not deleted, just skip them.
if (E->getOpcode() == Instruction::GetElementPtr &&
!isa<GetElementPtrInst>(Scalar))
continue;
Value *Vec = E->VectorizedValue;
assert(Vec && "Can't find vectorizable value");
auto ExtractAndExtendIfNeeded = [&](Value *Vec) {
if (isa<InsertValueInst>(Scalar))
return Vec;
if (Scalar->getType() != Vec->getType()) {
Value *Ex = nullptr;
Value *ExV = nullptr;
auto *Inst = dyn_cast<Instruction>(Scalar);
bool ReplaceInst = Inst && ExternalUsesAsOriginalScalar.contains(Inst);
// For struct-typed scalars, the User must be an ExtractValueInst that
// describes which struct field is being extracted. Copy its indices
// into an owning SmallVector so the cache key survives erasure of the
// ExtractValueInst.
SmallVector<unsigned, 1> Indices;
if (isa<StructType>(Scalar->getType())) {
assert(User && "User expected for StructType extract.");
const auto *EV = cast<ExtractValueInst>(User);
Indices.assign(EV->getIndices());
}
auto Key = std::make_pair(Scalar, Indices);
auto It = ScalarToEEs.find(Key);
if (It != ScalarToEEs.end()) {
// No need to emit many extracts, just move the only one in the
// current block.
auto EEIt = It->second.find(ReplaceInst ? Inst->getParent()
: Builder.GetInsertBlock());
if (EEIt != It->second.end()) {
Value *PrevV = EEIt->second.first;
if (auto *I = dyn_cast<Instruction>(PrevV);
I && !ReplaceInst &&
Builder.GetInsertPoint() != Builder.GetInsertBlock()->end() &&
Builder.GetInsertPoint()->comesBefore(I)) {
// For struct extracts the cached extractelement depends on a
// cached extractvalue (FieldVec). Move FieldVec up first so it
// still dominates the relocated extractelement after the move.
if (isa<StructType>(Scalar->getType())) {
auto FieldKey = std::make_pair(Vec, Indices);
auto FieldIt = StructFieldExtracts.find(FieldKey);
if (FieldIt != StructFieldExtracts.end()) {
auto BBIt = FieldIt->second.find(Builder.GetInsertBlock());
if (BBIt != FieldIt->second.end()) {
if (auto *FieldI = dyn_cast<Instruction>(BBIt->second);
FieldI &&
FieldI->getParent() == Builder.GetInsertBlock() &&
Builder.GetInsertPoint()->comesBefore(FieldI))
FieldI->moveBefore(*Builder.GetInsertBlock(),
Builder.GetInsertPoint());
}
}
}
I->moveBefore(*Builder.GetInsertPoint()->getParent(),
Builder.GetInsertPoint());
if (auto *CI = dyn_cast<Instruction>(EEIt->second.second))
CI->moveAfter(I);
}
Ex = PrevV;
ExV = EEIt->second.second ? EEIt->second.second : Ex;
}
}
if (!Ex) {
// "Reuse" the existing extract to improve final codegen.
if (ReplaceInst) {
// Leave the instruction as is, if it cheaper extracts and all
// operands are scalar.
if (auto *EE = dyn_cast<ExtractElementInst>(Inst)) {
IgnoredExtracts.insert(EE);
Ex = EE;
} else {
auto *CloneInst = Inst->clone();
CloneInst->insertBefore(Inst->getIterator());
if (Inst->hasName())
CloneInst->takeName(Inst);
Ex = CloneInst;
}
} else if (auto *ES = dyn_cast<ExtractElementInst>(Scalar);
ES && isa<Instruction>(Vec)) {
Value *V = ES->getVectorOperand();
auto *IVec = cast<Instruction>(Vec);
if (ArrayRef<TreeEntry *> ETEs = getTreeEntries(V); !ETEs.empty())
V = ETEs.front()->VectorizedValue;
if (auto *IV = dyn_cast<Instruction>(V);
!IV || IV == Vec || IV->getParent() != IVec->getParent() ||
IV->comesBefore(IVec))
Ex = Builder.CreateExtractElement(V, ES->getIndexOperand());
else
Ex = Builder.CreateExtractElement(Vec, ExternalUse.Lane);
} else if (auto *VecTy =
dyn_cast<FixedVectorType>(Scalar->getType())) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
unsigned VecTyNumElements = VecTy->getNumElements();
// When REVEC is enabled, we need to extract a vector.
// Note: The element size of Scalar may be different from the
// element size of Vec.
Ex = createExtractVector(Builder, Vec, VecTyNumElements,
ExternalUse.Lane * VecTyNumElements);
} else {
if (isa<StructType>(Vec->getType())) {
assert(isa<StructType>(Scalar->getType()) &&
"Vec is struct of vectors only when Scalar is struct.");
auto FieldKey = std::make_pair(Vec, Indices);
BasicBlock *EVBB = Builder.GetInsertBlock();
Value *FieldVec = nullptr;
auto FieldIt = StructFieldExtracts.find(FieldKey);
if (FieldIt != StructFieldExtracts.end()) {
auto BBIt = FieldIt->second.find(EVBB);
if (BBIt != FieldIt->second.end())
FieldVec = BBIt->second;
}
if (!FieldVec) {
FieldVec = Builder.CreateExtractValue(Vec, Indices);
StructFieldExtracts[FieldKey][EVBB] = FieldVec;
} else if (auto *FieldI = dyn_cast<Instruction>(FieldVec);
FieldI && Builder.GetInsertPoint() != EVBB->end() &&
Builder.GetInsertPoint()->comesBefore(FieldI)) {
// Cached extractvalue is below the current insertion point;
// move it up so the extractelement we are about to emit can
// use it.
FieldI->moveBefore(*EVBB, Builder.GetInsertPoint());
}
Vec = FieldVec;
}
if (Opts.slp_revec && isVectorizedTy(Scalar->getType())) {
unsigned VecTyNumElements = getNumElements(Scalar->getType());
// When REVEC is enabled, we need to extract a vector.
// Note: The element size of Scalar may be different from the
// element size of Vec.
Ex = createExtractVector(Builder, Vec, VecTyNumElements,
ExternalUse.Lane * VecTyNumElements);
} else {
Ex = Builder.CreateExtractElement(Vec, ExternalUse.Lane);
}
}
// If necessary, sign-extend or zero-extend ScalarRoot
// to the larger type.
ExV = Ex;
if (!isa<StructType>(Scalar->getType()) &&
Scalar->getType() != Ex->getType())
ExV = Builder.CreateIntCast(
Ex, Scalar->getType(),
!isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
auto *I = dyn_cast<Instruction>(Ex);
ScalarToEEs[Key].try_emplace(I ? I->getParent() : &F->getEntryBlock(),
std::make_pair(Ex, ExV));
if (Inst && ExV != Inst && ExV->getType() == Inst->getType())
redirectDbgValues(*Inst, *ExV);
}
// The then branch of the previous if may produce constants, since 0
// operand might be a constant.
if (auto *ExI = dyn_cast<Instruction>(Ex);
ExI && !isa<PHINode>(ExI) && !mayHaveNonDefUseDependency(*ExI)) {
GatherShuffleExtractSeq.insert(ExI);
CSEBlocks.insert(ExI->getParent());
}
return ExV;
}
assert(isa<FixedVectorType>(Scalar->getType()) &&
isa<InsertElementInst>(Scalar) &&
"In-tree scalar of vector type is not insertelement?");
auto *IE = cast<InsertElementInst>(Scalar);
VectorToInsertElement.try_emplace(Vec, IE);
return Vec;
};
// If User == nullptr, the Scalar remains as scalar in vectorized
// instructions or is used as extra arg. Generate ExtractElement instruction
// and update the record for this scalar in ExternallyUsedValues.
if (!User) {
if (!ScalarsWithNullptrUser.insert(Scalar).second)
continue;
assert(
(ExternallyUsedValues.count(Scalar) ||
ExternalUsesWithNonUsers.count(Scalar) ||
ExternalUsesAsOriginalScalar.contains(Scalar) ||
any_of(
Scalar->users(),
[&, TTI = TTI](llvm::User *U) {
if (ExternalUsesAsOriginalScalar.contains(U))
return true;
ArrayRef<TreeEntry *> UseEntries = getTreeEntries(U);
return !UseEntries.empty() &&
(E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize ||
E->State == TreeEntry::CompressVectorize ||
E->State == TreeEntry::BlendedLoadVectorize) &&
any_of(UseEntries, [&, TTI = TTI](TreeEntry *UseEntry) {
return (UseEntry->State == TreeEntry::Vectorize ||
UseEntry->State ==
TreeEntry::StridedVectorize ||
UseEntry->State ==
TreeEntry::ExpandVectorize ||
UseEntry->State ==
TreeEntry::CompressVectorize ||
UseEntry->State ==
TreeEntry::BlendedLoadVectorize) &&
doesInTreeUserNeedToExtract(
Scalar, getRootEntryInstruction(*UseEntry),
TLI, TTI);
});
})) &&
"Scalar with nullptr User must be registered in "
"ExternallyUsedValues map or remain as scalar in vectorized "
"instructions");
if (auto *VecI = dyn_cast<Instruction>(Vec)) {
if (auto *PHI = dyn_cast<PHINode>(VecI)) {
if (PHI->getParent()->isLandingPad())
Builder.SetInsertPoint(std::next(
PHI->getParent()->getLandingPadInst()->getIterator()));
else
Builder.SetInsertPoint(PHI->getParent()->getFirstNonPHIIt());
} else {
Builder.SetInsertPoint(std::next(VecI->getIterator()));
}
} else {
Builder.SetInsertPoint(F->getEntryBlock().begin());
}
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
// Required to update internally referenced instructions.
if (Scalar != NewInst) {
assert((!isa<ExtractElementInst>(Scalar) ||
!IgnoredExtracts.contains(cast<ExtractElementInst>(Scalar))) &&
"Extractelements should not be replaced.");
// Reduced values that are not operands of the reduction ops (e.g.
// narrowed reduction leaves) lose all users once the tree scalars are
// erased; keep the replacement alive for the reduction epilogue.
bool NeedsProtection =
UserIgnoreList && none_of(Scalar->users(), [&](llvm::User *U) {
return UserIgnoreList->contains(U);
});
Scalar->replaceAllUsesWith(NewInst);
if (NeedsProtection)
if (auto *I = dyn_cast<Instruction>(NewInst))
ExternalUseReplacements.insert(I);
}
continue;
}
if (auto *VU = dyn_cast<InsertElementInst>(User);
VU && VU->getOperand(1) == Scalar) {
// Skip if the scalar is another vector op or Vec is not an instruction.
if (!Scalar->getType()->isVectorTy() && isa<Instruction>(Vec)) {
if (auto *FTy = dyn_cast<FixedVectorType>(User->getType())) {
if (!UsedInserts.insert(VU).second)
continue;
// Need to use original vector, if the root is truncated.
auto BWIt = MinBWs.find(E);
if (BWIt != MinBWs.end() && Vec->getType() != VU->getType()) {
auto *ScalarTy = FTy->getElementType();
auto Key = std::make_pair(Vec, ScalarTy);
auto VecIt = VectorCasts.find(Key);
if (VecIt == VectorCasts.end()) {
IRBuilderBase::InsertPointGuard Guard(Builder);
if (auto *IVec = dyn_cast<PHINode>(Vec)) {
if (IVec->getParent()->isLandingPad())
Builder.SetInsertPoint(std::next(
IVec->getParent()->getLandingPadInst()->getIterator()));
else
Builder.SetInsertPoint(
IVec->getParent()->getFirstNonPHIOrLifetime());
} else if (auto *IVec = dyn_cast<Instruction>(Vec)) {
Builder.SetInsertPoint(IVec->getNextNode());
}
Vec = Builder.CreateIntCast(
Vec,
getWidenedType(
ScalarTy,
cast<FixedVectorType>(Vec->getType())->getNumElements()),
BWIt->second.second);
VectorCasts.try_emplace(Key, Vec);
} else {
Vec = VecIt->second;
}
}
std::optional<unsigned> InsertIdx = getElementIndex(VU);
if (InsertIdx) {
auto *It = find_if(
ShuffledInserts, [VU](const ShuffledInsertData<Value *> &Data) {
// Checks if 2 insertelements are from the same buildvector.
InsertElementInst *VecInsert = Data.InsertElements.front();
return areTwoInsertFromSameBuildVector(
VU, VecInsert,
[](InsertElementInst *II) { return II->getOperand(0); });
});
unsigned Idx = *InsertIdx;
if (It == ShuffledInserts.end()) {
(void)ShuffledInserts.emplace_back();
It = std::next(ShuffledInserts.begin(),
ShuffledInserts.size() - 1);
}
SmallVectorImpl<int> &Mask = It->ValueMasks[Vec];
if (Mask.empty())
Mask.assign(FTy->getNumElements(), PoisonMaskElem);
Mask[Idx] = ExternalUse.Lane;
It->InsertElements.push_back(cast<InsertElementInst>(User));
continue;
}
}
}
}
// Generate extracts for out-of-tree users.
// Find the insertion point for the extractelement lane.
if (auto *VecI = dyn_cast<Instruction>(Vec)) {
if (PHINode *PH = dyn_cast<PHINode>(User)) {
for (unsigned I : seq<unsigned>(0, PH->getNumIncomingValues())) {
if (PH->getIncomingValue(I) == Scalar) {
Instruction *IncomingTerminator =
PH->getIncomingBlock(I)->getTerminator();
if (isa<CatchSwitchInst>(IncomingTerminator)) {
Builder.SetInsertPoint(std::next(VecI->getIterator()));
} else {
Builder.SetInsertPoint(PH->getIncomingBlock(I)->getTerminator());
}
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
PH->setOperand(I, NewInst);
}
}
} else {
Builder.SetInsertPoint(cast<Instruction>(User));
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
if (isa<StructType>(Scalar->getType()) &&
isa_and_nonnull<ExtractValueInst>(User) &&
!isa<StructType>(NewInst->getType())) {
User->replaceAllUsesWith(NewInst);
eraseInstruction(cast<Instruction>(User));
} else {
User->replaceUsesOfWith(Scalar, NewInst);
}
}
} else {
Builder.SetInsertPoint(F->getEntryBlock().begin());
Value *NewInst = ExtractAndExtendIfNeeded(Vec);
User->replaceUsesOfWith(Scalar, NewInst);
}
LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
}
auto CreateShuffle = [&](Value *V1, Value *V2, ArrayRef<int> Mask) {
SmallVector<int> CombinedMask1(Mask.size(), PoisonMaskElem);
SmallVector<int> CombinedMask2(Mask.size(), PoisonMaskElem);
int VF = cast<FixedVectorType>(V1->getType())->getNumElements();
for (int I = 0, E = Mask.size(); I < E; ++I) {
if (Mask[I] < VF)
CombinedMask1[I] = Mask[I];
else
CombinedMask2[I] = Mask[I] - VF;
}
ShuffleInstructionBuilder ShuffleBuilder(
cast<VectorType>(V1->getType())->getElementType(), Builder, *this);
ShuffleBuilder.add(V1, CombinedMask1);
if (V2)
ShuffleBuilder.add(V2, CombinedMask2);
return ShuffleBuilder.finalize({}, {}, {});
};
auto &&ResizeToVF = [&CreateShuffle](Value *Vec, ArrayRef<int> Mask,
bool ForSingleMask) {
unsigned VF = Mask.size();
unsigned VecVF = cast<FixedVectorType>(Vec->getType())->getNumElements();
if (VF != VecVF) {
if (any_of(Mask, [VF](int Idx) { return Idx >= static_cast<int>(VF); })) {
Vec = CreateShuffle(Vec, nullptr, Mask);
return std::make_pair(Vec, true);
}
if (!ForSingleMask) {
SmallVector<int> ResizeMask(VF, PoisonMaskElem);
for (unsigned I = 0; I < VF; ++I) {
if (Mask[I] != PoisonMaskElem)
ResizeMask[Mask[I]] = Mask[I];
}
Vec = CreateShuffle(Vec, nullptr, ResizeMask);
}
}
return std::make_pair(Vec, false);
};
// Perform shuffling of the vectorize tree entries for better handling of
// external extracts.
for (int I = 0, E = ShuffledInserts.size(); I < E; ++I) {
// Find the first and the last instruction in the list of insertelements.
sort(ShuffledInserts[I].InsertElements, isFirstInsertElement);
InsertElementInst *FirstInsert = ShuffledInserts[I].InsertElements.front();
InsertElementInst *LastInsert = ShuffledInserts[I].InsertElements.back();
Builder.SetInsertPoint(LastInsert);
auto Vector = ShuffledInserts[I].ValueMasks.takeVector();
Value *NewInst = performExtractsShuffleAction<Value>(
MutableArrayRef(Vector.data(), Vector.size()),
FirstInsert->getOperand(0),
[](Value *Vec) {
return cast<VectorType>(Vec->getType())
->getElementCount()
.getKnownMinValue();
},
ResizeToVF,
[FirstInsert, &CreateShuffle](ArrayRef<int> Mask,
ArrayRef<Value *> Vals) {
assert((Vals.size() == 1 || Vals.size() == 2) &&
"Expected exactly 1 or 2 input values.");
if (Vals.size() == 1) {
// Do not create shuffle if the mask is a simple identity
// non-resizing mask.
if (Mask.size() != cast<FixedVectorType>(Vals.front()->getType())
->getNumElements() ||
!ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
return CreateShuffle(Vals.front(), nullptr, Mask);
return Vals.front();
}
return CreateShuffle(Vals.front() ? Vals.front()
: FirstInsert->getOperand(0),
Vals.back(), Mask);
});
auto It = ShuffledInserts[I].InsertElements.rbegin();
// Rebuild buildvector chain.
InsertElementInst *II = nullptr;
if (It != ShuffledInserts[I].InsertElements.rend())
II = *It;
SmallVector<Instruction *> Inserts;
while (It != ShuffledInserts[I].InsertElements.rend()) {
assert(II && "Must be an insertelement instruction.");
if (*It == II)
++It;
else
Inserts.push_back(II);
II = dyn_cast<InsertElementInst>(II->getOperand(0));
}
for (Instruction *II : reverse(Inserts)) {
II->replaceUsesOfWith(II->getOperand(0), NewInst);
if (auto *NewI = dyn_cast<Instruction>(NewInst))
if (II->getParent() == NewI->getParent() && II->comesBefore(NewI))
II->moveAfter(NewI);
NewInst = II;
}
LastInsert->replaceAllUsesWith(NewInst);
for (InsertElementInst *IE : reverse(ShuffledInserts[I].InsertElements)) {
IE->replaceUsesOfWith(IE->getOperand(0),
PoisonValue::get(IE->getOperand(0)->getType()));
IE->replaceUsesOfWith(IE->getOperand(1),
PoisonValue::get(IE->getOperand(1)->getType()));
eraseInstruction(IE);
}
CSEBlocks.insert(LastInsert->getParent());
}
SmallVector<Instruction *> RemovedInsts;
// For each vectorized value:
for (auto &TEPtr : VectorizableTree) {
TreeEntry *Entry = TEPtr.get();
// No need to handle users of gathered values.
if (Entry->isGather() || Entry->State == TreeEntry::SplitVectorize ||
DeletedNodes.contains(Entry) ||
TransformedToGatherNodes.contains(Entry))
continue;
if (Entry->CombinedOp == TreeEntry::ReducedBitcast ||
Entry->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
Entry->CombinedOp == TreeEntry::ReducedBitcastLoads ||
Entry->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
Entry->CombinedOp == TreeEntry::ReducedBitPack ||
Entry->CombinedOp == TreeEntry::ReducedCmpBitcast) {
// Skip constant node
if (!Entry->hasState()) {
assert(allConstant(Entry->Scalars) && "Expected constants only.");
continue;
}
for (Instruction *I : make_isa_range<Instruction>(Entry->Scalars)) {
if (Entry->isCopyableElement(I))
continue;
LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *I << ".\n");
RemovedInsts.push_back(I);
}
continue;
}
assert(Entry->VectorizedValue && "Can't find vectorizable value");
// For each lane:
for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
Value *Scalar = Entry->Scalars[Lane];
if (Entry->getOpcode() == Instruction::GetElementPtr &&
!isa<GetElementPtrInst>(Scalar))
continue;
if (auto *EE = dyn_cast<ExtractElementInst>(Scalar);
EE && IgnoredExtracts.contains(EE))
continue;
if (!isa<Instruction>(Scalar) || Entry->isCopyableElement(Scalar))
continue;
#ifndef NDEBUG
Type *Ty = Scalar->getType();
if (!Ty->isVoidTy()) {
for (User *U : Scalar->users()) {
LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
// It is legal to delete users in the ignorelist.
assert((isVectorized(U) ||
(UserIgnoreList && UserIgnoreList->contains(U)) ||
(isa_and_nonnull<Instruction>(U) &&
isDeleted(cast<Instruction>(U)))) &&
"Deleting out-of-tree value");
}
}
#endif
LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
auto *I = cast<Instruction>(Scalar);
RemovedInsts.push_back(I);
}
// Erase peeled intermediates (not listed in Scalars), except the ones
// kept because the tree still claims them.
for (Value *V :
make_filter_range(Entry->getReassocScalars(), [&](Value *V) {
return !KeptReassocScalars.contains(V);
})) {
#ifndef NDEBUG
for (User *U : V->users()) {
LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
// Single-use peeled scalars are consumed by this entry or a sibling
// in the same chain.
assert((isVectorized(U) ||
(UserIgnoreList && UserIgnoreList->contains(U)) ||
(isa_and_nonnull<Instruction>(U) &&
isDeleted(cast<Instruction>(U)))) &&
"Deleting out-of-tree value");
}
#endif
LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *V << ".\n");
RemovedInsts.push_back(cast<Instruction>(V));
}
}
// Collect tree-entry vector instructions that ended up without any non-dead
// uses (can happen with the non-power-of-2 paths, where intermediate vectors
// get replaced but the original node still holds a reference).
// FIXME: track and clear those references at replacement time instead of
// sweeping after the fact. Done BEFORE mergeDIAssignID so any DIAssignID
// from intermediate vectors is folded into the root vector instead of being
// dropped on erase.
{
Value *RootVec = getRootNode().VectorizedValue;
// Seed with the scalars already queued for removal - a vector whose only
// user is a scalar-to-be-removed is also dead.
SmallPtrSet<Value *, 16> DeadSet(RemovedInsts.begin(), RemovedInsts.end());
auto AllUsesAreDead = [&](const Value *V) {
return all_of(V->users(),
[&](const User *U) { return DeadSet.contains(U); });
};
SmallPtrSet<const Value *, 16> Candidates;
SmallVector<Instruction *, 16> Worklist;
for (const auto &TEPtr : VectorizableTree) {
Value *VV = TEPtr->VectorizedValue;
if (!VV || VV == RootVec || VV->getType()->isVoidTy() ||
DeadSet.contains(VV))
continue;
auto *I = dyn_cast<Instruction>(VV);
if (!I)
continue;
if (Candidates.insert(I).second)
Worklist.push_back(I);
}
// Worklist sweep: when a vector becomes dead, its operand candidates
// may have lost their last live user, so re-enqueue them. Cascaded dead
// chains converge in O(N + total uses) instead of O(depth * N * uses).
while (!Worklist.empty()) {
Instruction *I = Worklist.pop_back_val();
if (DeadSet.contains(I))
continue;
if (!I->use_empty() && !AllUsesAreDead(I))
continue;
DeadSet.insert(I);
RemovedInsts.push_back(I);
for (Instruction *OI : make_isa_range<Instruction>(I->operand_values()))
if (Candidates.contains(OI) && !DeadSet.contains(OI))
Worklist.push_back(OI);
}
}
// Merge the DIAssignIDs from the about-to-be-deleted instructions into the
// new vector instruction.
if (auto *V = dyn_cast<Instruction>(getRootNode().VectorizedValue))
V->mergeDIAssignID(RemovedInsts);
// Clear up reduction references, if any.
if (UserIgnoreList) {
for (Instruction *I : RemovedInsts) {
ArrayRef<TreeEntry *> Entries = getTreeEntries(I);
// Dead intermediate vector instructions collected above have no tree
// entry - nothing to unlink here.
if (Entries.empty())
continue;
const TreeEntry *IE = Entries.front();
if (ArrayRef<TreeEntry *> SplitEntries = getSplitTreeEntries(I);
!SplitEntries.empty() && SplitEntries.front()->Idx < IE->Idx)
IE = SplitEntries.front();
if (IE->Idx != 0 &&
!(getRootNode().isGather() && IE->UserTreeIndex &&
(ValueToGatherNodes.lookup(I).contains(&getRootNode()) ||
(IE->UserTreeIndex.UserTE == &getRootNode() &&
IE->UserTreeIndex.EdgeIdx == UINT_MAX))) &&
!(getRootNode().State == TreeEntry::SplitVectorize &&
IE->UserTreeIndex && is_contained(getRootNodeScalars(), I)) &&
!(GatheredLoadsEntriesFirst.has_value() &&
IE->Idx >= *GatheredLoadsEntriesFirst && getRootNode().isGather() &&
is_contained(getRootNodeScalars(), I)) &&
!(!getRootNode().isGather() && getRootNode().isCopyableElement(I)) &&
// Dropped narrowed reduction chain instructions may still use
// non-root scalars; such uses must be cleared as well.
none_of(I->users(),
[&](User *U) { return NarrowedChainInsts.contains(U); }))
continue;
SmallVector<SelectInst *> LogicalOpSelects;
I->replaceUsesWithIf(PoisonValue::get(I->getType()), [&](Use &U) {
// Do not replace condition of the logical op in form select <cond>.
bool IsPoisoningLogicalOp = isa<SelectInst>(U.getUser()) &&
(match(U.getUser(), m_LogicalAnd()) ||
match(U.getUser(), m_LogicalOr())) &&
U.getOperandNo() == 0;
if (IsPoisoningLogicalOp) {
LogicalOpSelects.push_back(cast<SelectInst>(U.getUser()));
return false;
}
return UserIgnoreList->contains(U.getUser());
});
// Replace conditions of the poisoning logical ops with the non-poison
// constant value.
for (SelectInst *SI : LogicalOpSelects)
SI->setCondition(Constant::getNullValue(SI->getCondition()->getType()));
}
}
// Externally used values, which are not operands of the reduction ops (e.g.
// gathered narrowed reduction leaves), may lose all users once the tree
// scalars are erased; keep them alive for the reduction epilogue. Too many
// users - keep conservatively.
if (UserIgnoreList)
for (Instruction *I : make_isa_range<Instruction>(ExternallyUsedValues))
if (I->hasNUsesOrMore(UsesLimit) || none_of(I->users(), [&](User *U) {
return UserIgnoreList->contains(U);
}))
ExternalUseReplacements.insert(I);
// Retain to-be-deleted instructions for some debug-info bookkeeping and alias
// cache correctness.
// NOTE: removeInstructionAndOperands only marks the instruction for deletion
// - instructions are not deleted until later.
removeInstructionsAndOperands(ArrayRef(RemovedInsts), VectorValuesAndScales);
// The emitted vectorized code and the erased scalars may invalidate the
// cached analyses results.
Cache.clear();
// The new vector code may capture pointers that were previously not captured
// (e.g. the pointer operands of a masked gather), so the capture results
// cached in BatchAA are stale. Alias queries against the new instructions
// would otherwise treat such pointers as inaccessible to them.
BatchAA.emplace(*AA);
Builder.ClearInsertionPoint();
InstrElementSize.clear();
const TreeEntry &RootTE = getRootNode();
Value *Vec = RootTE.VectorizedValue;
if (auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 &&
It != MinBWs.end() &&
ReductionBitWidth != It->second.first) {
IRBuilder<>::InsertPointGuard Guard(Builder);
Builder.SetInsertPoint(ReductionRoot->getIterator());
if (isReducedBitcastRoot() || isReducedCmpBitcastRoot()) {
Vec = Builder.CreateIntCast(Vec, Builder.getIntNTy(ReductionBitWidth),
It->second.second);
} else {
Vec = Builder.CreateIntCast(
Vec,
VectorType::get(Builder.getIntNTy(ReductionBitWidth),
cast<VectorType>(Vec->getType())->getElementCount()),
It->second.second);
}
}
return Vec;
}
void BoUpSLP::optimizeGatherSequence() {
LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherShuffleExtractSeq.size()
<< " gather sequences instructions.\n");
// LICM InsertElementInst sequences.
for (Instruction *I :
make_filter_range(GatherShuffleExtractSeq,
[&](Instruction *I) { return !isDeleted(I); })) {
// Check if this block is inside a loop.
Loop *L = LI->getLoopFor(I->getParent());
if (!L)
continue;
// Check if it has a preheader.
BasicBlock *PreHeader = L->getLoopPreheader();
if (!PreHeader)
continue;
// If the vector or the element that we insert into it are
// instructions that are defined in this basic block then we can't
// hoist this instruction.
if (any_of(make_isa_range<Instruction>(I->operands()),
[L](Instruction *OpI) { return L->contains(OpI); }))
continue;
// We can hoist this instruction. Move it to the pre-header.
I->moveBefore(PreHeader->getTerminator()->getIterator());
CSEBlocks.insert(PreHeader);
}
// Make a list of all reachable blocks in our CSE queue.
SmallVector<const DomTreeNode *, 8> CSEWorkList;
CSEWorkList.reserve(CSEBlocks.size());
for (BasicBlock *BB : CSEBlocks)
if (DomTreeNode *N = DT->getNode(BB))
CSEWorkList.push_back(N);
// Sort blocks by domination. This ensures we visit a block after all blocks
// dominating it are visited.
llvm::sort(CSEWorkList, [](const DomTreeNode *A, const DomTreeNode *B) {
assert((A == B) == (A->getDFSNumIn() == B->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
return A->getDFSNumIn() < B->getDFSNumIn();
});
// Less defined shuffles can be replaced by the more defined copies.
// Between two shuffles one is less defined if it has the same vector operands
// and its mask indeces are the same as in the first one or undefs. E.g.
// shuffle %0, poison, <0, 0, 0, undef> is less defined than shuffle %0,
// poison, <0, 0, 0, 0>.
auto &&IsIdenticalOrLessDefined = [this](Instruction *I1, Instruction *I2,
SmallVectorImpl<int> &NewMask) {
if (I1->getType() != I2->getType())
return false;
auto *SI1 = dyn_cast<ShuffleVectorInst>(I1);
auto *SI2 = dyn_cast<ShuffleVectorInst>(I2);
if (!SI1 || !SI2)
return I1->isIdenticalTo(I2);
if (SI1->isIdenticalTo(SI2))
return true;
for (int I = 0, E = SI1->getNumOperands(); I < E; ++I)
if (SI1->getOperand(I) != SI2->getOperand(I))
return false;
// Check if the second instruction is more defined than the first one.
NewMask.assign(SI2->getShuffleMask().begin(), SI2->getShuffleMask().end());
ArrayRef<int> SM1 = SI1->getShuffleMask();
// Count trailing undefs in the mask to check the final number of used
// registers.
unsigned LastUndefsCnt = 0;
for (int I = 0, E = NewMask.size(); I < E; ++I) {
if (SM1[I] == PoisonMaskElem)
++LastUndefsCnt;
else
LastUndefsCnt = 0;
if (NewMask[I] != PoisonMaskElem && SM1[I] != PoisonMaskElem &&
NewMask[I] != SM1[I])
return false;
if (NewMask[I] == PoisonMaskElem)
NewMask[I] = SM1[I];
}
// Check if the last undefs actually change the final number of used vector
// registers.
return SM1.size() - LastUndefsCnt > 1 &&
getNumberOfParts(SI1->getType(), SI1->getType()->getElementType()) ==
getNumberOfParts(getWidenedType(SI1->getType()->getElementType(),
SM1.size() - LastUndefsCnt),
SI1->getType()->getElementType());
};
// Perform O(N^2) search over the gather/shuffle sequences and merge identical
// instructions. TODO: We can further optimize this scan if we split the
// instructions into different buckets based on the insert lane.
SmallVector<Instruction *, 16> Visited;
for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
assert(*I &&
(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
"Worklist not sorted properly!");
BasicBlock *BB = (*I)->getBlock();
// For all instructions in blocks containing gather sequences:
for (Instruction &In : llvm::make_early_inc_range(*BB)) {
if (isDeleted(&In))
continue;
if (!isa<InsertElementInst, ExtractElementInst, ShuffleVectorInst>(&In) &&
!GatherShuffleExtractSeq.contains(&In))
continue;
// Check if we can replace this instruction with any of the
// visited instructions.
bool Replaced = false;
for (Instruction *&V : Visited) {
SmallVector<int> NewMask;
if (IsIdenticalOrLessDefined(&In, V, NewMask) &&
DT->dominates(V->getParent(), In.getParent())) {
In.replaceAllUsesWith(V);
eraseInstruction(&In);
if (auto *SI = dyn_cast<ShuffleVectorInst>(V))
if (!NewMask.empty())
SI->setShuffleMask(NewMask);
Replaced = true;
break;
}
if (isa<ShuffleVectorInst>(In) && isa<ShuffleVectorInst>(V) &&
GatherShuffleExtractSeq.contains(V) &&
IsIdenticalOrLessDefined(V, &In, NewMask) &&
DT->dominates(In.getParent(), V->getParent())) {
In.moveAfter(V);
V->replaceAllUsesWith(&In);
eraseInstruction(V);
if (auto *SI = dyn_cast<ShuffleVectorInst>(&In))
if (!NewMask.empty())
SI->setShuffleMask(NewMask);
V = &In;
Replaced = true;
break;
}
}
if (!Replaced) {
assert(!is_contained(Visited, &In));
Visited.push_back(&In);
}
}
}
CSEBlocks.clear();
GatherShuffleExtractSeq.clear();
}
BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI) {
auto &BundlePtr =
ScheduledBundlesList.emplace_back(std::make_unique<ScheduleBundle>());
for (Value *V : make_filter_range(
VL, [&](Value *V) { return !S.isNonSchedulable(V); })) {
auto *I = cast<Instruction>(V);
if (S.isCopyableElement(V)) {
// Add a copyable element model.
ScheduleCopyableData &SD =
addScheduleCopyableData(EI, I, SchedulingRegionID, *BundlePtr);
// Group the instructions to a bundle.
BundlePtr->add(&SD);
continue;
}
ScheduleData *BundleMember = getScheduleData(V);
assert(BundleMember && "no ScheduleData for bundle member "
"(maybe not in same basic block)");
// Group the instructions to a bundle.
BundlePtr->add(BundleMember);
ScheduledBundles.try_emplace(I).first->getSecond().push_back(
BundlePtr.get());
}
assert(BundlePtr && *BundlePtr && "Failed to find schedule bundle");
return *BundlePtr;
}
// Groups the instructions to a bundle (which is then a single scheduling entity)
// and schedules instructions until the bundle gets ready.
std::optional<BoUpSLP::ScheduleBundle *>
BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
const InstructionsState &S,
const EdgeInfo &EI) {
// No need to schedule PHIs, insertelement, extractelement and extractvalue
// instructions.
if (isa<PHINode>(S.getMainOp()) ||
isVectorLikeInstWithConstOps(S.getMainOp()))
return nullptr;
bool HasCopyables = S.areInstructionsWithCopyableElements();
bool DoesNotRequireScheduling =
(!HasCopyables && doesNotNeedToSchedule(VL)) ||
all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); });
if (S.areInstructionsWithCopyableElements() && EI && EI.UserTE->hasState() &&
EI.UserTE->hasCopyableElements() &&
EI.UserTE->getMainOp()->getParent() == S.getMainOp()->getParent() &&
all_of(VL, [&](Value *V) {
if (S.isCopyableElement(V))
return true;
return isUsedOutsideBlock(V);
}))
return std::nullopt;
// Drop a copyable scalar if it has a same-block, non-PHI, non-schedulable
// user that is reused outside this block via a non-PHI use. Dependency
// tracking across multiple nodes may then be incorrect.
if (S.areInstructionsWithCopyableElements() && EI && EI.UserTE->hasState() &&
any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
if (!S.isCopyableElement(I))
return false;
return any_of(I->users(), [&](User *U) {
auto *IU = dyn_cast<Instruction>(U);
if (!IU || IU->getParent() != I->getParent())
return false;
return !isa<PHINode>(IU) && !IU->hasOneUse() &&
isUsedOutsideBlock(IU) &&
!all_of(IU->users(), IsaPred<PHINode>);
});
}))
return std::nullopt;
// If any instruction is used outside block only and its operand is placed
// immediately before it, do not schedule, it may cause wrong def-use chain.
if (S.areInstructionsWithCopyableElements() && any_of(VL, [&](Value *V) {
if (isa<PoisonValue>(V) || S.isCopyableElement(V))
return false;
if (isUsedOutsideBlock(V)) {
for (Instruction *I :
make_isa_range<Instruction>(cast<Instruction>(V)->operands()))
return SLP->isVectorized(I) && I->getNextNode() == V;
}
return false;
}))
return std::nullopt;
if (S.areInstructionsWithCopyableElements() && EI) {
bool IsNonSchedulableWithParentPhiNode =
EI.UserTE->doesNotNeedToSchedule() && EI.UserTE->UserTreeIndex &&
EI.UserTE->UserTreeIndex.UserTE->hasState() &&
EI.UserTE->UserTreeIndex.UserTE->State != TreeEntry::SplitVectorize &&
EI.UserTE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI;
if (IsNonSchedulableWithParentPhiNode) {
ArrayRef<Value *> PhiScalars = EI.UserTE->UserTreeIndex.UserTE->Scalars;
for (const auto [Idx, V] : enumerate(PhiScalars)) {
Value *Op = EI.UserTE->UserTreeIndex.UserTE->getOperand(
EI.UserTE->UserTreeIndex.EdgeIdx)[Idx];
auto *I = dyn_cast<Instruction>(Op);
if (!I || !isCommutative(I))
continue;
// Bail out only when I has a user that is an instruction outside the
// grandparent PHI's Scalars. Multiple uses that all land in the same
// vectorized PHI are tracked by the existing dependency machinery;
// uses outside it (e.g. a scalar PHI in a different block) break the
// scheduler's dep accounting for non-schedulable copyable bundles.
if (I->hasOneUse())
continue;
if (any_of(I->users(), [&](User *U) {
auto *UI = dyn_cast<Instruction>(U);
return UI && !is_contained(PhiScalars, UI);
}))
return std::nullopt;
}
} else {
// If any of the parent requires scheduling - exit, complex dep between
// schedulable/non-schedulable parents.
if (any_of(EI.UserTE->Scalars, [&](Value *V) {
if (EI.UserTE->hasCopyableElements() &&
EI.UserTE->isCopyableElement(V))
return false;
ArrayRef<TreeEntry *> Entries = SLP->getTreeEntries(V);
return any_of(Entries, [](const TreeEntry *TE) {
return TE->doesNotNeedToSchedule() && TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->hasState() &&
TE->UserTreeIndex.UserTE->State !=
TreeEntry::SplitVectorize &&
TE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI;
});
}))
return std::nullopt;
}
}
// A scalar that is a non-copyable (i.e. really vectorized) element in this
// PHI-operand node, but a copyable element in another PHI-operand node, is
// vectorized here and only gathered/reused there. Both nodes feed different
// incoming values of the same vectorized PHI and may be emitted in different
// predecessor blocks, so the gathered reuse can end up not dominated by the
// vectorized value, producing broken IR. Bail out of scheduling to avoid it.
if (EI && EI.UserTE->State == TreeEntry::Vectorize &&
EI.UserTE->getOpcode() == Instruction::PHI &&
any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
if (HasCopyables && S.isCopyableElement(I))
return false;
return any_of(
SLP->VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE->UserTreeIndex &&
TE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
TE->UserTreeIndex.UserTE->getOpcode() ==
Instruction::PHI &&
TE->hasCopyableElements() && TE->isCopyableElement(I);
});
}))
return std::nullopt;
// A scalar that is a non-schedulable copyable element of another node in the
// same block (not the user of this node) is reused at the insertion point of
// that node, which does not depend on this node in the schedule. The
// vectorized value of this node may be emitted after the reuse, producing
// broken IR. Bail out of scheduling to avoid it.
if (any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
if ((HasCopyables && S.isCopyableElement(I)) ||
!doesNotNeedToBeScheduled(I))
return false;
return any_of(
SLP->VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
return TE.get() != EI.UserTE && !TE->isGather() &&
TE->hasCopyableElements() && TE->isCopyableElement(I) &&
TE->getMainOp()->getParent() == I->getParent() &&
TE->getOperations().isNonSchedulable(I);
});
}))
return std::nullopt;
// The same conflict when the node with the real element is built first: a
// non-schedulable copyable element of this node whose scalar is already
// vectorized in another node of the block, or belongs to a split node, would
// be reused from that node's vector, which may be emitted after this node.
if (HasCopyables &&
any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
return I->getParent() == BB && !isa<PHINode>(I) &&
S.isCopyableElement(I) && S.isNonSchedulable(I) &&
(SLP->isVectorized(I) || !SLP->getSplitTreeEntries(I).empty());
}))
return std::nullopt;
if (DoesNotRequireScheduling) {
// If all operands were replaced by copyables, the operands of this node
// might be not, so need to recalculate dependencies for schedule data,
// replaced by copyable schedule data.
for (Instruction *I : make_isa_range<Instruction>(VL)) {
if (HasCopyables && S.isCopyableElement(I))
continue;
// Expanded binops (e.g. shl X, 1 modeled as add X, X) double the
// dependency count of the duplicated operand. This non-schedulable
// path does not clear operand ScheduleData dependencies the way the
// regular scheduling path does (see CheckIfNeedToClearDeps below), so
// when that operand has more uses than this bundle member, or its
// ScheduleData already has computed dependencies that did not see
// the expanded duplicate use, the schedule's decrement count exceeds
// calculateDependencies' increment count and UnscheduledDeps goes
// negative. Bail out instead of producing an inconsistent schedule.
if (S.isExpandedBinOp(I) &&
any_of(enumerate(I->operands()), [&](const auto &P) {
if (S.isExpandedOperand(I, P.index()))
return false;
auto *OpI = dyn_cast<Instruction>(P.value());
if (!OpI)
return false;
if (!OpI->hasOneUse())
return true;
ScheduleData *SD = getScheduleData(OpI);
return SD && SD->hasValidDependencies();
}))
return std::nullopt;
SmallDenseMap<std::pair<Instruction *, Value *>, unsigned> UserOpToNumOps;
for (const Use &U : I->operands()) {
unsigned &NumOps =
UserOpToNumOps.try_emplace(std::make_pair(I, U.get()), 0)
.first->getSecond();
++NumOps;
if (auto *Op = dyn_cast<Instruction>(U.get());
Op && areAllOperandsReplacedByCopyableData(I, Op, *SLP, NumOps)) {
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && OpSD->hasValidDependencies())
// TODO: investigate how to improve it instead of early exiting.
return std::nullopt;
}
}
}
return nullptr;
}
// Any schedulable copyable with split vectorize parent - skip, not supported
// currently.
// TODO: investigate fix for this early exit.
if (S.areInstructionsWithCopyableElements() && EI.UserTE &&
EI.UserTE->State == TreeEntry::SplitVectorize &&
any_of(VL, [&](Value *V) {
return !S.isNonSchedulable(V) && S.isCopyableElement(V);
}))
return std::nullopt;
// Reject modeling the duplicated source operand of an expanded binop as a
// copyable element. shl X, 1 is represented as add X, X, so the parent uses X
// on a second operand edge that stays a plain (gathered) operand. At schedule
// time that duplicated use is decremented against X's own ScheduleData, which
// calculateDependencies attributed to the copyable instead, so X's
// unscheduled-deps counter goes negative and trips the assertion. Only the
// values this bundle models as copyable that are the expanded binop source
// operand are checked, matched by value so the detection is unaffected by the
// parent node's operand reordering or reuse shuffle mask.
// TODO: investigate modeling such operands correctly.
if (S.areInstructionsWithCopyableElements() && EI.UserTE &&
EI.UserTE->hasState()) {
for (Value *V : make_filter_range(
VL, [&](Value *V) { return S.isCopyableElement(V); })) {
if (any_of(make_isa_range<Instruction>(EI.UserTE->Scalars),
[&](Instruction *I) {
return EI.UserTE->isExpandedBinOp(I) &&
I->getOperand(0) == V;
}))
return std::nullopt;
}
}
// The lanes cannot depend on each other, so the bundle with the operand of
// another member is never scheduled.
if (!HasCopyables &&
any_of(make_isa_range<Instruction>(VL), [&](Instruction *I) {
return any_of(I->operand_values(),
[&](const Value *Op) { return is_contained(VL, Op); });
}))
return std::nullopt;
// Initialize the instruction bundle.
Instruction *OldScheduleEnd = ScheduleEnd;
LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.getMainOp() << "\n");
auto TryScheduleBundleImpl = [=](bool ReSchedule, ScheduleBundle &Bundle) {
// Clear deps or recalculate the region, if the memory instruction is a
// copyable. It may have memory deps, which must be recalculated.
SmallVector<ScheduleData *> ControlDependentMembers;
// Recompute the direct dependencies of copyable operands deferred by a
// previously scheduled bundle. The node that introduced the extra direct
// use is now part of the tree, so clearing and recalculating the
// dependencies here accounts for that use.
// If the schedule is not reset in this round, skip already scheduled
// operands: clearing their dependencies also resets the scheduled flag,
// so they would enter the ready list and be scheduled twice, decrementing
// the dependency counts of their operands once too many.
const bool WillReSchedule =
ReSchedule || (OldScheduleEnd && ScheduleEnd != OldScheduleEnd);
auto NeedsDependencyUpdate = [&](ScheduleData *SD) {
return isInSchedulingRegion(*SD) &&
(WillReSchedule || !SD->isScheduled());
};
for (ScheduleData *SD :
make_filter_range(RecalcCopyableOperandDeps, NeedsDependencyUpdate)) {
if (SD->hasValidDependencies())
SD->clearDirectDependencies();
ControlDependentMembers.push_back(SD);
}
RecalcCopyableOperandDeps.clear();
auto CheckIfNeedToClearDeps = [&](ScheduleBundle &Bundle) {
SmallDenseMap<std::pair<Instruction *, Value *>, unsigned> UserOpToNumOps;
for (ScheduleEntity *SE : Bundle.getBundle()) {
if (ScheduleCopyableData *SD = dyn_cast<ScheduleCopyableData>(SE)) {
if (ScheduleData *BundleMember = getScheduleData(SD->getInst());
BundleMember && BundleMember->hasValidDependencies()) {
BundleMember->clearDirectDependencies();
if (RegionHasStackSave ||
!isGuaranteedToTransferExecutionToSuccessor(
BundleMember->getInst()) ||
isWriteVisibleOnUnwind(BundleMember->getInst()))
ControlDependentMembers.push_back(BundleMember);
}
continue;
}
auto *SD = cast<ScheduleData>(SE);
if (SD->hasValidDependencies() &&
(!S.areInstructionsWithCopyableElements() ||
!S.isCopyableElement(SD->getInst())) &&
!getScheduleCopyableData(SD->getInst()).empty() && EI.UserTE &&
EI.UserTE->hasState() &&
(!EI.UserTE->hasCopyableElements() ||
!EI.UserTE->isCopyableElement(SD->getInst())))
SD->clearDirectDependencies();
const bool IsExpandedBinOp = S.isExpandedBinOp(SD->getInst());
for (const Use &U : SD->getInst()->operands()) {
unsigned &NumOps =
UserOpToNumOps
.try_emplace(std::make_pair(SD->getInst(), U.get()), 0)
.first->getSecond();
++NumOps;
auto *Op = dyn_cast<Instruction>(U.get());
if (Op && areAllOperandsReplacedByCopyableData(SD->getInst(), Op,
*SLP, NumOps)) {
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && OpSD->hasValidDependencies()) {
OpSD->clearDirectDependencies();
if (RegionHasStackSave ||
!isGuaranteedToTransferExecutionToSuccessor(
OpSD->getInst()) ||
isWriteVisibleOnUnwind(OpSD->getInst()))
ControlDependentMembers.push_back(OpSD);
// areAllOperandsReplacedByCopyableData() returned true, so every
// tree entry that currently contains this instruction models Op
// as a copyable element and Op's recomputed dependencies do not
// count this def-use edge. The bundle being scheduled is a new
// node for the same instruction that is not registered in the
// tree yet and that uses Op directly (Op is a real operand here).
// Once that node is added to the tree the direct use must be
// counted, otherwise the scheduler decrements Op more times than
// its dependency count and trips the unscheduled-deps assertion.
// Defer the operand's dependency recomputation to the next bundle
// scheduling, when the new node is part of the tree and the
// direct use is accounted for.
RecalcCopyableOperandDeps.insert(OpSD);
continue;
}
}
if (IsExpandedBinOp) {
if (ScheduleData *OpSD = getScheduleData(U.get());
OpSD && OpSD->hasValidDependencies()) {
OpSD->clearDirectDependencies();
ControlDependentMembers.push_back(OpSD);
continue;
}
}
// areAllOperandsReplacedByCopyableData() can be false while Op is
// still modeled as a copyable element in one or more already built
// nodes (e.g. when another node that shares an instruction with this
// one uses Op directly). This newly built node, not registered in the
// tree yet, also uses Op directly, so Op's real ScheduleData
// dependency count - which mixes direct def-use edges with copyable
// uses - does not yet account for this direct use. Defer its
// recomputation to the next bundle scheduling, when this node is part
// of the tree, so the direct use is counted; otherwise the scheduler
// decrements Op more times than its dependency count and trips the
// unscheduled-deps assertion. Op's current dependencies are kept (not
// cleared) so it stays schedulable for this bundle's dependency
// calculation.
if (Op) {
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && !getScheduleCopyableData(Op).empty())
RecalcCopyableOperandDeps.insert(OpSD);
}
}
}
};
// The scheduling region got new instructions at the lower end (or it is a
// new region for the first bundle). This makes it necessary to
// recalculate all dependencies.
// It is seldom that this needs to be done a second time after adding the
// initial bundle to the region.
if (OldScheduleEnd && ScheduleEnd != OldScheduleEnd) {
clearDependencies();
ReSchedule = true;
}
SmallPtrSet<Value *, 4> ExpandedOps;
for (Value *V : VL) {
if (S.isExpandedBinOp(V))
ExpandedOps.insert(V);
}
// Check if the bundle data has deps for copyable elements already. In
// this case need to reset deps and recalculate it.
if (Bundle && !Bundle.getBundle().empty()) {
if (!ScheduleCopyableDataMap.empty() || !ExpandedOps.empty())
CheckIfNeedToClearDeps(Bundle);
LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << Bundle << " in block "
<< BB->getName() << "\n");
calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP,
ExpandedOps, ControlDependentMembers);
} else if (!ControlDependentMembers.empty()) {
ScheduleBundle Invalid = ScheduleBundle::invalid();
calculateDependencies(Invalid, /*InsertInReadyList=*/!ReSchedule, SLP,
ExpandedOps, ControlDependentMembers);
}
if (ReSchedule) {
resetSchedule();
initialFillReadyList(ReadyInsts);
}
// Now try to schedule the new bundle or (if no bundle) just calculate
// dependencies. As soon as the bundle is "ready" it means that there are no
// cyclic dependencies and we can schedule it. Note that's important that we
// don't "schedule" the bundle yet.
while (((!Bundle && ReSchedule) || (Bundle && !Bundle.isReady())) &&
!ReadyInsts.empty()) {
ScheduleEntity *Picked = ReadyInsts.pop_back_val();
assert(Picked->isReady() && "must be ready to schedule");
schedule(*SLP, S, EI, Picked, ReadyInsts);
if (Picked == &Bundle)
break;
}
};
// Make sure that the scheduling region contains all
// instructions of the bundle.
for (Value *V : make_filter_range(
VL, [&](Value *V) { return !S.isNonSchedulable(V); })) {
if (!extendSchedulingRegion(V, S)) {
// If the scheduling region got new instructions at the lower end (or it
// is a new region for the first bundle). This makes it necessary to
// recalculate all dependencies.
// Otherwise the compiler may crash trying to incorrectly calculate
// dependencies and emit instruction in the wrong order at the actual
// scheduling.
ScheduleBundle Invalid = ScheduleBundle::invalid();
TryScheduleBundleImpl(/*ReSchedule=*/false, Invalid);
return std::nullopt;
}
}
bool ReSchedule = false;
for (Value *V : make_filter_range(
VL, [&](Value *V) { return !S.isNonSchedulable(V); })) {
SmallVector<ScheduleCopyableData *> CopyableData =
getScheduleCopyableData(cast<Instruction>(V));
if (!CopyableData.empty()) {
for (ScheduleCopyableData *SD : CopyableData)
ReadyInsts.remove(SD);
}
ScheduleData *BundleMember = getScheduleData(V);
assert((BundleMember || S.isCopyableElement(V)) &&
"no ScheduleData for bundle member (maybe not in same basic block)");
if (!BundleMember)
continue;
// Make sure we don't leave the pieces of the bundle in the ready list when
// whole bundle might not be ready.
ReadyInsts.remove(BundleMember);
if (ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(V);
!Bundles.empty()) {
for (ScheduleBundle *B : Bundles)
ReadyInsts.remove(B);
}
if (!S.isCopyableElement(V) && !S.isExpandedBinOp(V) &&
!BundleMember->isScheduled())
continue;
// A bundle member was scheduled as single instruction before and now
// needs to be scheduled as part of the bundle. We just get rid of the
// existing schedule.
// A bundle member has deps calculated before it was copyable element - need
// to reschedule.
LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
<< " was already scheduled\n");
ReSchedule = true;
}
ScheduleBundle &Bundle = buildBundle(VL, S, EI);
TryScheduleBundleImpl(ReSchedule, Bundle);
if (!Bundle.isReady()) {
// Copyable data scheduling is just removed.
for (ScheduleEntity *BD :
make_filter_range(Bundle.getBundle(), [](ScheduleEntity *BD) {
return !isa<ScheduleCopyableData>(BD);
})) {
if (BD->isReady()) {
ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(BD->getInst());
if (Bundles.empty()) {
ReadyInsts.insert(BD);
continue;
}
for (ScheduleBundle *B : Bundles)
if (B->isReady())
ReadyInsts.insert(B);
}
}
cancelScheduling(Bundle);
ScheduledBundlesList.pop_back();
SmallVector<ScheduleData *> ControlDependentMembers;
for (Value *V : make_filter_range(
VL, [&](Value *V) { return !S.isNonSchedulable(V); })) {
auto *I = cast<Instruction>(V);
if (S.isExpandedBinOp(I)) {
for (Value *Op : I->operands()) {
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && OpSD->hasValidDependencies()) {
OpSD->clearDirectDependencies();
ControlDependentMembers.push_back(OpSD);
}
}
}
if (S.isCopyableElement(I)) {
// Need to recalculate dependencies for the actual schedule data, even
// if they were already cleared by the failed bundle scheduling attempt.
if (ScheduleData *OpSD = getScheduleData(I)) {
OpSD->clearDirectDependencies();
ControlDependentMembers.push_back(OpSD);
if (any_of(VL, [&](Value *V) { return S.isExpandedBinOp(V); })) {
// Clear scheduling data for all operands, if this node is operand
// of the expanded instruction.
for (Value *Op : I->operands()) {
if (ScheduleData *OpSD = getScheduleData(Op);
OpSD && OpSD->hasValidDependencies()) {
OpSD->clearDirectDependencies();
ControlDependentMembers.push_back(OpSD);
}
}
}
}
}
}
if (!ControlDependentMembers.empty()) {
ScheduleBundle Invalid = ScheduleBundle::invalid();
SmallPtrSet<Value *, 4> ExpandedOps;
calculateDependencies(Invalid, /*InsertInReadyList=*/false, SLP,
ExpandedOps, ControlDependentMembers);
}
return std::nullopt;
}
return &Bundle;
}
BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
// Allocate a new ScheduleData for the instruction.
if (ChunkPos >= ChunkSize) {
ScheduleDataChunks.push_back(std::make_unique<ScheduleData[]>(ChunkSize));
ChunkPos = 0;
}
return &(ScheduleDataChunks.back()[ChunkPos++]);
}
bool BoUpSLP::BlockScheduling::extendSchedulingRegion(
Value *V, const InstructionsState &S) {
Instruction *I = dyn_cast<Instruction>(V);
assert(I && "bundle member must be an instruction");
if (getScheduleData(I))
return true;
if (!ScheduleStart) {
// It's the first instruction in the new region.
initScheduleData(I, I->getNextNode(), nullptr, nullptr);
ScheduleStart = I;
ScheduleEnd = I->getNextNode();
assert(ScheduleEnd && "tried to vectorize a terminator?");
LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
return true;
}
// Search up and down at the same time, because we don't know if the new
// instruction is above or below the existing scheduling region.
// Ignore debug info (and other "AssumeLike" intrinsics) so that's not counted
// against the budget. Otherwise debug info could affect codegen.
BasicBlock::reverse_iterator UpIter =
++ScheduleStart->getIterator().getReverse();
BasicBlock::reverse_iterator UpperEnd = BB->rend();
BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
BasicBlock::iterator LowerEnd = BB->end();
// Never skip the looked-for instruction itself: a bundle member may be an
// assume-like intrinsic (e.g. a copyable lane), and skipping it makes the
// scan miss it and extend the region in the wrong direction.
auto IsAssumeLikeIntr = [I](const Instruction &Cur) {
auto *II = dyn_cast<IntrinsicInst>(&Cur);
return &Cur != I && II && II->isAssumeLikeIntrinsic();
};
UpIter = std::find_if_not(UpIter, UpperEnd, IsAssumeLikeIntr);
DownIter = std::find_if_not(DownIter, LowerEnd, IsAssumeLikeIntr);
while (UpIter != UpperEnd && DownIter != LowerEnd && &*UpIter != I &&
&*DownIter != I) {
if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
return false;
}
++UpIter;
++DownIter;
UpIter = std::find_if_not(UpIter, UpperEnd, IsAssumeLikeIntr);
DownIter = std::find_if_not(DownIter, LowerEnd, IsAssumeLikeIntr);
}
if (DownIter == LowerEnd || (UpIter != UpperEnd && &*UpIter == I)) {
assert(I->getParent() == ScheduleStart->getParent() &&
"Instruction is in wrong basic block.");
initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
ScheduleStart = I;
LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I
<< "\n");
return true;
}
assert((UpIter == UpperEnd || (DownIter != LowerEnd && &*DownIter == I)) &&
"Expected to reach top of the basic block or instruction down the "
"lower end.");
assert(I->getParent() == ScheduleEnd->getParent() &&
"Instruction is in wrong basic block.");
initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
nullptr);
ScheduleEnd = I->getNextNode();
assert(ScheduleEnd && "tried to vectorize a terminator?");
LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
return true;
}
void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
Instruction *ToI,
ScheduleData *PrevLoadStore,
ScheduleData *NextLoadStore) {
ScheduleData *CurrentLoadStore = PrevLoadStore;
for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
// No need to allocate data for non-schedulable instructions.
if (isa<PHINode>(I))
continue;
ScheduleData *SD = ScheduleDataMap.lookup(I);
if (!SD) {
SD = allocateScheduleDataChunks();
ScheduleDataMap[I] = SD;
}
assert(!isInSchedulingRegion(*SD) &&
"new ScheduleData already in scheduling region");
SD->init(SchedulingRegionID, I);
auto CanIgnoreLoad = [](const Instruction *I) {
const auto *LI = dyn_cast<LoadInst>(I);
// If there is a simple load marked as invariant, we can ignore it.
// But, in the (unlikely) case of non-simple invariant load,
// we should not ignore it.
return LI && LI->isSimple() &&
LI->getMetadata(LLVMContext::MD_invariant_load);
};
if (I->mayReadOrWriteMemory() &&
// Simple InvariantLoad does not depend on other memory accesses.
!CanIgnoreLoad(I) &&
(!isa<IntrinsicInst>(I) ||
(cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect &&
cast<IntrinsicInst>(I)->getIntrinsicID() !=
Intrinsic::pseudoprobe))) {
// Update the linked list of memory accessing instructions.
if (CurrentLoadStore) {
CurrentLoadStore->setNextLoadStore(SD);
} else {
FirstLoadStoreInRegion = SD;
}
CurrentLoadStore = SD;
}
if (match(I, m_Intrinsic<Intrinsic::stacksave>()) ||
match(I, m_Intrinsic<Intrinsic::stackrestore>()))
RegionHasStackSave = true;
if (I->mayThrow())
RegionHasMayThrow = true;
}
if (NextLoadStore) {
if (CurrentLoadStore)
CurrentLoadStore->setNextLoadStore(NextLoadStore);
} else {
LastLoadStoreInRegion = CurrentLoadStore;
}
}
void BoUpSLP::BlockScheduling::calculateDependencies(
ScheduleBundle &Bundle, bool InsertInReadyList, BoUpSLP *SLP,
const SmallPtrSetImpl<Value *> &ExpandedOps,
ArrayRef<ScheduleData *> ControlDeps) {
SmallVector<ScheduleEntity *> WorkList;
auto ProcessNode = [&](ScheduleEntity *SE) {
if (auto *CD = dyn_cast<ScheduleCopyableData>(SE)) {
if (CD->hasValidDependencies())
return;
LLVM_DEBUG(dbgs() << "SLP: update deps of " << *CD << "\n");
CD->initDependencies();
CD->resetUnscheduledDeps();
const EdgeInfo &EI = CD->getEdgeInfo();
if (EI.UserTE) {
ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
const auto *It = find(Op, CD->getInst());
assert(It != Op.end() && "Lane not set");
SmallPtrSet<Instruction *, 4> Visited;
do {
int Lane = std::distance(Op.begin(), It);
assert(Lane >= 0 && "Lane not set");
if (isa<StoreInst, InsertValueInst>(EI.UserTE->Scalars[Lane]) &&
!EI.UserTE->ReorderIndices.empty())
Lane = EI.UserTE->ReorderIndices[Lane];
assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
"Couldn't find extract lane");
auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
if (EI.UserTE->isCopyableElement(In)) {
// We may have not have related copyable scheduling data, if the
// instruction is non-schedulable.
if (ScheduleCopyableData *UseSD =
getScheduleCopyableData(EI.UserTE->UserTreeIndex, In)) {
CD->incDependencies();
if (!UseSD->isScheduled())
CD->incrementUnscheduledDeps(1);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
}
} else if (Visited.insert(In).second) {
if (ScheduleData *UseSD = getScheduleData(In)) {
unsigned ExtraDeps = 1;
// Increment twice, since the operand was expanded in binop.
if (EI.UserTE->isExpandedBinOp(In))
ExtraDeps = 2;
for_each(seq(ExtraDeps),
[&](unsigned) { CD->incDependencies(); });
if (!UseSD->isScheduled())
CD->incrementUnscheduledDeps(ExtraDeps);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
}
}
It = find(make_range(std::next(It), Op.end()), CD->getInst());
} while (It != Op.end());
if (CD->isReady() && CD->getDependencies() == 0 &&
(EI.UserTE->hasState() &&
(EI.UserTE->getMainOp()->getParent() !=
CD->getInst()->getParent() ||
(isa<PHINode>(EI.UserTE->getMainOp()) &&
(EI.UserTE->getMainOp()->hasNUsesOrMore(UsesLimit) ||
any_of(EI.UserTE->getMainOp()->users(), [&](User *U) {
auto *IU = dyn_cast<Instruction>(U);
if (!IU)
return true;
return IU->getParent() == EI.UserTE->getMainOp()->getParent();
})))))) {
// If no uses in the block - mark as having pseudo-use, which cannot
// be scheduled.
// Prevents incorrect def-use tracking between external user and
// actual instruction.
CD->incDependencies();
CD->incrementUnscheduledDeps(1);
}
}
return;
}
auto *BundleMember = cast<ScheduleData>(SE);
if (BundleMember->hasValidDependencies())
return;
LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
BundleMember->initDependencies();
BundleMember->resetUnscheduledDeps();
// Handle def-use chain dependencies.
SmallDenseMap<Value *, unsigned> UserToNumOps;
for (User *U : BundleMember->getInst()->users()) {
if (isa<PHINode>(U))
continue;
if (ScheduleData *UseSD = getScheduleData(U)) {
// The operand is a copyable element - skip.
unsigned &NumOps = UserToNumOps.try_emplace(U, 0).first->getSecond();
++NumOps;
if (areAllOperandsReplacedByCopyableData(
cast<Instruction>(U), BundleMember->getInst(), *SLP, NumOps))
continue;
unsigned ExtraDeps = 1;
// Increment twice, since the operand was expanded in binop.
for (const TreeEntry *UserTE : SLP->getTreeEntries(U)) {
if (UserTE->isExpandedBinOp(U))
++ExtraDeps;
}
if (ExpandedOps.contains(U))
++ExtraDeps;
for_each(seq(ExtraDeps),
[&](unsigned) { BundleMember->incDependencies(); });
if (!UseSD->isScheduled())
BundleMember->incrementUnscheduledDeps(ExtraDeps);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
}
}
for (ScheduleCopyableData *UseSD :
getScheduleCopyableDataUsers(BundleMember->getInst())) {
BundleMember->incDependencies();
if (!UseSD->isScheduled())
BundleMember->incrementUnscheduledDeps(1);
if (!UseSD->hasValidDependencies() ||
(InsertInReadyList && UseSD->isReady()))
WorkList.push_back(UseSD);
}
SmallPtrSet<const Instruction *, 4> Visited;
auto MakeControlDependent = [&](Instruction *I) {
// Do not mark control dependent twice.
if (!Visited.insert(I).second)
return;
auto *DepDest = getScheduleData(I);
assert(DepDest && "must be in schedule window");
DepDest->addControlDependency(BundleMember);
BundleMember->incDependencies();
if (!DepDest->isScheduled())
BundleMember->incrementUnscheduledDeps(1);
if (!DepDest->hasValidDependencies() ||
(InsertInReadyList && DepDest->isReady()))
WorkList.push_back(DepDest);
};
// Any instruction which isn't safe to speculate at the beginning of the
// block is control depend on any early exit or non-willreturn call
// which proceeds it.
if (!isGuaranteedToTransferExecutionToSuccessor(BundleMember->getInst())) {
for (Instruction *I = BundleMember->getInst()->getNextNode();
I != ScheduleEnd; I = I->getNextNode()) {
if (isSafeToSpeculativelyExecute(I, &*BB->begin(), SLP->AC))
continue;
// Add the dependency
MakeControlDependent(I);
if (!isGuaranteedToTransferExecutionToSuccessor(I))
// Everything past here must be control dependent on I.
break;
}
}
// A write visible after unwinding must precede the next instruction that
// may throw.
if (isWriteVisibleOnUnwind(BundleMember->getInst())) {
for (Instruction *I = BundleMember->getInst()->getNextNode();
I != ScheduleEnd; I = I->getNextNode()) {
if (!I->mayThrow())
continue;
// Add the dependency
MakeControlDependent(I);
break;
}
}
if (RegionHasStackSave) {
// If we have an inalloc alloca instruction, it needs to be scheduled
// after any preceeding stacksave. We also need to prevent any alloca
// from reordering above a preceeding stackrestore.
if (match(BundleMember->getInst(), m_Intrinsic<Intrinsic::stacksave>()) ||
match(BundleMember->getInst(),
m_Intrinsic<Intrinsic::stackrestore>())) {
for (Instruction *I = BundleMember->getInst()->getNextNode();
I != ScheduleEnd; I = I->getNextNode()) {
if (match(I, m_Intrinsic<Intrinsic::stacksave>()) ||
match(I, m_Intrinsic<Intrinsic::stackrestore>()))
// Any allocas past here must be control dependent on I, and I
// must be memory dependend on BundleMember->Inst.
break;
if (!isa<AllocaInst>(I))
continue;
// Add the dependency
MakeControlDependent(I);
}
}
// In addition to the cases handle just above, we need to prevent
// allocas and loads/stores from moving below a stacksave or a
// stackrestore. Avoiding moving allocas below stackrestore is currently
// thought to be conservatism. Moving loads/stores below a stackrestore
// can lead to incorrect code.
if (isa<AllocaInst>(BundleMember->getInst()) ||
BundleMember->getInst()->mayReadOrWriteMemory()) {
for (Instruction *I = BundleMember->getInst()->getNextNode();
I != ScheduleEnd; I = I->getNextNode()) {
if (!match(I, m_Intrinsic<Intrinsic::stacksave>()) &&
!match(I, m_Intrinsic<Intrinsic::stackrestore>()))
continue;
// Add the dependency
MakeControlDependent(I);
break;
}
}
}
// Handle the memory dependencies (if any).
ScheduleData *NextLoadStore = BundleMember->getNextLoadStore();
if (!NextLoadStore)
return;
Instruction *SrcInst = BundleMember->getInst();
assert(SrcInst->mayReadOrWriteMemory() &&
"NextLoadStore list for non memory effecting bundle?");
MemoryLocation SrcLoc = getLocation(SrcInst);
bool SrcMayWrite = SrcInst->mayWriteToMemory();
unsigned NumAliased = 0;
unsigned DistToSrc = 1;
bool IsNonSimpleSrc = !SrcLoc.Ptr || !isSimple(SrcInst);
for (ScheduleData *DepDest = NextLoadStore; DepDest;
DepDest = DepDest->getNextLoadStore()) {
assert(isInSchedulingRegion(*DepDest) && "Expected to be in region");
// We have two limits to reduce the complexity:
// 1) AliasedCheckLimit: It's a small limit to reduce calls to
// SLP->isAliased (which is the expensive part in this loop).
// 2) MaxMemDepDistance: It's for very large blocks and it aborts
// the whole loop (even if the loop is fast, it's quadratic).
// It's important for the loop break condition (see below) to
// check this limit even between two read-only instructions.
Instruction *DepInst = DepDest->getInst();
bool MayConflict = SrcMayWrite || DepInst->mayWriteToMemory();
if (DistToSrc >= MaxMemDepDistance ||
(MayConflict && (IsNonSimpleSrc || NumAliased >= AliasedCheckLimit ||
SLP->isAliased(SrcLoc, SrcInst, DepInst)))) {
// Try to turn a may-alias dependency between distinct, range-checkable
// base objects into a runtime alias check. This covers both concrete
// aliases and the conservative dependencies added once the
// per-instruction alias-check limit is reached, so that a high alias
// count cannot block a region that is otherwise versionable. The
// dropped dependency is recorded so the final scheduleBlock()
// recomputation keeps it dropped consistently.
bool Dropped = false;
if (MayConflict && SLP->Opts.slp_vectorize_with_runtime_alias_checks) {
auto Key = std::make_pair(SrcInst, DepInst);
// A dependency dropped during buildTree() must stay dropped on every
// later recomputation (e.g. the final scheduleBlock()), independent
// of the current mode, so the schedule remains consistent.
if (IgnoredMemDeps.contains(Key)) {
Dropped = true;
} else if (SLP->isCoveredByExistingVersionCheck(BB, SrcInst,
DepInst)) {
// This block is the fast path of an earlier versioning whose
// runtime check already proved these bases disjoint, so the
// dependency can be dropped without emitting a new check.
Dropped = true;
} else if (SLP->isTryingRuntimeAliasChecks()) {
if (SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst) &&
SLP->recordRuntimeAliasCheck(BB, SrcInst, DepInst)) {
IgnoredMemDeps.insert(Key);
Dropped = true;
}
} else if (!SLP->hasNonCheckableMemBlocker()) {
// A dependency on a non-simple access (call, a volatile/atomic, or
// anything without a single load/store pointer) cannot be
// range-checked, so it is a hard blocker a retry cannot drop. A
// simple runtime-checkable may-alias pair could instead be dropped
// by a runtime check. Other simple pairs (e.g. same base) are
// resolved statically and are not blockers.
if (IsNonSimpleSrc || !getLoadStorePointerOperand(DepInst) ||
!isSimple(DepInst))
SLP->setHasNonCheckableMemBlocker(true);
else if (!SLP->hasRuntimeCheckableBlockers() &&
SLP->isRuntimeCheckableAliasPair(SrcInst, DepInst))
SLP->setHasRuntimeCheckableBlockers(true);
}
}
if (!Dropped) {
// Count only kept dependencies toward the alias-check limit. Counting
// dropped ones could prematurely trigger the conservative
// (limit-based) dependencies above and add spurious dependencies
// (e.g. between non-overlapping stores to the same base) that would
// block scheduling of an otherwise versionable region.
NumAliased++;
DepDest->addMemoryDependency(BundleMember);
BundleMember->incDependencies();
if (!DepDest->isScheduled())
BundleMember->incrementUnscheduledDeps(1);
if (!DepDest->hasValidDependencies() ||
(InsertInReadyList && DepDest->isReady()))
WorkList.push_back(DepDest);
}
}
// Example, explaining the loop break condition: Let's assume our
// starting instruction is i0 and MaxMemDepDistance = 3.
//
// +--------v--v--v
// i0,i1,i2,i3,i4,i5,i6,i7,i8
// +--------^--^--^
//
// MaxMemDepDistance let us stop alias-checking at i3 and we add
// dependencies from i0 to i3,i4,.. (even if they are not aliased).
// Previously we already added dependencies from i3 to i6,i7,i8
// (because of MaxMemDepDistance). As we added a dependency from
// i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
// and we can abort this loop at i6.
if (DistToSrc >= 2 * MaxMemDepDistance)
break;
DistToSrc++;
}
};
assert((Bundle || !ControlDeps.empty()) &&
"expected at least one instruction to schedule");
if (Bundle)
WorkList.push_back(Bundle.getBundle().front());
WorkList.append(ControlDeps.begin(), ControlDeps.end());
SmallPtrSet<ScheduleBundle *, 16> Visited;
while (!WorkList.empty()) {
ScheduleEntity *SD = WorkList.pop_back_val();
SmallVector<ScheduleBundle *, 1> CopyableBundle;
ArrayRef<ScheduleBundle *> Bundles;
if (auto *CD = dyn_cast<ScheduleCopyableData>(SD)) {
CopyableBundle.push_back(&CD->getBundle());
Bundles = CopyableBundle;
} else {
Bundles = getScheduleBundles(SD->getInst());
}
if (Bundles.empty()) {
if (!SD->hasValidDependencies())
ProcessNode(SD);
if (InsertInReadyList && SD->isReady()) {
ReadyInsts.insert(SD);
LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD << "\n");
}
continue;
}
for (ScheduleBundle *Bundle :
make_filter_range(Bundles, [&](ScheduleBundle *Bundle) {
return !Bundle->hasValidDependencies() &&
Visited.insert(Bundle).second;
})) {
assert(isInSchedulingRegion(*Bundle) &&
"ScheduleData not in scheduling region");
for_each(Bundle->getBundle(), ProcessNode);
}
if (InsertInReadyList && SD->isReady()) {
for (ScheduleBundle *Bundle : Bundles) {
assert(isInSchedulingRegion(*Bundle) &&
"ScheduleData not in scheduling region");
if (!Bundle->isReady())
continue;
ReadyInsts.insert(Bundle);
LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *Bundle
<< "\n");
}
}
}
}
void BoUpSLP::BlockScheduling::resetSchedule() {
assert(ScheduleStart &&
"tried to reset schedule on block which has not been scheduled");
for_each(ScheduleDataMap, [&](auto &P) {
if (BB != P.first->getParent())
return;
ScheduleData *SD = P.second;
if (isInSchedulingRegion(*SD)) {
SD->setScheduled(/*Scheduled=*/false);
SD->resetUnscheduledDeps();
}
});
for_each(ScheduleCopyableDataMapByInst, [&](auto &P) {
for_each(P.second, [&](ScheduleCopyableData *SD) {
if (isInSchedulingRegion(*SD)) {
SD->setScheduled(/*Scheduled=*/false);
SD->resetUnscheduledDeps();
}
});
});
for_each(ScheduledBundles, [&](auto &P) {
for_each(P.second, [&](ScheduleBundle *Bundle) {
if (isInSchedulingRegion(*Bundle))
Bundle->setScheduled(/*Scheduled=*/false);
});
});
// Reset schedule data for copyable elements.
for (auto &P : ScheduleCopyableDataMap) {
if (isInSchedulingRegion(*P.second)) {
P.second->setScheduled(/*Scheduled=*/false);
P.second->resetUnscheduledDeps();
}
}
ReadyInsts.clear();
}
void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
if (!BS->ScheduleStart)
return;
LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
// The scalars of the trimmed nodes are not vectorized and must not be
// grouped as bundles. Cancel their bundles, recalculate all the dependencies
// and schedule the scalars as separate instructions: they keep their
// original order and stay close to their users.
auto IsTrimmed = [&](const TreeEntry *TE) {
return DeletedNodes.contains(TE) || TransformedToGatherNodes.contains(TE);
};
const size_t NumBundles = BS->ScheduledBundlesList.size();
SmallPtrSet<const ScheduleData *, 16> ClearedSDs;
erase_if(BS->ScheduledBundlesList,
[&](const std::unique_ptr<ScheduleBundle> &Bundle) {
const TreeEntry *TE = Bundle->getTreeEntry();
if (!IsTrimmed(TE))
return false;
LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of the trimmed node "
<< TE->Idx << " bundle " << *Bundle << "\n");
for (ScheduleEntity *SE : Bundle->getBundle())
if (ScheduleData *SD = BS->getScheduleData(SE->getInst()))
ClearedSDs.insert(SD);
BS->cancelScheduling(*Bundle);
return true;
});
if (BS->ScheduledBundlesList.size() != NumBundles) {
// Need to schedule all the instructions with the calculated dependencies,
// not only the users of the remaining bundles, otherwise they are moved
// above all the scheduled instructions.
for (Instruction &I : make_range(BS->ScheduleStart->getIterator(),
BS->ScheduleEnd->getIterator()))
if (ScheduleData *SD = BS->getScheduleData(&I);
SD && SD->hasValidDependencies())
ClearedSDs.insert(SD);
BS->clearDependencies();
}
// A key point - if we got here, pre-scheduling was able to find a valid
// scheduling of the sub-graph of the scheduling window which consists
// of all vector bundles and their transitive users. As such, we do not
// need to reschedule anything *outside of* that subgraph.
BS->resetSchedule();
// For the real scheduling we use a more sophisticated ready-list: it is
// sorted by the original instruction location. This lets the final schedule
// be as close as possible to the original instruction order.
// WARNING: If changing this order causes a correctness issue, that means
// there is some missing dependence edge in the schedule data graph.
struct ScheduleDataCompare {
bool operator()(const ScheduleEntity *SD1,
const ScheduleEntity *SD2) const {
return SD2->getSchedulingPriority() < SD1->getSchedulingPriority();
}
};
std::set<ScheduleEntity *, ScheduleDataCompare> ReadyInsts;
// Ensure that all dependency data is updated (for nodes in the sub-graph)
// and fill the ready-list with initial instructions.
int Idx = 0;
for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
I = I->getNextNode()) {
ArrayRef<ScheduleBundle *> Bundles = BS->getScheduleBundles(I);
if (!Bundles.empty()) {
// An instruction that is modeled as a copyable element in some node may
// also be used directly by another node that was registered only after
// this instruction's dependencies were last computed. The deferred
// copyable-operand recomputation (RecalcCopyableOperandDeps) is consumed
// at the next bundle scheduling, which clears the operand's dependencies
// before that node joins the tree, so the recomputation misses the
// direct def-use edge and the count stays too low. The scheduler would
// then decrement the instruction more times than its dependency count,
// tripping the unscheduled-deps assertion. The whole tree is registered
// now, so clear and recompute such dependencies against the full tree.
if (ScheduleData *SD = BS->getScheduleData(I);
SD && SD->hasValidDependencies() &&
!BS->getScheduleCopyableData(I).empty())
SD->clearDirectDependencies();
for (ScheduleBundle *Bundle : Bundles) {
Bundle->setSchedulingPriority(Idx++);
if (const TreeEntry *TE = Bundle->getTreeEntry();
TE && Bundle->hasValidDependencies() && TE->UserTreeIndex &&
((TE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
TE->UserTreeIndex.UserTE->doesNotNeedToSchedule() &&
any_of(TE->UserTreeIndex.UserTE->Scalars,
[&](Value *V) {
return TE->UserTreeIndex.UserTE->isExpandedBinOp(V);
})) ||
(TE->UserTreeIndex.EdgeIdx == UINT_MAX &&
TE->UserTreeIndex.UserTE->isGather()))) {
for (ScheduleData *SD :
make_isa_range<ScheduleData>(Bundle->getBundle()))
SD->clearDirectDependencies();
}
if (!Bundle->hasValidDependencies()) {
SmallPtrSet<Value *, 4> ExpandedOps;
BS->calculateDependencies(*Bundle, /*InsertInReadyList=*/false, this,
ExpandedOps);
}
}
SmallVector<ScheduleCopyableData *> SDs = BS->getScheduleCopyableData(I);
for (ScheduleCopyableData *SD : reverse(SDs)) {
ScheduleBundle &Bundle = SD->getBundle();
Bundle.setSchedulingPriority(Idx++);
if (!Bundle.hasValidDependencies()) {
SmallPtrSet<Value *, 4> ExpandedOps;
BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
ExpandedOps);
}
}
continue;
}
SmallVector<ScheduleCopyableData *> CopyableData =
BS->getScheduleCopyableDataUsers(I);
if (ScheduleData *SD = BS->getScheduleData(I)) {
// The bundles of the trimmed nodes are cancelled, check the first
// remaining node only.
[[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(I);
[[maybe_unused]] const auto *SDTEIt = find_if_not(SDTEs, IsTrimmed);
assert((isVectorLikeInstWithConstOps(SD->getInst()) ||
SDTEIt == SDTEs.end() || (*SDTEIt)->doesNotNeedToSchedule() ||
doesNotNeedToBeScheduled(I)) &&
"scheduler and vectorizer bundle mismatch");
SD->setSchedulingPriority(Idx++);
if ((!SD->hasValidDependencies() && ClearedSDs.contains(SD)) ||
!CopyableData.empty() ||
any_of(R.ValueToGatherNodes.lookup(I), [&](const TreeEntry *TE) {
assert(TE->isGather() && "expected gather node");
return TE->hasState() && TE->hasCopyableElements() &&
TE->isCopyableElement(I);
})) {
SD->clearDirectDependencies();
// Need to calculate deps for these nodes to correctly handle copyable
// dependencies, even if they were cancelled.
// If copyables bundle was cancelled, the deps are cleared and need to
// recalculate them.
ScheduleBundle Bundle;
Bundle.add(SD);
SmallPtrSet<Value *, 4> ExpandedOps;
BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
ExpandedOps);
}
}
for (ScheduleCopyableData *SD : reverse(CopyableData)) {
ScheduleBundle &Bundle = SD->getBundle();
Bundle.setSchedulingPriority(Idx++);
if (!Bundle.hasValidDependencies()) {
SmallPtrSet<Value *, 4> ExpandedOps;
BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this,
ExpandedOps);
}
}
}
BS->initialFillReadyList(ReadyInsts);
Instruction *LastScheduledInst = BS->ScheduleEnd;
// Do the "real" scheduling.
SmallPtrSet<Instruction *, 16> Scheduled;
while (!ReadyInsts.empty()) {
auto *Picked = *ReadyInsts.begin();
ReadyInsts.erase(ReadyInsts.begin());
// Move the scheduled instruction(s) to their dedicated places, if not
// there yet.
if (auto *Bundle = dyn_cast<ScheduleBundle>(Picked)) {
for (const ScheduleEntity *BundleMember : Bundle->getBundle()) {
Instruction *PickedInst = BundleMember->getInst();
// If copyable must be schedule as part of something else, skip it.
bool IsCopyable = Bundle->getTreeEntry()->isCopyableElement(PickedInst);
if ((IsCopyable && BS->getScheduleData(PickedInst)) ||
(!IsCopyable && !Scheduled.insert(PickedInst).second))
continue;
if (PickedInst->getNextNode() != LastScheduledInst)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
}
if (Bundle->getTreeEntry()->hasCopyableElements()) {
Instruction *MainOp = Bundle->getTreeEntry()->getMainOp();
for (Instruction *I :
make_isa_range<Instruction>(Bundle->getTreeEntry()->Scalars))
if (!I->hasOneUse() && Bundle->getTreeEntry()->isCopyableElement(I) &&
I->getParent() == MainOp->getParent() &&
doesNotNeedToBeScheduled(I) &&
!Bundle->getTreeEntry()->getOperations().isNonSchedulable(I))
I->moveBeforePreserving(LastScheduledInst->getIterator());
}
EntryToLastInstruction.try_emplace(Bundle->getTreeEntry(),
LastScheduledInst);
} else {
auto *SD = cast<ScheduleData>(Picked);
Instruction *PickedInst = SD->getInst();
if (PickedInst->getNextNode() != LastScheduledInst)
PickedInst->moveAfter(LastScheduledInst->getPrevNode());
LastScheduledInst = PickedInst;
}
auto Invalid = InstructionsState::invalid();
BS->schedule(R, Invalid, EdgeInfo(), Picked, ReadyInsts);
}
// Check that we didn't break any of our invariants.
#ifdef EXPENSIVE_CHECKS
BS->verify();
#endif
#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
// Check that all schedulable entities got scheduled
for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
I = I->getNextNode()) {
ArrayRef<ScheduleBundle *> Bundles = BS->getScheduleBundles(I);
assert(all_of(Bundles,
[](const ScheduleBundle *Bundle) {
return Bundle->isScheduled();
}) &&
"must be scheduled at this point");
}
#endif
// Avoid duplicate scheduling of the block.
BS->ScheduleStart = nullptr;
}
unsigned BoUpSLP::getVectorElementSize(Value *V) {
// If V is a store, just return the width of the stored value (or value
// truncated just before storing) without traversing the expression tree.
// This is the common case.
if (auto *Store = dyn_cast<StoreInst>(V))
return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
if (auto *IEI = dyn_cast<InsertElementInst>(V))
return getVectorElementSize(IEI->getOperand(1));
if (auto *IVI = dyn_cast<InsertValueInst>(V))
return getVectorElementSize(IVI->getOperand(1));
auto E = InstrElementSize.find(V);
if (E != InstrElementSize.end())
return E->second;
// If V is not a store, we can traverse the expression tree to find loads
// that feed it. The type of the loaded value may indicate a more suitable
// width than V's type. We want to base the vector element size on the width
// of memory operations where possible.
SmallVector<std::tuple<Instruction *, BasicBlock *, unsigned>> Worklist;
SmallPtrSet<Instruction *, 16> Visited;
if (auto *I = dyn_cast<Instruction>(V)) {
Worklist.emplace_back(I, I->getParent(), 0);
Visited.insert(I);
}
// Traverse the expression tree in bottom-up order looking for loads. If we
// encounter an instruction we don't yet handle, we give up.
auto Width = 0u;
Value *FirstNonBool = nullptr;
while (!Worklist.empty()) {
auto [I, Parent, Level] = Worklist.pop_back_val();
// We should only be looking at scalar instructions here. If the current
// instruction has a vector type, skip.
auto *Ty = I->getType();
if (isa<VectorType>(Ty))
continue;
if (Ty != Builder.getInt1Ty() && !FirstNonBool)
FirstNonBool = I;
if (Level > Opts.slp_recursion_max_depth)
continue;
// Can we propagate widths through a call instruction
// We can for simple intrinsics
auto IsCompatibleIntrinsic = [](const Instruction *I) -> bool {
const auto *CI = dyn_cast<IntrinsicInst>(I);
if (!CI)
return false;
switch (CI->getIntrinsicID()) {
case Intrinsic::abs:
case Intrinsic::fabs:
case Intrinsic::fma:
case Intrinsic::fmuladd:
return true;
default:
return false;
}
};
// If the current instruction is a load, update MaxWidth to reflect the
// width of the loaded value.
if (isa<LoadInst, ExtractElementInst, ExtractValueInst>(I))
Width = std::max<unsigned>(Width, DL->getTypeSizeInBits(Ty));
// Otherwise, we need to visit the operands of the instruction. We only
// handle the interesting cases from buildTree here. If an operand is an
// instruction we haven't yet visited and from the same basic block as the
// user or the use is a PHI node, we add it to the worklist.
else if (isa<PHINode, CastInst, GetElementPtrInst, CmpInst, SelectInst,
BinaryOperator, UnaryOperator>(I) ||
IsCompatibleIntrinsic(I)) {
for (Use &U : I->operands()) {
if (auto *J = dyn_cast<Instruction>(U.get()))
if (Visited.insert(J).second &&
(isa<PHINode>(I) || J->getParent() == Parent)) {
Worklist.emplace_back(J, J->getParent(), Level + 1);
continue;
}
if (!FirstNonBool && U.get()->getType() != Builder.getInt1Ty())
FirstNonBool = U.get();
}
} else {
break;
}
}
// If we didn't encounter a memory access in the expression tree, or if we
// gave up for some reason, just return the width of V. Otherwise, return the
// maximum width we found.
if (!Width) {
if (V->getType() == Builder.getInt1Ty() && FirstNonBool)
V = FirstNonBool;
Width = DL->getTypeSizeInBits(V->getType());
}
for (Instruction *I : Visited)
InstrElementSize[I] = Width;
return Width;
}
bool BoUpSLP::collectValuesToDemote(
const TreeEntry &E, bool IsProfitableToDemoteRoot, unsigned &BitWidth,
SmallVectorImpl<unsigned> &ToDemote, DenseSet<const TreeEntry *> &Visited,
const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
bool &IsProfitableToDemote, bool IsTruncRoot) const {
// We can always demote constants.
if (all_of(E.Scalars, IsaPred<Constant>))
return true;
unsigned OrigBitWidth =
DL->getTypeSizeInBits(E.Scalars.front()->getType()->getScalarType());
if (OrigBitWidth == BitWidth) {
MaxDepthLevel = 1;
return true;
}
// Check if the node was analyzed already and must keep its original bitwidth.
if (NodesToKeepBWs.contains(E.Idx))
return false;
// If the value is not a vectorized instruction in the expression and not used
// by the insertelement instruction and not used in multiple vector nodes, it
// cannot be demoted.
bool IsSignedNode = any_of(E.Scalars, [&](Value *R) {
if (isa<PoisonValue>(R))
return false;
return !isKnownNonNegative(R, SimplifyQuery(*DL));
});
auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool {
if (isa<PoisonValue>(V))
return true;
if (getTreeEntries(V).size() > 1)
return false;
// For lat shuffle of sext/zext with many uses need to check the extra bit
// for unsigned values, otherwise may have incorrect casting for reused
// scalars.
bool IsSignedVal = !isKnownNonNegative(V, SimplifyQuery(*DL));
if ((!IsSignedNode || IsSignedVal) && OrigBitWidth > BitWidth) {
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL)))
return true;
}
unsigned NumSignBits = ComputeNumSignBits(V, *DL, AC, nullptr, DT);
unsigned BitWidth1 = OrigBitWidth - NumSignBits;
if (IsSignedNode)
++BitWidth1;
if (auto *I = dyn_cast<Instruction>(V)) {
APInt Mask = DB->getDemandedBits(I);
unsigned BitWidth2 =
std::max<unsigned>(1, Mask.getBitWidth() - Mask.countl_zero());
while (!IsSignedNode && BitWidth2 < OrigBitWidth) {
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth2 - 1);
if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL)))
break;
BitWidth2 *= 2;
}
BitWidth1 = std::min(BitWidth1, BitWidth2);
}
BitWidth = std::max(BitWidth, BitWidth1);
return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2);
};
auto FinalAnalysis = [&]() {
if (!IsProfitableToDemote)
return false;
bool Res = all_of(
E.Scalars, std::bind(IsPotentiallyTruncated, _1, std::ref(BitWidth)));
// Demote gathers.
if (Res && E.isGather()) {
if (E.hasState()) {
if (const TreeEntry *SameTE =
getSameValuesTreeEntry(E.getMainOp(), E.Scalars))
if (collectValuesToDemote(*SameTE, IsProfitableToDemoteRoot, BitWidth,
ToDemote, Visited, NodesToKeepBWs,
MaxDepthLevel, IsProfitableToDemote,
IsTruncRoot)) {
ToDemote.push_back(E.Idx);
return true;
}
}
// Check possible extractelement instructions bases and final vector
// length.
SmallPtrSet<Value *, 4> UniqueBases;
for (ExtractElementInst *EE :
make_isa_range<ExtractElementInst>(E.Scalars))
UniqueBases.insert(EE->getVectorOperand());
const unsigned VF = E.Scalars.size();
Type *OrigScalarTy = E.Scalars.front()->getType();
if (UniqueBases.size() <= 2 ||
getNumberOfParts(getWidenedType(OrigScalarTy, VF), OrigScalarTy) >=
getNumberOfParts(
getWidenedType(
IntegerType::get(OrigScalarTy->getContext(), BitWidth),
VF),
IntegerType::get(OrigScalarTy->getContext(), BitWidth))) {
ToDemote.push_back(E.Idx);
return true;
}
}
return Res;
};
if (E.isGather() || !Visited.insert(&E).second ||
any_of(E.Scalars, [&](Value *V) {
return !isa<Constant>(V) && all_of(V->users(), [&](User *U) {
return isa<InsertElementInst>(U) && !isVectorized(U);
});
}))
return FinalAnalysis();
if (any_of(E.Scalars, [&](Value *V) {
return !isa<Constant>(V) && !all_of(V->users(), [=](User *U) {
return isVectorized(U) ||
(E.Idx == 0 && UserIgnoreList &&
UserIgnoreList->contains(U)) ||
(!isa<CmpInst>(U) && U->getType()->isSized() &&
!U->getType()->isScalableTy() &&
DL->getTypeSizeInBits(U->getType()) <= BitWidth);
}) && !IsPotentiallyTruncated(V, BitWidth);
}))
return false;
auto ProcessOperands = [&](ArrayRef<const TreeEntry *> Operands,
bool &NeedToExit) {
NeedToExit = false;
unsigned InitLevel = MaxDepthLevel;
for (const TreeEntry *Op : Operands) {
unsigned Level = InitLevel;
if (!collectValuesToDemote(*Op, IsProfitableToDemoteRoot, BitWidth,
ToDemote, Visited, NodesToKeepBWs, Level,
IsProfitableToDemote, IsTruncRoot)) {
if (!IsProfitableToDemote)
return false;
NeedToExit = true;
if (!FinalAnalysis())
return false;
continue;
}
MaxDepthLevel = std::max(MaxDepthLevel, Level);
}
return true;
};
auto AttemptCheckBitwidth =
[&](function_ref<bool(unsigned, unsigned)> Checker, bool &NeedToExit) {
// Try all bitwidth < OrigBitWidth.
NeedToExit = false;
unsigned BestFailBitwidth = 0;
for (; BitWidth < OrigBitWidth; BitWidth *= 2) {
if (Checker(BitWidth, OrigBitWidth))
return true;
if (BestFailBitwidth == 0 && FinalAnalysis())
BestFailBitwidth = BitWidth;
}
if (BitWidth >= OrigBitWidth) {
if (BestFailBitwidth == 0) {
BitWidth = OrigBitWidth;
return false;
}
MaxDepthLevel = 1;
BitWidth = BestFailBitwidth;
NeedToExit = true;
return true;
}
return false;
};
auto TryProcessInstruction =
[&](unsigned &BitWidth, ArrayRef<const TreeEntry *> Operands = {},
function_ref<bool(unsigned, unsigned)> Checker = {}) {
if (Operands.empty()) {
if (!IsTruncRoot)
MaxDepthLevel = 1;
for (Value *V : E.Scalars)
(void)IsPotentiallyTruncated(V, BitWidth);
} else {
// Several vectorized uses? Check if we can truncate it, otherwise -
// exit.
if (any_of(E.Scalars, [&](Value *V) {
return !V->hasOneUse() && !IsPotentiallyTruncated(V, BitWidth);
}))
return false;
bool NeedToExit = false;
if (Checker && !AttemptCheckBitwidth(Checker, NeedToExit))
return false;
if (NeedToExit)
return true;
if (!ProcessOperands(Operands, NeedToExit))
return false;
if (NeedToExit)
return true;
}
++MaxDepthLevel;
// Record the entry that we can demote.
ToDemote.push_back(E.Idx);
return IsProfitableToDemote;
};
if (E.State == TreeEntry::SplitVectorize)
return TryProcessInstruction(
BitWidth,
{VectorizableTree[E.CombinedEntriesWithIndices.front().first].get(),
VectorizableTree[E.CombinedEntriesWithIndices.back().first].get()});
if (E.isAltShuffle()) {
// Combining these opcodes may lead to incorrect analysis, skip for now.
auto IsDangerousOpcode = [](unsigned Opcode) {
switch (Opcode) {
case Instruction::Shl:
case Instruction::AShr:
case Instruction::LShr:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem:
return true;
default:
break;
}
return false;
};
if (IsDangerousOpcode(E.getAltOpcode()))
return FinalAnalysis();
}
switch (E.getOpcode()) {
// We can always demote truncations and extensions. Since truncations can
// seed additional demotion, we save the truncated value.
case Instruction::Trunc:
if (IsProfitableToDemoteRoot)
IsProfitableToDemote = true;
return TryProcessInstruction(BitWidth);
case Instruction::ZExt:
case Instruction::SExt:
if (E.UserTreeIndex.UserTE && E.UserTreeIndex.UserTE->hasState() &&
E.UserTreeIndex.UserTE->getOpcode() == Instruction::BitCast &&
E.UserTreeIndex.UserTE->getMainOp()->getType()->isFPOrFPVectorTy())
return false;
IsProfitableToDemote = true;
return TryProcessInstruction(BitWidth);
// Demote binary ops when all operands can be demoted (may be >2 operands).
case Instruction::Add:
case Instruction::Sub:
case Instruction::Mul:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
return TryProcessInstruction(
BitWidth, map_to_vector(seq<unsigned>(E.getNumOperands()),
[&](unsigned Idx) -> const TreeEntry * {
return getOperandEntry(&E, Idx);
}));
}
case Instruction::Freeze:
return TryProcessInstruction(BitWidth, getOperandEntry(&E, 0));
case Instruction::Shl: {
// If we are truncating the result of this SHL, and if it's a shift of an
// inrange amount, we can always perform a SHL in a smaller type.
auto ShlChecker = [&](unsigned BitWidth, unsigned) {
// Check the node operands rather than the scalar instructions: lanes
// converted from another opcode (e.g. mul by a power of 2) are emitted
// with the converted shift amounts.
return all_of(E.getOperand(1), [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
KnownBits AmtKnownBits = computeKnownBits(V, *DL);
return AmtKnownBits.getMaxValue().ult(BitWidth);
});
};
return TryProcessInstruction(
BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)}, ShlChecker);
}
case Instruction::LShr: {
// If this is a truncate of a logical shr, we can truncate it to a smaller
// lshr iff we know that the bits we would otherwise be shifting in are
// already zeros.
auto LShrChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
return all_of(E.Scalars, [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
if (E.isCopyableElement(V))
return MaskedValueIsZero(V, ShiftedBits, SimplifyQuery(*DL));
auto *I = cast<Instruction>(V);
KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
return AmtKnownBits.getMaxValue().ult(BitWidth) &&
MaskedValueIsZero(I->getOperand(0), ShiftedBits,
SimplifyQuery(*DL));
});
};
return TryProcessInstruction(
BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)},
LShrChecker);
}
case Instruction::AShr: {
// If this is a truncate of an arithmetic shr, we can truncate it to a
// smaller ashr iff we know that all the bits from the sign bit of the
// original type and the sign bit of the truncate type are similar.
auto AShrChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
return all_of(E.Scalars, [&](Value *V) {
if (isa<PoisonValue>(V))
return true;
unsigned ShiftedBits = OrigBitWidth - BitWidth;
if (E.isCopyableElement(V))
return ShiftedBits < ComputeNumSignBits(V, *DL, AC, nullptr, DT);
auto *I = cast<Instruction>(V);
KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
return AmtKnownBits.getMaxValue().ult(BitWidth) &&
ShiftedBits <
ComputeNumSignBits(I->getOperand(0), *DL, AC, nullptr, DT);
});
};
return TryProcessInstruction(
BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)},
AShrChecker);
}
case Instruction::UDiv:
case Instruction::URem: {
// UDiv and URem can be truncated if all the truncated bits are zero.
auto Checker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
assert(BitWidth <= OrigBitWidth && "Unexpected bitwidths!");
return all_of(E.Scalars, [&](Value *V) {
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
if (E.hasCopyableElements() && E.isCopyableElement(V))
return MaskedValueIsZero(V, Mask, SimplifyQuery(*DL));
auto *I = cast<Instruction>(V);
return MaskedValueIsZero(I->getOperand(0), Mask, SimplifyQuery(*DL)) &&
MaskedValueIsZero(I->getOperand(1), Mask, SimplifyQuery(*DL));
});
};
return TryProcessInstruction(
BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)}, Checker);
}
// We can demote selects if we can demote their true and false values.
case Instruction::Select: {
return TryProcessInstruction(
BitWidth, {getOperandEntry(&E, 1), getOperandEntry(&E, 2)});
}
// We can demote phis if we can demote all their incoming operands.
case Instruction::PHI: {
const unsigned NumOps = E.getNumOperands();
SmallVector<const TreeEntry *> Ops(NumOps);
transform(seq<unsigned>(0, NumOps), Ops.begin(),
[&](unsigned Idx) { return getOperandEntry(&E, Idx); });
return TryProcessInstruction(BitWidth, Ops);
}
case Instruction::Call: {
auto *IC = dyn_cast<IntrinsicInst>(E.getMainOp());
if (!IC)
break;
Intrinsic::ID ID = getVectorIntrinsicIDForCall(IC, TLI);
if (ID != Intrinsic::abs && ID != Intrinsic::smin &&
ID != Intrinsic::smax && ID != Intrinsic::umin && ID != Intrinsic::umax)
break;
SmallVector<const TreeEntry *, 2> Operands(1, getOperandEntry(&E, 0));
function_ref<bool(unsigned, unsigned)> CallChecker;
auto CompChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
assert(BitWidth <= OrigBitWidth && "Unexpected bitwidths!");
return all_of(E.Scalars, [&](Value *V) {
if (ID == Intrinsic::umin || ID == Intrinsic::umax) {
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
// A copyable V stands for umax/umin(V, identity), which equals V.
if (E.isCopyableElement(V))
return MaskedValueIsZero(V, Mask, SimplifyQuery(*DL));
auto *I = cast<Instruction>(V);
return MaskedValueIsZero(I->getOperand(0), Mask,
SimplifyQuery(*DL)) &&
MaskedValueIsZero(I->getOperand(1), Mask, SimplifyQuery(*DL));
}
assert((ID == Intrinsic::smin || ID == Intrinsic::smax) &&
"Expected min/max intrinsics only.");
unsigned SignBits = OrigBitWidth - BitWidth;
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth - 1);
auto CheckOp = [&](Value *Op) {
unsigned OpSignBits = ComputeNumSignBits(Op, *DL, AC, nullptr, DT);
return SignBits <= OpSignBits &&
((SignBits != OpSignBits &&
!isKnownNonNegative(Op, SimplifyQuery(*DL))) ||
MaskedValueIsZero(Op, Mask, SimplifyQuery(*DL)));
};
// A copyable V stands for smax/smin(V, identity), which equals V.
if (E.isCopyableElement(V))
return CheckOp(V) &&
(isTypeLimitNeutral(V, ID == Intrinsic::smax, *DL) ||
ComputeMaxSignificantBits(V, *DL, nullptr, nullptr, DT) <=
BitWidth);
auto *I = cast<Instruction>(V);
return CheckOp(I->getOperand(0)) && CheckOp(I->getOperand(1));
});
};
auto AbsChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) {
assert(BitWidth <= OrigBitWidth && "Unexpected bitwidths!");
return all_of(E.Scalars, [&](Value *V) {
auto *I = cast<Instruction>(V);
unsigned SignBits = OrigBitWidth - BitWidth;
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth - 1);
unsigned Op0SignBits =
ComputeNumSignBits(I->getOperand(0), *DL, AC, nullptr, DT);
return SignBits <= Op0SignBits &&
((SignBits != Op0SignBits &&
!isKnownNonNegative(I->getOperand(0), SimplifyQuery(*DL))) ||
MaskedValueIsZero(I->getOperand(0), Mask, SimplifyQuery(*DL)));
});
};
if (ID != Intrinsic::abs) {
Operands.push_back(getOperandEntry(&E, 1));
CallChecker = CompChecker;
} else {
CallChecker = AbsChecker;
}
InstructionCost BestCost =
std::numeric_limits<InstructionCost::CostType>::max();
unsigned BestBitWidth = BitWidth;
unsigned VF = E.Scalars.size();
// Choose the best bitwidth based on cost estimations.
auto Checker = [&](unsigned BitWidth, unsigned) {
unsigned MinBW = PowerOf2Ceil(BitWidth);
SmallVector<Type *> ArgTys =
buildIntrinsicArgTypes(IC, ID, VF, MinBW, TTI);
auto VecCallCosts = getVectorCallCosts(
IC, getWidenedType(IntegerType::get(IC->getContext(), MinBW), VF),
TTI, TLI, ArgTys, CostKind);
InstructionCost Cost = std::min(VecCallCosts.first, VecCallCosts.second);
if (Cost < BestCost) {
BestCost = Cost;
BestBitWidth = BitWidth;
}
return false;
};
[[maybe_unused]] bool NeedToExit;
(void)AttemptCheckBitwidth(Checker, NeedToExit);
BitWidth = BestBitWidth;
return TryProcessInstruction(BitWidth, Operands, CallChecker);
}
// Otherwise, conservatively give up.
default:
break;
}
MaxDepthLevel = 1;
return FinalAnalysis();
}
static RecurKind getRdxKind(Value *V);
void BoUpSLP::computeMinimumValueSizes() {
// Detect booleanized i1 and/or nodes first: they are registered in MinBWs
// like the demoted nodes and processed the same way.
detectBooleanizedNodes();
// We only attempt to truncate integer expressions.
bool IsStoreOrInsertElt =
getRootNode().hasState() &&
(getRootNode().getOpcode() == Instruction::Store ||
getRootNode().getOpcode() == Instruction::InsertElement ||
getRootNode().getOpcode() == Instruction::InsertValue);
if ((IsStoreOrInsertElt || UserIgnoreList) &&
ExtraBitWidthNodes.size() <= 1 &&
(!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 ||
CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2))
return;
unsigned NodeIdx = 0;
if (IsStoreOrInsertElt && !getRootNode().isGather())
NodeIdx = 1;
// Ensure the roots of the vectorizable tree don't form a cycle.
assert((VectorizableTree[NodeIdx]->isGather() || NodeIdx != 0 ||
!VectorizableTree[NodeIdx]->UserTreeIndex) &&
"Unexpected tree is graph.");
// The first value node for store/insertelement is sext/zext/trunc? Skip it,
// resize to the final type.
bool IsTruncRoot = false;
bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
SmallVector<unsigned> RootDemotes;
SmallDenseSet<unsigned, 8> NodesToKeepBWs;
if (NodeIdx != 0 &&
VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
assert(IsStoreOrInsertElt && "Expected store/insertelement seeded graph.");
IsTruncRoot = true;
RootDemotes.push_back(NodeIdx);
IsProfitableToDemoteRoot = true;
++NodeIdx;
}
auto IsAnalyzedMinBWVal = [&](Value *V, unsigned SizeExGathers) {
auto It = AnalyzedMinBWVals.find(V);
return It != AnalyzedMinBWVals.end() && It->second >= SizeExGathers;
};
// Analyzed the reduction already and not profitable - exit.
if (IsAnalyzedMinBWVal(VectorizableTree[NodeIdx]->Scalars.front(),
getTreeSizeExcludingGathers()))
return;
SmallVector<unsigned> ToDemote;
auto ComputeMaxBitWidth =
[&](const TreeEntry &E, bool IsTopRoot, bool IsProfitableToDemoteRoot,
unsigned Limit, bool IsTruncRoot, bool IsSignedCmp) -> unsigned {
ToDemote.clear();
// Check if the root is trunc and the next node is gather/buildvector, then
// keep trunc in scalars, which is free in most cases.
if (E.isGather() && IsTruncRoot && E.UserTreeIndex &&
!NodesToKeepBWs.contains(E.Idx) &&
E.Idx > (IsStoreOrInsertElt ? 2u : 1u) &&
all_of(E.Scalars, [&](Value *V) {
return V->hasOneUse() || isa<Constant>(V) ||
(!V->hasNUsesOrMore(UsesLimit) &&
none_of(V->users(), [&](User *U) {
ArrayRef<TreeEntry *> TEs = getTreeEntries(U);
const TreeEntry *UserTE = E.UserTreeIndex.UserTE;
if (TEs.empty() || is_contained(TEs, UserTE))
return false;
if (!isa<CastInst, BinaryOperator, FreezeInst, PHINode,
SelectInst>(U) ||
isa<SIToFPInst, UIToFPInst>(U) ||
(UserTE->hasState() &&
(!isa<CastInst, BinaryOperator, FreezeInst, PHINode,
SelectInst>(UserTE->getMainOp()) ||
isa<SIToFPInst, UIToFPInst>(UserTE->getMainOp()))))
return true;
unsigned UserTESz = DL->getTypeSizeInBits(
UserTE->Scalars.front()->getType());
if (all_of(TEs, [&](const TreeEntry *TE) {
auto It = MinBWs.find(TE);
return It != MinBWs.end() &&
It->second.first > UserTESz;
}))
return true;
return DL->getTypeSizeInBits(U->getType()) > UserTESz;
}));
})) {
ToDemote.push_back(E.Idx);
const TreeEntry *UserTE = E.UserTreeIndex.UserTE;
auto It = MinBWs.find(UserTE);
if (It != MinBWs.end())
return It->second.first;
unsigned MaxBitWidth =
DL->getTypeSizeInBits(UserTE->Scalars.front()->getType());
MaxBitWidth = bit_ceil(MaxBitWidth);
if (MaxBitWidth < 8 && MaxBitWidth > 1)
MaxBitWidth = 8;
return MaxBitWidth;
}
if (!E.hasState())
return 0u;
unsigned VF = E.getVectorFactor();
Type *ScalarTy = E.Scalars.front()->getType();
auto *TreeRootIT = dyn_cast<IntegerType>(ScalarTy->getScalarType());
if (!TreeRootIT)
return 0u;
unsigned SizeExGathers = getTreeSizeExcludingGathers();
if (any_of(E.Scalars,
[&](Value *V) { return IsAnalyzedMinBWVal(V, SizeExGathers); }))
return 0u;
unsigned NumParts =
getNumberOfParts(getWidenedType(ScalarTy, VF), ScalarTy);
// The maximum bit width required to represent all the values that can be
// demoted without loss of precision. It would be safe to truncate the roots
// of the expression to this width.
unsigned MaxBitWidth = 1u;
// True if the roots can be zero-extended back to their original type,
// rather than sign-extended. We know that if the leading bits are not
// demanded, we can safely zero-extend. So we initialize IsKnownPositive to
// True.
// Determine if the sign bit of all the roots is known to be zero. If not,
// IsKnownPositive is set to False.
bool IsKnownPositive = !IsSignedCmp && all_of(E.Scalars, [&](Value *R) {
if (isa<PoisonValue>(R))
return true;
KnownBits Known = computeKnownBits(R, *DL);
return Known.isNonNegative();
});
if (!IsKnownPositive && !IsTopRoot && E.UserTreeIndex &&
E.UserTreeIndex.UserTE->hasState() &&
E.UserTreeIndex.UserTE->getOpcode() == Instruction::UIToFP)
MaxBitWidth =
std::min(DL->getTypeSizeInBits(
E.UserTreeIndex.UserTE->Scalars.front()->getType()),
DL->getTypeSizeInBits(ScalarTy));
// We first check if all the bits of the roots are demanded. If they're not,
// we can truncate the roots to this narrower type.
auto IsNotPoison = [](Value *Root) { return !isa<PoisonValue>(Root); };
for (Value *Root : make_filter_range(E.Scalars, IsNotPoison)) {
unsigned NumSignBits = ComputeNumSignBits(Root, *DL, AC, nullptr, DT);
TypeSize NumTypeBits =
DL->getTypeSizeInBits(Root->getType()->getScalarType());
unsigned BitWidth1 = NumTypeBits - NumSignBits;
// If we can't prove that the sign bit is zero, we must add one to the
// maximum bit width to account for the unknown sign bit. This preserves
// the existing sign bit so we can safely sign-extend the root back to the
// original type. Otherwise, if we know the sign bit is zero, we will
// zero-extend the root instead.
//
// FIXME: This is somewhat suboptimal, as there will be cases where adding
// one to the maximum bit width will yield a larger-than-necessary
// type. In general, we need to add an extra bit only if we can't
// prove that the upper bit of the original type is equal to the
// upper bit of the proposed smaller type. If these two bits are
// the same (either zero or one) we know that sign-extending from
// the smaller type will result in the same value. Here, since we
// can't yet prove this, we are just making the proposed smaller
// type larger to ensure correctness.
if (!IsKnownPositive)
++BitWidth1;
auto *I = dyn_cast<Instruction>(Root);
if (!I) {
MaxBitWidth = std::max(BitWidth1, MaxBitWidth);
continue;
}
APInt Mask = DB->getDemandedBits(I);
unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
MaxBitWidth =
std::max<unsigned>(std::min(BitWidth1, BitWidth2), MaxBitWidth);
}
if (MaxBitWidth < 8 && MaxBitWidth > 1)
MaxBitWidth = 8;
// If the original type is large, but reduced type does not improve the reg
// use - ignore it.
if (NumParts > 1 &&
NumParts ==
getNumberOfParts(
getWidenedType(
IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)),
VF),
IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth))))
return 0u;
unsigned Opcode = E.getOpcode();
bool IsProfitableToDemote = Opcode == Instruction::Trunc ||
Opcode == Instruction::SExt ||
Opcode == Instruction::ZExt || NumParts > 1;
// Conservatively determine if we can actually truncate the roots of the
// expression. Collect the values that can be demoted in ToDemote and
// additional roots that require investigating in Roots.
DenseSet<const TreeEntry *> Visited;
unsigned MaxDepthLevel = IsTruncRoot ? Limit : 1;
bool NeedToDemote = IsProfitableToDemote;
if (!collectValuesToDemote(E, IsProfitableToDemoteRoot, MaxBitWidth,
ToDemote, Visited, NodesToKeepBWs, MaxDepthLevel,
NeedToDemote, IsTruncRoot) ||
(MaxDepthLevel <= Limit &&
!(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
(!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
DL->getTypeSizeInBits(TreeRootIT) /
DL->getTypeSizeInBits(
E.getMainOp()->getOperand(0)->getType()) >
2)))))
return 0u;
// Round MaxBitWidth up to the next power-of-two.
MaxBitWidth = bit_ceil(MaxBitWidth);
return MaxBitWidth;
};
// If we can truncate the root, we must collect additional values that might
// be demoted as a result. That is, those seeded by truncations we will
// modify.
// Add reduction ops sizes, if any.
if (UserIgnoreList &&
isa<IntegerType>(getRootNodeScalars().front()->getType())) {
// Convert vector_reduce_add(ZExt(<n x i1>)) to ZExtOrTrunc(ctpop(bitcast <n
// x i1> to in)).
if (all_of(make_isa_range<Instruction>(*UserIgnoreList),
[](Instruction *I) {
return I->getOpcode() == Instruction::Add;
}) &&
getRootNode().State == TreeEntry::Vectorize &&
getRootNode().getOpcode() == Instruction::ZExt &&
cast<CastInst>(getRootNode().getMainOp())->getSrcTy() ==
Builder.getInt1Ty()) {
ReductionBitWidth = 1;
} else {
for (Value *V : make_filter_range(*UserIgnoreList, [](Value *V) {
return !isa<PoisonValue>(V);
})) {
unsigned NumSignBits = ComputeNumSignBits(V, *DL, AC, nullptr, DT);
TypeSize NumTypeBits = DL->getTypeSizeInBits(V->getType());
unsigned BitWidth1 = NumTypeBits - NumSignBits;
if (!isKnownNonNegative(V, SimplifyQuery(*DL)))
++BitWidth1;
unsigned BitWidth2 = BitWidth1;
if (!RecurrenceDescriptor::isIntMinMaxRecurrenceKind(::getRdxKind(V))) {
APInt Mask = DB->getDemandedBits(cast<Instruction>(V));
BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
}
ReductionBitWidth =
std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth);
}
if (ReductionBitWidth < 8 && ReductionBitWidth > 1)
ReductionBitWidth = 8;
ReductionBitWidth = bit_ceil(ReductionBitWidth);
}
}
bool IsTopRoot = NodeIdx == 0;
while (NodeIdx < VectorizableTree.size() &&
VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
RootDemotes.push_back(NodeIdx);
++NodeIdx;
IsTruncRoot = true;
}
bool IsSignedCmp = false;
if (UserIgnoreList &&
all_of(*UserIgnoreList,
match_fn(m_CombineOr(m_SMin(m_Value(), m_Value()),
m_SMax(m_Value(), m_Value())))))
IsSignedCmp = true;
while (NodeIdx < VectorizableTree.size()) {
ArrayRef<Value *> TreeRoot = VectorizableTree[NodeIdx]->Scalars;
unsigned Limit = 2;
if (IsTopRoot &&
ReductionBitWidth ==
DL->getTypeSizeInBits(getRootNodeScalars().front()->getType()))
Limit = 3;
unsigned MaxBitWidth = ComputeMaxBitWidth(
*VectorizableTree[NodeIdx], IsTopRoot, IsProfitableToDemoteRoot, Limit,
IsTruncRoot, IsSignedCmp);
// The extension of a demoted scalar, which does not fit into the demoted
// type, restores only its demanded bits. Keep the original bitwidth if the
// scalar users may depend on the other bits via poison-generating
// annotations or there are too many users to check.
auto HasUnsafeScalarUsers = [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return false;
unsigned NumBits =
isKnownNonNegative(I, SimplifyQuery(*DL))
? computeKnownBits(I, *DL, AC, nullptr, DT).countMaxActiveBits()
: ComputeMaxSignificantBits(I, *DL, AC, nullptr, DT);
if (NumBits <= MaxBitWidth)
return false;
SmallVector<Instruction *> Worklist(1, I);
SmallPtrSet<Instruction *, 8> Visited;
int NumUsers = 0;
while (!Worklist.empty()) {
for (User *U : Worklist.pop_back_val()->users()) {
if (++NumUsers > UsesLimit)
return true;
auto *UI = cast<Instruction>(U);
if (isVectorized(UI) ||
(UserIgnoreList && UserIgnoreList->contains(UI)) ||
!UI->getType()->isIntOrIntVectorTy() ||
!Visited.insert(UI).second)
continue;
if (UI->hasPoisonGeneratingAnnotations())
return true;
if (!DB->getDemandedBits(UI).isAllOnes())
Worklist.push_back(UI);
}
}
return false;
};
if (MaxBitWidth != 0 && any_of(ToDemote, [&](unsigned Idx) {
const TreeEntry &TE = *VectorizableTree[Idx];
return !TE.isGather() && !MinBWs.contains(&TE) &&
any_of(TE.Scalars, HasUnsafeScalarUsers);
}))
MaxBitWidth = 0;
if (ReductionBitWidth != 0 && (IsTopRoot || !RootDemotes.empty())) {
if (MaxBitWidth != 0 && ReductionBitWidth < MaxBitWidth)
ReductionBitWidth = bit_ceil(MaxBitWidth);
else if (MaxBitWidth == 0)
ReductionBitWidth = 0;
}
for (unsigned Idx : RootDemotes) {
if (all_of(VectorizableTree[Idx]->Scalars, [&](Value *V) {
uint32_t OrigBitWidth =
DL->getTypeSizeInBits(V->getType()->getScalarType());
if (OrigBitWidth > MaxBitWidth) {
APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, MaxBitWidth);
return MaskedValueIsZero(V, Mask, SimplifyQuery(*DL));
}
return false;
}))
ToDemote.push_back(Idx);
}
RootDemotes.clear();
IsTopRoot = false;
IsProfitableToDemoteRoot = true;
if (ExtraBitWidthNodes.empty()) {
NodeIdx = VectorizableTree.size();
} else {
unsigned NewIdx = 0;
do {
NewIdx = *ExtraBitWidthNodes.begin();
ExtraBitWidthNodes.erase(ExtraBitWidthNodes.begin());
} while (NewIdx <= NodeIdx && !ExtraBitWidthNodes.empty());
NodeIdx = NewIdx;
IsTruncRoot =
NodeIdx < VectorizableTree.size() &&
VectorizableTree[NodeIdx]->UserTreeIndex &&
VectorizableTree[NodeIdx]->UserTreeIndex.EdgeIdx == 0 &&
VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->hasState() &&
VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->getOpcode() ==
Instruction::Trunc &&
!VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->isAltShuffle();
IsSignedCmp =
NodeIdx < VectorizableTree.size() &&
VectorizableTree[NodeIdx]->UserTreeIndex &&
VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->hasState() &&
VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->getOpcode() ==
Instruction::ICmp &&
any_of(
make_isa_range<ICmpInst>(
VectorizableTree[NodeIdx]->UserTreeIndex.UserTE->Scalars),
[&](ICmpInst *IC) {
return IC->isSigned() ||
!isKnownNonNegative(IC->getOperand(0),
SimplifyQuery(*DL)) ||
!isKnownNonNegative(IC->getOperand(1),
SimplifyQuery(*DL));
});
}
// If the maximum bit width we compute is less than the width of the roots'
// type, we can proceed with the narrowing. Otherwise, do nothing.
if (MaxBitWidth == 0 ||
MaxBitWidth >=
cast<IntegerType>(TreeRoot.front()->getType()->getScalarType())
->getBitWidth()) {
if (UserIgnoreList) {
unsigned Sz = getTreeSizeExcludingGathers();
for (Value *V : TreeRoot)
AnalyzedMinBWVals.insert_or_assign(V, Sz);
}
NodesToKeepBWs.insert_range(ToDemote);
continue;
}
// Finally, map the values we can demote to the maximum bit with we
// computed.
for (unsigned Idx : ToDemote) {
TreeEntry *TE = VectorizableTree[Idx].get();
if (MinBWs.contains(TE))
continue;
bool IsSigned = any_of(TE->Scalars, [&](Value *R) {
if (isa<PoisonValue>(R))
return false;
return !isKnownNonNegative(R, SimplifyQuery(*DL));
});
MinBWs.try_emplace(TE, MaxBitWidth, IsSigned);
}
}
}
PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
auto *AA = &AM.getResult<AAManager>(F);
auto *LI = &AM.getResult<LoopAnalysis>(F);
auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
auto *AC = &AM.getResult<AssumptionAnalysis>(F);
auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
// Runtime alias check versioning changes the CFG. In that case only the
// dominator tree is kept up to date (via DomTreeUpdater); other CFG analyses
// must be recomputed. Otherwise SLP leaves the CFG intact.
if (CFGChanged)
PA.preserve<DominatorTreeAnalysis>();
else
PA.preserveSet<CFGAnalyses>();
return PA;
}
bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
TargetTransformInfo *TTI_,
TargetLibraryInfo *TLI_, AAResults *AA_,
LoopInfo *LI_, DominatorTree *DT_,
AssumptionCache *AC_, DemandedBits *DB_,
OptimizationRemarkEmitter *ORE_) {
Opts = &F.getContext().getLibraryOptions().get<VectorizeOptions>();
if (!Opts->vectorize_slp)
return false;
SE = SE_;
TTI = TTI_;
TLI = TLI_;
AA = AA_;
LI = LI_;
DT = DT_;
AC = AC_;
DB = DB_;
DL = &F.getDataLayout();
Stores.clear();
GEPs.clear();
bool Changed = false;
// If the target claims to have no vector registers don't attempt
// vectorization.
if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true))) {
LLVM_DEBUG(
dbgs() << "SLP: Didn't find any vector registers for target, abort.\n");
return false;
}
// Don't vectorize when the attribute NoImplicitFloat is used.
if (F.hasFnAttribute(Attribute::NoImplicitFloat))
return false;
LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
// Use the bottom up slp vectorizer to construct chains that start with
// store instructions.
BoUpSLP R(*Opts, &F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
// A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
// delete instructions.
// Update DFS numbers now so that we can use them for ordering.
DT->updateDFSNumbers();
SmallSetVector<Instruction *, 8> FMACandidates;
// Scan the blocks in the function in post order.
auto POT = post_order(&F.getEntryBlock());
for (auto *BB : make_filter_range(POT, [](auto *BB) {
return !BB->isEHPad() && !isa<UnreachableInst>(BB->getTerminator());
})) {
// Skip scalar fallback blocks created by runtime alias check versioning:
// they duplicate the original scalar code and must not be vectorized again.
if (R.isScalarFallbackBlock(BB))
continue;
// Start new block - clear the list of reduction roots.
R.clearReductionData();
collectSeedInstructions(BB);
Changed |= vectorizeSplitStorePoorThroughputSeeds(BB, R);
// Vectorize trees that end at stores.
if (!Stores.empty()) {
LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
<< " underlying objects.\n");
Changed |= vectorizeStoreChains(R);
}
// Vectorize trees that end at reductions.
Changed |= vectorizeChainsInBlock(BB, R, FMACandidates);
// Vectorize the index computations of getelementptr instructions. This
// is primarily intended to catch gather-like idioms ending at
// non-consecutive loads.
if (!GEPs.empty()) {
LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
<< " underlying objects.\n");
Changed |= vectorizeGEPIndices(BB, R);
}
}
// Vectorized on their own, the FMA candidates lose the fusion with their fmul
// operands. They are retried after all the blocks, not to preempt the trees
// of the blocks processed later, e.g. the store chains of a loop body,
// processed after the latch PHIs reaching the same instructions.
R.clearReductionData();
SmallSetVector<Instruction *, 8> Empty;
for (Instruction *I : make_filter_range(
FMACandidates, [&](Instruction *I) { return !R.isDeleted(I); }))
Changed |= tryToVectorize(I, R, Empty, /*AllowFMACandidates=*/true);
assert(Empty.empty() &&
"No new FMA candidates expected during AllowFMACandidates retry.");
// Instructions with the single user require just one extract per lane, so
// they are used as the seeds for the very last attempt, after all the other
// roots in the function are exhausted.
if (Opts->slp_vectorize_once_used) {
auto OnceUsedPOT = post_order(&F.getEntryBlock());
for (auto *BB : make_filter_range(OnceUsedPOT, [&](auto *BB) {
return !BB->isEHPad() &&
!isa<UnreachableInst>(BB->getTerminator()) &&
!R.isScalarFallbackBlock(BB);
})) {
R.clearReductionData();
Changed |= vectorizeOnceUsedSeeds(BB, R);
}
}
if (Changed) {
R.optimizeGatherSequence();
LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
}
// Propagate whether runtime alias check versioning changed the CFG, so the
// caller can drop CFG-analysis preservation only when necessary.
CFGChanged = R.isCFGChanged();
return Changed;
}
std::optional<bool>
SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
unsigned &Size) {
std::optional<bool> Res = vectorizeStoreChainImpl(Chain, R, Idx, MinVF, Size);
// Retry the non-profitable tree, wider than the vector register, with the
// alternate stored values (looking through the cast) split into halves. The
// halves share the common operands instead of extracting them for the scalar
// stores of the other half.
if (Res && !*Res && Chain.size() > SmallNodeSize && Chain.size() % 2 == 0 &&
Chain.size() * R.getVectorElementSize(Chain.front()) >
R.getMaxVecRegSize() &&
R.canSplitStoredValues(Chain)) {
SaveAndRestore<bool> SplitGuard = R.splitStoredValues();
unsigned SplitSize;
std::optional<bool> SplitRes =
vectorizeStoreChainImpl(Chain, R, Idx, MinVF, SplitSize);
if (SplitRes.value_or(false)) {
Size = SplitSize;
return SplitRes;
}
}
assert(!R.isTryingRuntimeAliasChecks() &&
"Unexpected nested runtime alias check attempt");
// Retry once with runtime alias checks, but only when the normal attempt
// could not even schedule the bundle, i.e. it was blocked by
// a memory dependency, and that dependency is runtime-checkable. If the
// region also kept a non-checkable blocker, dropping the checkable deps
// cannot unblock it.
if (Res.has_value() || !Opts->slp_vectorize_with_runtime_alias_checks ||
!R.hasRuntimeCheckableBlockers() || R.hasNonCheckableMemBlocker())
return Res;
// getTreeCost() unconditionally rejects a VF=2 tree whose vector
// instruction count exceeds its scalar instruction count (the
// SLPInstCountCheck heuristic), because the cost model is known to
// underestimate shuffle/insert/extract overhead at that width. That check
// returns an invalid cost that no threshold, including the runtime-checks
// one, can outweigh. Retrying such a chain would pay the full tree rebuild
// cost for the one width the cost model is least trusted for, on top of
// the extra guard overhead, so skip it instead of wasting the attempt.
if (Chain.size() == 2)
return Res;
// Only retry when scheduling blocked the chain (a store or its value bundle
// could not be scheduled), which dropping may-alias deps can fix. A chain
// gathered for structural reasons (e.g. non-consecutive addresses) would
// gather again, so the second buildTree() would be pure overhead with no
// vectorization benefit.
Value *FirstStore = Chain.front();
if (!R.isNotScheduled(FirstStore) &&
!R.isNotScheduled(cast<StoreInst>(FirstStore)->getValueOperand()))
return Res;
BasicBlock *BB = cast<Instruction>(FirstStore)->getParent();
// If an earlier optimistic attempt already failed for this block, do not
// retry for other chains or smaller VFs in the same block.
if (R.runtimeChecksFailedForBlock(BB))
return Res;
// The retry rebuilds the whole tree; also skip it when the chain's block can
// never be versioned (e.g. the function is optimized for size).
if (!R.canVersionBlockForRuntimeChecks(BB)) {
R.markRuntimeChecksFailedForBlock(BB);
return Res;
}
R.setTryRuntimeAliasChecks(true);
unsigned RTSize = Size;
std::optional<bool> RTRes =
vectorizeStoreChainImpl(Chain, R, Idx, MinVF, RTSize);
R.setTryRuntimeAliasChecks(false);
if (RTRes && *RTRes) {
Size = RTSize;
return RTRes;
}
// The versioning attempt failed for this block, skip the (expensive) retry.
R.markRuntimeChecksFailedForBlock(BB);
return Res;
}
std::optional<bool>
SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
unsigned Idx, unsigned MinVF,
unsigned &Size) {
Size = 0;
R.resetRuntimeAliasCheckState();
LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size()
<< "\n");
const unsigned Sz = R.getVectorElementSize(Chain[0]);
unsigned VF = Chain.size();
if (!has_single_bit(Sz) ||
!hasFullVectorsOrPowerOf2(
*TTI, cast<StoreInst>(Chain.front())->getValueOperand()->getType(),
VF, Opts->slp_revec) ||
VF < 2 || VF < MinVF) {
// Check if vectorizing with a non-power-of-2 VF should be considered; see
// isAllowedNonPowerOf2VF for supported widths.
if (!Opts->slp_vectorize_non_power_of_2 || (VF < MinVF && VF + 1 != MinVF))
return false;
}
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
<< "\n");
SetVector<Value *> ValOps;
for (Value *V : Chain)
ValOps.insert(cast<StoreInst>(V)->getValueOperand());
// Operands are not same/alt opcodes or non-power-of-2 uniques - exit.
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
InstructionsState S =
Analysis.buildInstructionsState(ValOps.getArrayRef(), R);
if (all_of(ValOps, IsaPred<Instruction>) && ValOps.size() > 1) {
DenseSet<Value *> Stores(Chain.begin(), Chain.end());
bool IsAllowedSize =
hasFullVectorsOrPowerOf2(*TTI, ValOps.front()->getType(), ValOps.size(),
Opts->slp_revec) ||
isAllowedNonPowerOf2VF(ValOps.size(),
Opts->slp_vectorize_non_power_of_2);
if ((!IsAllowedSize && S && S.getOpcode() != Instruction::Load &&
(!S.getMainOp()->isSafeToRemove() ||
any_of(ValOps.getArrayRef(),
[&](Value *V) {
return !isa<ExtractElementInst>(V) &&
(V->getNumUses() > Chain.size() ||
any_of(V->users(), [&](User *U) {
return !Stores.contains(U);
}));
}))) ||
(ValOps.size() > Chain.size() / 2 && !S)) {
Size = (!IsAllowedSize && S) ? 1 : 2;
return false;
}
// Every value is reused several times outside of the chain: such a chain
// only merges the stores, while the scalars remain live for the other users
// and all the lanes are gathered back. A single outside use may still be a
// part of the larger vectorizable graph, same for the values, fed by the
// loads, where the vector loads may pay off the gathering. Two outside
// uses may still be part of the same profitable tree when they look like
// carry deps (cmp, select, non-div/rem binop, or a two-incoming phi);
// reject 3+ outside users, or exactly 2 when at least one is not benign.
auto IsBenignOutsideUser = [](Instruction *UI) {
assert(UI && "Expected instruction.");
if (auto *PN = dyn_cast<PHINode>(UI))
return PN->getNumIncomingValues() == 2;
if (auto *BO = dyn_cast<BinaryOperator>(UI))
return !BO->isIntDivRem() && !BO->isFPDivRem();
return isa<CmpInst, SelectInst>(UI);
};
auto HasTooManyOutsideUsers = [&](Instruction *I) {
// To save compilation time, bail out if the use list is huge.
if (I->hasNUsesOrMore(UsesLimit))
return true;
unsigned Outside = 0;
bool HasNonBenign = false;
for (User *U : I->users()) {
if (Stores.contains(U))
continue;
++Outside;
if (Outside > 2)
return true;
if (!IsBenignOutsideUser(cast<Instruction>(U)))
HasNonBenign = true;
if (Outside == 2 && HasNonBenign)
return true;
}
return false;
};
if (S && S.getOpcode() != Instruction::Load &&
all_of(ValOps.getArrayRef(), [&](Value *V) {
auto *I = cast<Instruction>(V);
return none_of(I->operand_values(), IsaPred<LoadInst>) &&
HasTooManyOutsideUsers(I);
})) {
Size = 1;
return false;
}
}
R.buildTree(Chain);
// Check if tree tiny and store itself or its value is not vectorized.
if (R.isTreeTinyAndNotFullyVectorizable()) {
if (R.isGathered(Chain.front()) ||
R.isNotScheduled(cast<StoreInst>(Chain.front())->getValueOperand()))
return std::nullopt;
Size = R.getCanonicalGraphSize();
return false;
}
if (R.isProfitableToReorder()) {
R.reorderTopToBottom();
R.reorderBottomToTop();
}
if (R.isTryingRuntimeAliasChecks())
R.captureRuntimeCheckBodySnapshot();
R.transformNodes();
R.computeMinimumValueSizes();
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
R.buildExternalUses();
Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries();
if (S && S.getOpcode() == Instruction::Load)
Size = 2; // cut off masked gather small trees
InstructionCost Cost = R.getTreeCost(TreeCost);
// When the tree was scheduled by dropping may-alias dependencies, account
// for the runtime alias checks in the cost and make sure the region can
// actually be versioned. If it cannot, abandon this attempt: the dropped
// dependencies are unused because the tree is not vectorized.
// The full check cost is charged against the vector-path benefit here; the
// scalar-path overhead is bounded separately in canVersionForRuntimeChecks()
// (a fraction of the scalar body cost).
if (R.isTryingRuntimeAliasChecks() && R.hasRuntimeAliasChecks()) {
// getRuntimeChecksCost() only ever sums non-negative per-instruction
// throughput costs (address adds, compares, and/or, the guard branch), so
// adding it can only push Cost up, never down, and an already-invalid
// Cost (e.g. rejected by one of getTreeCost()'s own heuristics) stays
// invalid either way. So if Cost already fails the threshold below, no
// checks cost can rescue it: skip the (expensive) SCEV/dominance/escape
// analysis in canVersionForRuntimeChecks() instead of running it on a
// tree that is already rejected.
if (Cost >= -R.getCostThreshold())
return false;
if (!R.canVersionForRuntimeChecks())
return false;
// TODO: Add probability-weighted model (benefit scaled by the expected
// non-alias odds) would be more precise but needs
// branch-probability/profile data.
Cost += R.getRuntimeChecksCost();
}
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for VF=" << VF << "\n");
if (Cost < -R.getCostThreshold()) {
LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost = " << Cost << "\n");
using namespace ore;
R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
cast<StoreInst>(Chain[0]))
<< "Stores SLP vectorized with cost " << NV("Cost", Cost)
<< " and with tree size "
<< NV("TreeSize", R.getTreeSize()));
R.vectorizeTree();
return true;
}
return false;
}
namespace {
/// A group of related stores which we are in the process of vectorizing,
/// a subset of which may already be vectorized. Stores context information
/// about the group as a whole as well as information about what VFs need
/// to be attempted still.
class StoreChainContext {
public:
using SizePair = std::pair<unsigned, unsigned>;
explicit StoreChainContext(const VectorizeOptions &Opts,
ArrayRef<Value *> Ops,
ArrayRef<SizePair> RangeSizes,
SmallVector<unsigned> &RangeSizesByIdx,
unsigned Stride)
: Opts(Opts), Operands(Ops), RangeSizesStorage(RangeSizes),
RangeSizesByIdx(RangeSizesByIdx), Stride(Stride) {}
/// Set up initial values using the already set Operands
bool initializeContext(
BoUpSLP &R, const DataLayout &DL, const TargetTransformInfo &TTI,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited);
/// Get the current VF
std::optional<unsigned> getCurrentVF() const;
/// Return the maximum VF for the context
unsigned getMaxVF() const { return MaxVF; }
/// Return the stride of the context
unsigned getStride() const { return Stride; }
/// Attempt to vectorize Operands for the given VF
/// Returns false if no more attempts should be made for the context
bool vectorizeOneVF(const TargetTransformInfo &TTI, unsigned VF,
BoUpSLP::ValueSet &VectorizedStores, bool &Changed,
llvm::function_ref<std::optional<bool>(
ArrayRef<Value *>, unsigned, unsigned, unsigned &)>
VectorizeStoreChain);
/// Add an additional store to the chain
/// \p Store store too append to Operands
/// \p Idx position within VectorizeStores::Stores
void addOperand(Value *Store, unsigned Idx);
private:
bool isNotVectorized(const SizePair &P) const {
return P.first != LocallyUnvectorizable && RangeSizesByIdx[P.first] > 0;
}
bool isVectorized(const SizePair &P) const {
return P.first == LocallyUnvectorizable || RangeSizesByIdx[P.first] == 0;
}
bool isVFProfitable(unsigned Size, const SizePair &P) const {
assert(P.first != LocallyUnvectorizable && RangeSizesByIdx[P.first] &&
"Cannot check profitability of vectorized element");
return Size >= RangeSizesByIdx[P.first];
}
bool firstSizeSame(unsigned Size, const SizePair &P) const {
assert(P.first != LocallyUnvectorizable && RangeSizesByIdx[P.first] &&
"Cannot check profitability of vectorized element");
return Size == RangeSizesByIdx[P.first];
}
/// Return the index of the first unvectorized store after \p StartIdx
unsigned getFirstUnvecStore(unsigned StartIdx = 0) const;
/// Return the index of the first vectorized store after \p StartIdx
unsigned getFirstVecStoreAfter(unsigned StartIdx) const;
/// Return true if all stores have been vectorized
bool allVectorized() const;
/// Return true if all elements in the given range match \p TreeSize
bool isFirstSizeSameRange(unsigned StartIdx, unsigned Length,
unsigned TreeSize) const;
/// Return true if the \p TreeSize is profitable for all elements in the range
bool allOfRangeProfitable(unsigned StartIdx, unsigned Length,
unsigned TreeSize) const;
/// Update the live (first) range sizes from the cached values (second)
void updateRangeSizesFromCache();
/// Update the cached (second) range sizes with the given \p TreeSize
void updateCachedRangeSizes(unsigned StartIdx, unsigned Length,
unsigned TreeSize);
/// Update CandidateVFs for secondary iterations
bool updateCandidateVFs(const TargetTransformInfo &TTI);
/// Remove the current VF from the queue
void incrementVF() {
if (!CandidateVFs.empty())
CandidateVFs.pop();
}
/// Record vectorization of the provided range
void markRangeVectorized(unsigned StartIdx, unsigned Length,
unsigned &FirstUnvecStore, unsigned &MaxSliceEnd);
/// Checks if the quadratic mean deviation is less than 90% of the mean size.
bool checkTreeSizes(const unsigned SliceStartIdx, const unsigned VF) const;
/// In RangeSizes, element has not been vectorized, but due to the elements
/// around it being vectorized, it does not have enough neighboring elements
/// to make a chain longer than MinVF as part of the current Context
static constexpr unsigned LocallyUnvectorizable =
std::numeric_limits<unsigned>::max();
/// Maximum number of iterations through CandidateVFs
static constexpr unsigned MaxAttempts = 4;
const VectorizeOptions &Opts;
/// For the StoreTy/Stride in the given group, what is the smallest VF
/// that can be used
unsigned MinVF = 0;
/// Maximum number of instructions that can be vectorized, either
/// constrained by register width or operands size.
unsigned MaxVF = 0;
/// MaxRegVF represents the number of instructions (scalar, or vector in
/// case of revec) that can be vectorized to naturally fit in a vector
/// register.
unsigned MaxRegVF = 0;
/// The largest VF checked in the current Repeat
unsigned ProbeVF = 0;
/// Type of the Stores in `Operands`
Type *StoreTy = nullptr;
/// Which VFs do we want to attempt for this chain
std::queue<unsigned> CandidateVFs;
/// Stores that compose this chain
BoUpSLP::ValueList Operands;
/// Track the TreeSizes of prior vectorization attempts using each element,
/// to help us find early exit cases
/// - first: contains pointer into RangeSizesByIdx to help us track
/// vectorization of elements that belong to multiple chains
/// - second: contains cached TreeSize value for that element
SmallVector<SizePair> RangeSizesStorage;
MutableArrayRef<SizePair> RangeSizes;
/// RangeSize information for all elements in any chain
/// Needed since may be overlap between chains
SmallVector<unsigned> &RangeSizesByIdx;
/// What element index is the end of the to be vectorized Operands
/// i.e. Operands.size() == 16, and 12-15 were vectorized, then End == 12
unsigned End = 0;
/// How many times has CandidateVFs been refilled, prevents excessive
/// attempts at vectorizing large VFs
unsigned Repeat = 1;
/// Did any vectorization occur for the current iteration over CandidateVFs
bool RepeatChanged = false;
/// For constant strided stores, what is the stride amount
const unsigned Stride = 0;
/// Store information about failed vectorization attempts due to scheduling
SmallDenseMap<Value *, SizePair> NonSchedulable;
};
void StoreChainContext::addOperand(Value *Store, unsigned Idx) {
Operands.push_back(Store);
RangeSizesStorage.push_back({Idx, 1});
}
void StoreChainContext::markRangeVectorized(unsigned StartIdx, unsigned Length,
unsigned &FirstUnvecStore,
unsigned &MaxSliceEnd) {
for (SizePair &P : RangeSizes.slice(StartIdx, Length))
RangeSizesByIdx[P.first] = P.second = 0;
if (StartIdx < FirstUnvecStore + MinVF) {
for (SizePair &P :
RangeSizes.slice(FirstUnvecStore, StartIdx - FirstUnvecStore)) {
P.first = LocallyUnvectorizable;
P.second = 0;
}
FirstUnvecStore = StartIdx + Length;
}
if (StartIdx + Length > MaxSliceEnd - MinVF) {
for (SizePair &P : RangeSizes.slice(StartIdx + Length,
MaxSliceEnd - (StartIdx + Length))) {
P.first = LocallyUnvectorizable;
P.second = 0;
}
if (MaxSliceEnd == End)
End = StartIdx;
MaxSliceEnd = StartIdx;
}
}
bool StoreChainContext::initializeContext(
BoUpSLP &R, const DataLayout &DL, const TargetTransformInfo &TTI,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
&Visited) {
if (!Visited
.insert({Operands.front(),
cast<StoreInst>(Operands.front())->getValueOperand(),
Operands.back(),
cast<StoreInst>(Operands.back())->getValueOperand(),
Operands.size()})
.second)
return false;
// Initialize range tracking in context.
RangeSizes = MutableArrayRef(RangeSizesStorage);
unsigned MaxVecRegSize = R.getMaxVecRegSize();
unsigned EltSize = R.getVectorElementSize(Operands[0]);
unsigned MaxElts = llvm::bit_floor(MaxVecRegSize / EltSize);
MaxVF = std::min(R.getMaximumVF(EltSize, Instruction::Store), MaxElts);
auto *Store = cast<StoreInst>(Operands[0]);
StoreTy = Store->getValueOperand()->getType();
Type *ValueTy = StoreTy;
if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
ValueTy = Trunc->getSrcTy();
// When REVEC is enabled, StoreTy and ValueTy may be FixedVectorType. But
// getStoreMinimumVF only support scalar type as arguments. As a result,
// we need to use the element type of StoreTy and ValueTy to retrieve the
// VF and then transform it back.
// Remember: VF is defined as the number we want to vectorize, not the
// number of elements in the final vector.
Type *StoreScalarTy = StoreTy->getScalarType();
MinVF = PowerOf2Ceil(TTI.getStoreMinimumVF(
R.getMinVF(DL.getTypeStoreSizeInBits(StoreScalarTy)), StoreScalarTy,
ValueTy->getScalarType(), Store->getAlign(),
Store->getPointerAddressSpace()));
MinVF /= getNumElements(StoreTy);
MinVF = std::max<unsigned>(2, MinVF);
if (Stride > 1)
MinVF = std::max(MinVF, Opts.slp_min_strided_stores);
if (MaxVF < MinVF) {
LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
<< ") < "
<< "MinVF (" << MinVF << ")\n");
return false;
}
// First try a supported non-power-of-2 VF (see isAllowedNonPowerOf2VF).
unsigned NonPowerOf2VF = 0;
unsigned CandVF = std::clamp<unsigned>(Operands.size(), MinVF, MaxVF);
if (isAllowedNonPowerOf2VF(CandVF, Opts.slp_vectorize_non_power_of_2)) {
NonPowerOf2VF = CandVF;
assert(NonPowerOf2VF != MaxVF &&
"Non-power-of-2 VF should not be equal to MaxVF");
}
MaxRegVF = MaxVF;
MaxVF = std::min<unsigned>(MaxVF, bit_floor(Operands.size()));
if (MaxVF < MinVF) {
LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
<< ") < "
<< "MinVF (" << MinVF << ")\n");
return false;
}
for (unsigned VF = std::max(MaxVF, NonPowerOf2VF); VF >= MinVF;
VF = divideCeil(VF, 2))
CandidateVFs.push(VF);
End = Operands.size();
ProbeVF = MaxVF;
return true;
}
// Return the index of the first unvectorized store after \p StartIdx
unsigned StoreChainContext::getFirstUnvecStore(unsigned StartIdx) const {
return std::distance(
RangeSizes.begin(),
find_if(RangeSizes.drop_front(StartIdx),
[this](const SizePair &P) { return this->isNotVectorized(P); }));
}
// Return the index of the first vectorized store after \p StartIdx
unsigned StoreChainContext::getFirstVecStoreAfter(unsigned StartIdx) const {
return std::distance(
RangeSizes.begin(),
find_if(RangeSizes.drop_front(StartIdx),
[this](const SizePair &P) { return this->isVectorized(P); }));
}
// Return true if all stores have been vectorized
bool StoreChainContext::allVectorized() const {
return all_of(RangeSizes,
[this](const SizePair &P) { return this->isVectorized(P); });
}
// Return true if all elements in the given range match \p TreeSize
bool StoreChainContext::isFirstSizeSameRange(unsigned StartIdx, unsigned Length,
unsigned TreeSize) const {
return all_of(RangeSizes.slice(StartIdx, Length),
[TreeSize, this](const SizePair &P) {
return firstSizeSame(TreeSize, P);
});
}
// Return true if the \p TreeSize is profitable for all elements in the range
bool StoreChainContext::allOfRangeProfitable(unsigned StartIdx, unsigned Length,
unsigned TreeSize) const {
return all_of(RangeSizes.slice(StartIdx, Length),
[TreeSize, this](const SizePair &P) {
return isVFProfitable(TreeSize, P);
});
}
// Update the live (first) range sizes from the cached values (second)
void StoreChainContext::updateRangeSizesFromCache() {
for (SizePair &P : RangeSizes) {
if (P.first != LocallyUnvectorizable && RangeSizesByIdx[P.first] != 0)
RangeSizesByIdx[P.first] = std::max(P.second, RangeSizesByIdx[P.first]);
}
}
// Update the cached (second) range sizes with the given \p TreeSize
void StoreChainContext::updateCachedRangeSizes(unsigned StartIdx,
unsigned Length,
unsigned TreeSize) {
for (SizePair &P : RangeSizes.slice(StartIdx, Length))
P.second = std::max(P.second, TreeSize);
}
bool StoreChainContext::updateCandidateVFs(const TargetTransformInfo &TTI) {
assert(CandidateVFs.empty() && "Did not use all VFs before refilling");
constexpr unsigned StoresLimit = 64;
const unsigned MaxTotalNum = std::min<unsigned>(
Operands.size(), static_cast<unsigned>(End - getFirstUnvecStore()));
unsigned VF = bit_ceil(ProbeVF) * 2;
if (VF > MaxTotalNum || VF >= StoresLimit)
return false;
// Attempt again to vectorize even larger chains if all previous
// attempts were unsuccessful because of the cost issues.
unsigned Limit = getFloorFullVectorNumberOfElements(TTI, StoreTy, MaxTotalNum,
Opts.slp_revec);
if (bit_floor(Limit) == VF && Limit != VF)
CandidateVFs.push(Limit);
CandidateVFs.push(VF);
ProbeVF = CandidateVFs.front();
++Repeat;
RepeatChanged = false;
return true;
}
// Get the current VF
std::optional<unsigned> StoreChainContext::getCurrentVF() const {
if (CandidateVFs.empty())
return std::nullopt;
return CandidateVFs.front();
}
bool StoreChainContext::checkTreeSizes(const unsigned SliceStartIdx,
const unsigned VF) const {
auto Sizes = RangeSizes.slice(SliceStartIdx, VF);
unsigned Num = 0;
uint64_t Sum = std::accumulate(
Sizes.begin(), Sizes.end(), static_cast<uint64_t>(0),
[&](uint64_t V, const std::pair<unsigned, unsigned> &Val) {
unsigned Size = Val.first == StoreChainContext::LocallyUnvectorizable
? 0
: RangeSizesByIdx[Val.first];
if (Size == 1)
return V;
++Num;
return V + Size;
});
if (Num == 0)
return true;
uint64_t Mean = Sum / Num;
if (Mean == 0)
return true;
uint64_t Dev =
accumulate(Sizes, static_cast<uint64_t>(0),
[&](uint64_t V, const std::pair<unsigned, unsigned> &Val) {
unsigned P =
Val.first == StoreChainContext::LocallyUnvectorizable
? 0
: RangeSizesByIdx[Val.first];
if (P == 1)
return V;
return V + (P - Mean) * (P - Mean);
}) /
Num;
return Dev * 96 / (Mean * Mean) == 0;
}
bool StoreChainContext::vectorizeOneVF(
const TargetTransformInfo &TTI, unsigned VF,
BoUpSLP::ValueSet &VectorizedStores, bool &Changed,
llvm::function_ref<std::optional<bool>(ArrayRef<Value *>, unsigned,
unsigned, unsigned &)>
VectorizeStoreChain) {
bool AnyProfitableGraph = false;
bool PreferWideSlices = false;
unsigned FirstUnvecStore = getFirstUnvecStore();
// Form slices of size VF starting from FirstUnvecStore and try to
// vectorize them.
while (FirstUnvecStore < End) {
unsigned FirstVecStore = getFirstVecStoreAfter(FirstUnvecStore);
unsigned MaxSliceEnd = FirstVecStore >= End ? End : FirstVecStore;
for (unsigned SliceStartIdx = FirstUnvecStore;
SliceStartIdx + VF <= MaxSliceEnd;) {
if (!checkTreeSizes(SliceStartIdx, VF)) {
++SliceStartIdx;
continue;
}
ArrayRef<Value *> Slice = ArrayRef(Operands).slice(SliceStartIdx, VF);
assert(all_of(Slice,
[&](Value *V) {
return cast<StoreInst>(V)->getValueOperand()->getType() ==
cast<StoreInst>(Slice.front())
->getValueOperand()
->getType();
}) &&
"Expected all operands of same type.");
if (!NonSchedulable.empty()) {
auto [NonSchedSizeMax, NonSchedSizeMin] =
NonSchedulable.lookup(Slice.front());
if (NonSchedSizeMax > 0 && NonSchedSizeMin <= VF) {
// VF is too ambitious. Try to vectorize another slice before
// trying a smaller VF.
SliceStartIdx += NonSchedSizeMax;
continue;
}
}
auto MarkVectorized = [&](unsigned Len) {
// Mark the vectorized stores so that we don't vectorize them
// again.
VectorizedStores.insert_range(
ArrayRef(Operands).slice(SliceStartIdx, Len));
AnyProfitableGraph = RepeatChanged = Changed = true;
// If we vectorized initial block, no need to try to vectorize
// it again.
markRangeVectorized(SliceStartIdx, Len, FirstUnvecStore, MaxSliceEnd);
SliceStartIdx += Len;
++NumStoreChains;
if (Stride > 1)
++NumStridedStoreChains;
NumVectorizedStores += Len;
};
// With the register VF 2 the instruction count check may reject the
// leading slice, shifting it strands its stores and the wider VFs are
// tried only if nothing is vectorized: try the wider slice at the same
// position. Once the chain took the wider slices, try them first.
auto TryWideSlice = [&]() {
if (MaxRegVF != 2 || VF != MaxRegVF ||
SliceStartIdx != FirstUnvecStore ||
SliceStartIdx + 2 * VF > MaxSliceEnd)
return false;
unsigned WideTreeSize;
if (!VectorizeStoreChain(
ArrayRef(Operands).slice(SliceStartIdx, 2 * VF), SliceStartIdx,
MinVF, WideTreeSize)
.value_or(false))
return false;
MarkVectorized(2 * VF);
return true;
};
const bool WideFirst = PreferWideSlices;
if (WideFirst && TryWideSlice())
continue;
PreferWideSlices = false;
unsigned TreeSize;
std::optional<bool> Res =
VectorizeStoreChain(Slice, SliceStartIdx, MinVF, TreeSize);
if (!Res) {
// Update the range of non schedulable VFs for slices starting
// at SliceStartIdx.
NonSchedulable.try_emplace(Slice.front(), std::make_pair(VF, VF))
.first->getSecond()
.second = VF;
} else if (*Res) {
MarkVectorized(VF);
continue;
}
if (Res && !WideFirst && TryWideSlice()) {
PreferWideSlices = true;
continue;
}
if (VF > 2 && Res && !allOfRangeProfitable(SliceStartIdx, VF, TreeSize)) {
SliceStartIdx += VF;
continue;
}
// Check for the very big VFs that we're not rebuilding same
// trees, just with larger number of elements.
if (VF > MaxRegVF && TreeSize > 1 &&
isFirstSizeSameRange(SliceStartIdx, VF, TreeSize)) {
SliceStartIdx += VF;
while (SliceStartIdx != MaxSliceEnd &&
isFirstSizeSameRange(SliceStartIdx, 1, TreeSize))
++SliceStartIdx;
continue;
}
if (TreeSize > 1)
updateCachedRangeSizes(SliceStartIdx, VF, TreeSize);
++SliceStartIdx;
AnyProfitableGraph = true;
}
if (FirstUnvecStore >= End)
break;
if (MaxSliceEnd - FirstUnvecStore < VF &&
MaxSliceEnd - FirstUnvecStore >= MinVF)
AnyProfitableGraph = true;
FirstUnvecStore = getFirstUnvecStore(MaxSliceEnd);
}
if (!AnyProfitableGraph && VF >= MaxRegVF && has_single_bit(VF))
while (!CandidateVFs.empty())
CandidateVFs.pop();
// For the MaxRegVF case, save RangeSizes to limit compile time
if (VF == MaxRegVF)
updateRangeSizesFromCache();
incrementVF();
if (!getCurrentVF()) {
// All values vectorized - exit.
if (allVectorized())
return false;
// Check if tried all attempts or no need for the last attempts at
// all.
if (Repeat >= MaxAttempts ||
(Repeat > 1 && (RepeatChanged || !AnyProfitableGraph)))
return false;
if (!updateCandidateVFs(TTI))
return false;
// Avoid double update of cache sizes
if (VF != MaxRegVF)
updateRangeSizesFromCache();
}
return true;
}
/// A group of stores that we'll try to bundle together using vector ops.
/// They are ordered using the signed distance of their address operand to the
/// address of this group's BaseInstr.
class RelatedStoreInsts {
public:
RelatedStoreInsts(unsigned BaseInstrIdx, ArrayRef<StoreInst *> AllStores)
: AllStores(AllStores) {
reset(BaseInstrIdx);
}
void reset(unsigned NewBaseInstr) {
assert(NewBaseInstr < AllStores.size() &&
"Instruction index out of bounds");
BaseInstrIdx = NewBaseInstr;
Instrs.clear();
insertOrLookup(NewBaseInstr, 0);
}
/// Tries to insert \p InstrIdx as the store with a pointer distance of
/// \p PtrDist.
/// Does nothing if there is already a store with that \p PtrDist.
/// \returns The previously associated Instruction index, or std::nullopt
std::optional<unsigned> insertOrLookup(unsigned InstrIdx, int64_t PtrDist) {
auto [It, Inserted] = Instrs.emplace(PtrDist, InstrIdx);
return Inserted ? std::nullopt : std::make_optional(It->second);
}
using DistToInstMap = std::map<int64_t, unsigned>;
const DistToInstMap &getStores() const { return Instrs; }
/// If \p SI is related to this group of stores, return the distance of its
/// pointer operand to the one the group's BaseInstr.
std::optional<int64_t> getPointerDiff(StoreInst &SI, const DataLayout &DL,
ScalarEvolution &SE) const {
StoreInst &BaseStore = *AllStores[BaseInstrIdx];
return getPointersDiff(
BaseStore.getValueOperand()->getType(), BaseStore.getPointerOperand(),
SI.getValueOperand()->getType(), SI.getPointerOperand(), DL, SE,
/*StrictCheck=*/true);
}
/// Recompute the pointer distances to be based on \p NewBaseInstIdx.
/// Stores whose index is less than \p MinSafeIdx will be dropped.
void rebase(unsigned MinSafeIdx, unsigned NewBaseInstIdx,
int64_t DistFromCurBase) {
DistToInstMap PrevSet = std::move(Instrs);
reset(NewBaseInstIdx);
// Re-insert stores that come after MinSafeIdx to try and vectorize them
// again. Their distance will be "rebased" to use NewBaseInstIdx as
// reference.
for (auto [Dist, InstIdx] : PrevSet) {
if (InstIdx >= MinSafeIdx)
insertOrLookup(InstIdx, Dist - DistFromCurBase);
}
}
private:
/// The index of the Base instruction, i.e. the one with a 0 pointer distance.
unsigned BaseInstrIdx;
/// Maps a pointer distance from \p BaseInstrIdx to an instruction index.
DistToInstMap Instrs;
/// Reference to all the stores in the BB being analyzed.
ArrayRef<StoreInst *> AllStores;
};
} // end anonymous namespace
bool SLPVectorizerPass::vectorizeStores(
ArrayRef<StoreInst *> Stores, BoUpSLP &R,
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>> &Visited,
bool AllowMaskedStores) {
// We may run into multiple chains that merge into a single chain. We mark the
// stores that we vectorized so that we don't visit the same store twice.
BoUpSLP::ValueSet VectorizedStores;
bool Changed = false;
SmallVector<std::unique_ptr<StoreChainContext>> AllContexts;
SmallVector<unsigned> RangeSizesByIdx(Stores.size(), 1);
auto ExtendContexts = [&](const RelatedStoreInsts::DistToInstMap &StoreSeq) {
BoUpSLP::ValueList Operands;
const unsigned MaxStride =
Opts->slp_enable_strided_stores ? Opts->slp_max_stride : 1;
// All chains that we're still building
struct PartialChainStatus {
// Has been added to AllContexts
// Wait to add to AllContexts until at least two stores are found
bool AddedToAllContexts;
union {
// If added, index into AllContexts
unsigned AllContextsIdx;
// Index into Stores if not added to AllContexts yet
unsigned StoresIdx;
};
// If not added to AllContexts, what is the single store in the chain
Value *FirstStore;
// What is the Stride of this chain
unsigned Stride;
};
SmallVector<SmallVector<PartialChainStatus, 1>> Chains(MaxStride + 1);
auto GetChainsKey = [&](int64_t Query) -> unsigned {
// Just modulo function, get the index into Chains for a given query
int64_t Rem = (Query % (int64_t)Chains.size());
if (Rem < 0)
Rem += (int64_t)Chains.size();
return (unsigned)Rem;
};
int64_t LastDist;
for (auto [Idx, Data] : enumerate(StoreSeq)) {
auto &[Dist, InstIdx] = Data;
// Clean up chains that can't be continued
if (Idx > 0)
for (int64_t D = LastDist;
D < std::min(Dist, LastDist + (int64_t)Chains.size()); ++D)
Chains[GetChainsKey(D)].clear();
LastDist = Dist;
// Track which stride lengths we found existing chains for
// Don't have to add new entries for these
SmallVector<bool> FoundStrides(Opts->slp_max_stride + 1, false);
for (auto &Status : Chains[GetChainsKey(Dist)]) {
if (Status.AddedToAllContexts) {
// Chain already in AllContexts()
AllContexts[Status.AllContextsIdx]->addOperand(Stores[InstIdx],
InstIdx);
} else {
// Chain just a single element, not yet in AllContexts()
SmallVector<StoreChainContext::SizePair> RS = {{Status.StoresIdx, 1},
{InstIdx, 1}};
BoUpSLP::ValueList Ops = {Status.FirstStore, Stores[InstIdx]};
AllContexts.emplace_back(std::make_unique<StoreChainContext>(
*Opts, Ops, RS, RangeSizesByIdx, Status.Stride));
Status.AllContextsIdx = AllContexts.size() - 1;
}
unsigned Key = GetChainsKey(Status.Stride + Dist);
Chains[Key].push_back({/*AddedToAllContexts=*/true,
{Status.AllContextsIdx},
/*FirstStore=*/nullptr,
Status.Stride});
FoundStrides[Status.Stride] = true;
}
Chains[GetChainsKey(Dist)].clear();
// For any stride lengths that we didn't append to a chain for,
// instead start a new chain
for (auto Stride : seq<unsigned>(1, MaxStride + 1)) {
if (FoundStrides[Stride])
continue;
unsigned Key = GetChainsKey(Dist + Stride);
Chains[Key].push_back({/*AddedToAllContexts=*/false,
{/*StoresIdx=*/(unsigned)InstIdx},
Stores[InstIdx],
Stride});
}
}
};
auto ActuallyVectorizeContexts = [&]() {
unsigned GlobalMaxVF = 0;
for (auto &CtxPtr : AllContexts)
if (CtxPtr->initializeContext(R, *DL, *TTI, Visited))
GlobalMaxVF = std::max(GlobalMaxVF, CtxPtr->getMaxVF());
else
CtxPtr.reset();
// Prioritize non-strided chains (Stride = 1)
llvm::stable_sort(AllContexts,
[](const std::unique_ptr<StoreChainContext> &A,
const std::unique_ptr<StoreChainContext> &B) {
return A && (!B || A->getStride() < B->getStride());
});
for (unsigned LimitVF = GlobalMaxVF; LimitVF > 0;
LimitVF = bit_ceil(LimitVF) / 2) {
for (auto &CtxPtr : AllContexts) {
if (!CtxPtr)
break;
StoreChainContext &Context = *CtxPtr;
for (std::optional<unsigned> VFUnval = Context.getCurrentVF();
VFUnval && *VFUnval >= LimitVF; VFUnval = Context.getCurrentVF()) {
unsigned VF = *VFUnval;
if (!Context.vectorizeOneVF(
*TTI, VF, VectorizedStores, Changed,
[this, &R](ArrayRef<Value *> Chain, unsigned Idx,
unsigned MinVF, unsigned &Size) {
return vectorizeStoreChain(Chain, R, Idx, MinVF, Size);
})) {
CtxPtr.reset();
break;
}
}
}
}
};
/// Groups of stores to vectorize
SmallVector<RelatedStoreInsts> SortedStores;
// Inserts the specified store SI with the given index Idx to the set of the
// stores. If the store with the same distance is found already - stop
// insertion, try to vectorize already found stores. If some stores from this
// sequence were not vectorized - try to vectorize them with the new store
// later. But this logic is applied only to the stores, that come before the
// previous store with the same distance.
// Example:
// 1. store x, %p
// 2. store y, %p+1
// 3. store z, %p+2
// 4. store a, %p
// 5. store b, %p+3
// - Scan this from the last to first store. The very first bunch of stores is
// {5, {{4, -3}, {2, -2}, {3, -1}, {5, 0}}} (the element in SortedStores
// vector).
// - The next store in the list - #1 - has the same distance from store #5 as
// the store #4.
// - Try to vectorize sequence of stores 4,2,3,5.
// - If all these stores are vectorized - just drop them.
// - If some of them are not vectorized (say, #3 and #5), do extra analysis.
// - Start new stores sequence.
// The new bunch of stores is {1, {1, 0}}.
// - Add the stores from previous sequence, that were not vectorized.
// Here we consider the stores in the reversed order, rather they are used in
// the IR (Stores are reversed already, see vectorizeStoreChains() function).
// Store #3 can be added -> comes after store #4 with the same distance as
// store #1.
// Store #5 cannot be added - comes before store #4.
// This logic allows to improve the compile time, we assume that the stores
// after previous store with the same distance most likely have memory
// dependencies and no need to waste compile time to try to vectorize them.
// - Try to vectorize the sequence {1, {1, 0}, {3, 2}}.
auto FillStoresSet = [&](unsigned Idx, StoreInst *SI) {
std::optional<int64_t> PtrDist;
auto *RelatedStores = find_if(
SortedStores, [&PtrDist, SI, this](const RelatedStoreInsts &StoreSeq) {
PtrDist = StoreSeq.getPointerDiff(*SI, *DL, *SE);
return PtrDist.has_value();
});
// We did not find a comparable store, start a new group.
if (RelatedStores == SortedStores.end()) {
SortedStores.emplace_back(Idx, Stores);
return;
}
// If there is already a store in the group with the same PtrDiff, try to
// vectorize the existing instructions before adding the current store.
// Otherwise, insert this store and keep collecting.
if (std::optional<unsigned> PrevInst =
RelatedStores->insertOrLookup(Idx, *PtrDist)) {
ExtendContexts(RelatedStores->getStores());
RelatedStores->rebase(/*MinSafeIdx=*/*PrevInst + 1,
/*NewBaseInstIdx=*/Idx,
/*DistFromCurBase=*/*PtrDist);
}
};
Type *PrevValTy = nullptr;
for (auto [I, SI] : make_filter_range(enumerate(Stores), [&](const auto &P) {
return !R.isDeleted(P.value());
})) {
if (!PrevValTy)
PrevValTy = SI->getValueOperand()->getType();
// Check that we do not try to vectorize stores of different types.
if (PrevValTy != SI->getValueOperand()->getType()) {
for (RelatedStoreInsts &StoreSeq : SortedStores)
ExtendContexts(StoreSeq.getStores());
SortedStores.clear();
PrevValTy = SI->getValueOperand()->getType();
}
FillStoresSet(I, SI);
}
// Final vectorization attempt.
for (RelatedStoreInsts &StoreSeq : SortedStores)
ExtendContexts(StoreSeq.getStores());
ActuallyVectorizeContexts();
// When some stores are still not vectorized, try to lower them together as a
// single masked store (expand + masked.store).
if (!AllowMaskedStores)
return Changed;
SmallVector<Value *> Remaining;
for (StoreInst *SI : Stores)
if (!R.isDeleted(SI) && !VectorizedStores.contains(SI))
Remaining.push_back(SI);
if (Remaining.size() < 2)
return Changed;
Type *ValTy = getValueType(Remaining.front(), Opts->slp_revec);
if (any_of(Remaining, [&](Value *V) {
return cast<StoreInst>(V)->getValueOperand()->getType() != ValTy;
}))
return Changed;
SmallVector<Value *> PointerOps;
PointerOps.reserve(Remaining.size());
for (Value *V : Remaining)
PointerOps.push_back(cast<StoreInst>(V)->getPointerOperand());
SmallVector<unsigned> Order;
if (!sortPtrAccesses(PointerOps, ValTy, *DL, *SE, Order))
return Changed;
SmallVector<Value *> Sorted(Remaining.size());
for (unsigned I : seq<unsigned>(Remaining.size()))
Sorted[I] = Remaining[Order.empty() ? I : Order[I]];
// No need to re-check for consecutive (stride-1) pairs here: the caller
// already disables masked stores for the whole base-object group when any
// consecutive pair exists, so this subset cannot contain one.
if (!Visited
.insert({Sorted.front(),
cast<StoreInst>(Sorted.front())->getValueOperand(),
Sorted.back(),
cast<StoreInst>(Sorted.back())->getValueOperand(),
Sorted.size()})
.second)
return Changed;
unsigned Size = 0;
if (vectorizeStoreChain(Sorted, R, /*Idx=*/0, /*MinVF=*/2, Size)
.value_or(false)) {
VectorizedStores.insert_range(Sorted);
Changed = true;
}
return Changed;
}
void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
// Initialize the collections. We will make a single pass over the block.
Stores.clear();
GEPs.clear();
// Visit the store and getelementptr instructions in BB and organize them in
// Stores and GEPs according to the underlying objects of their pointer
// operands.
for (Instruction &I : *BB) {
// Ignore store instructions that are volatile or have a pointer operand
// that doesn't point to a scalar type.
if (auto *SI = dyn_cast<StoreInst>(&I)) {
if (!SI->isSimple())
continue;
if (!isValidElementType(SI->getValueOperand()->getType(),
Opts->slp_revec))
continue;
Stores[getUnderlyingObject(SI->getPointerOperand())].push_back(SI);
}
// Ignore getelementptr instructions that have more than one index, a
// constant index, or a pointer operand that doesn't point to a scalar
// type.
else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
if (GEP->getNumIndices() != 1)
continue;
Value *Idx = GEP->idx_begin()->get();
if (isa<Constant>(Idx))
continue;
if (!isValidElementType(Idx->getType(), Opts->slp_revec))
continue;
if (GEP->getType()->isVectorTy())
continue;
GEPs[GEP->getPointerOperand()].push_back(GEP);
}
}
}
bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
bool MaxVFOnly,
bool StandaloneSeeds) {
if (VL.size() < 2)
return false;
LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n");
// Check that all of the parts are instructions of the same type,
// we permit an alternate opcode via InstructionsState.
InstructionsState S = getSameOpcode(VL, *TLI);
if (!S)
return false;
Instruction *I0 = S.getMainOp();
// Make sure invalid types (including vector type) are rejected before
// determining vectorization factor for scalar instructions.
for (Value *V : VL) {
Type *Ty = V->getType();
if (!isa<InsertElementInst, InsertValueInst>(V) &&
!isValidElementType(Ty, Opts->slp_revec)) {
// NOTE: the following will give user internal llvm type name, which may
// not be useful.
R.getORE()->emit([&]() {
std::string TypeStr;
llvm::raw_string_ostream OS(TypeStr);
Ty->print(OS);
return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
<< "Cannot SLP vectorize list: type "
<< TypeStr + " is unsupported by vectorizer";
});
return false;
}
}
Type *ScalarTy =
getValueType(VL[0], Opts->slp_revec, /*LookThroughCmp=*/true);
unsigned Sz = R.getVectorElementSize(I0);
unsigned MinVF = R.getMinVF(Sz);
// The buildvector, not filling the whole registers, is vectorized as a whole,
// not as the power-of-2 part plus the scalar remainder.
auto *IE = dyn_cast<InsertElementInst>(I0);
const bool AllowNonPowerOf2 =
Opts->slp_vectorize_non_power_of_2 ||
(IE && getNumElements(IE->getType()) <
getFullVectorNumberOfElements(
*TTI, IE->getType()->getScalarType(),
getNumElements(IE->getType()), Opts->slp_revec));
unsigned MaxVF =
std::max<unsigned>(isAllowedNonPowerOf2VF(VL.size(), AllowNonPowerOf2)
? VL.size()
: getFloorFullVectorNumberOfElements(
*TTI, ScalarTy, VL.size(), Opts->slp_revec),
MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
// Standalone seeds only need one register worth of lanes.
if (StandaloneSeeds && Sz != 0)
MaxVF = std::min(MaxVF, std::max(MinVF, R.getMaxVecRegSize() / Sz));
if (MaxVF < 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
<< "Cannot SLP vectorize list: vectorization factor "
<< "less than 2 is not supported";
});
return false;
}
bool Changed = false;
bool CandidateFound = false;
InstructionCost MinCost = R.getCostThreshold();
// The trees of the PHI nodes may walk the loop-carried chains, so the
// overlapping windows of a long list rebuild almost the same large trees.
// Limit total size to avoid compile time blow up.
const unsigned RejectedTreesBudget =
isa<PHINode>(I0) ? Opts->slp_phi_list_budget
: std::numeric_limits<unsigned>::max();
unsigned RejectedTreesSize = 0;
unsigned NextInst = 0, MaxInst = VL.size();
for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1,
Opts->slp_revec)) {
// No actual vectorization should happen, if number of parts is the same as
// provided vectorization factor (i.e. the scalar type is used for vector
// code during codegen).
auto *VecTy = getWidenedType(ScalarTy, VF);
if (R.getNumberOfParts(VecTy, ScalarTy) == VF)
continue;
// The window revisiting the dependent seeds skipped by the last vectorized
// window, 0 if there is none.
unsigned RevisitI = 0;
for (unsigned I = NextInst;
I < MaxInst && RejectedTreesSize <= RejectedTreesBudget; ++I) {
// The skipped seeds are revisited, continue as if they were not skipped.
if (RevisitI && I > RevisitI) {
I = NextInst - 1;
RevisitI = 0;
continue;
}
unsigned ActualVF = std::min(MaxInst - I, VF);
if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF,
Opts->slp_revec) &&
(ActualVF != VL.size() ||
!isAllowedNonPowerOf2VF(ActualVF, AllowNonPowerOf2)))
continue;
if (MaxVFOnly && ActualVF < MaxVF)
break;
if ((VF > MinVF && ActualVF < VF) || (VF == MinVF && ActualVF < 2))
break;
SmallVector<Value *> Ops(ActualVF, nullptr);
unsigned Idx = 0;
SmallPtrSet<const Value *, 8> Taken;
std::optional<unsigned> FirstSkipped;
for (auto [Off, V] : enumerate(VL.drop_front(I))) {
// Check that a previous iteration of this loop did not delete the
// Value.
auto *Inst = dyn_cast<Instruction>(V);
if (Inst && R.isDeleted(Inst))
continue;
// The lanes cannot depend on each other, so the seed, fed by the
// already taken one, is left for the next window.
if (StandaloneSeeds && Inst &&
any_of(Inst->operand_values(),
[&](const Value *Op) { return Taken.contains(Op); })) {
if (!FirstSkipped)
FirstSkipped = I + Off;
continue;
}
Ops[Idx] = V;
if (StandaloneSeeds)
Taken.insert(V);
++Idx;
if (Idx == ActualVF)
break;
}
// Not enough vectorizable instructions - exit. A skipped dependent seed
// may start the next window, so the scan must not stop because of it.
if (Idx != ActualVF) {
if (!FirstSkipped)
break;
continue;
}
// The identical lanes compute the very same value, so the vector node
// only duplicates it, and the window, rejected by another attempt in the
// block, is not analyzed again.
if (R.isAnalyzedBundle(Ops) ||
(StandaloneSeeds && all_of(drop_begin(Ops), [&](Value *V) {
return cast<Instruction>(Ops.front())
->isIdenticalTo(cast<Instruction>(V));
}))) {
if (StandaloneSeeds)
I += ActualVF - 1;
continue;
}
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << ActualVF << " operations "
<< "\n");
R.buildTree(Ops);
if (R.isTreeTinyAndNotFullyVectorizable()) {
RejectedTreesSize += R.getTreeSize();
R.analyzedBundle(Ops);
if (StandaloneSeeds)
I += ActualVF - 1;
continue;
}
if (R.isProfitableToReorder()) {
R.reorderTopToBottom();
R.reorderBottomToTop(
!isa<InsertElementInst, InsertValueInst>(Ops.front()));
}
R.transformNodes();
R.computeMinimumValueSizes();
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable();
R.buildExternalUses();
InstructionCost Cost = R.getTreeCost(TreeCost);
CandidateFound = true;
MinCost = std::min(MinCost, Cost);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
<< " for VF=" << ActualVF << "\n");
if (Cost < -R.getCostThreshold()) {
LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
cast<Instruction>(Ops[0]))
<< "SLP vectorized with cost " << ore::NV("Cost", Cost)
<< " and with tree size "
<< ore::NV("TreeSize", R.getTreeSize()));
R.vectorizeTree();
// Move to the next bundle. The skipped dependent seeds are fed by the
// vectorized lanes now, revisit them first.
NextInst = I + VF;
RevisitI = FirstSkipped.value_or(0);
I = FirstSkipped.value_or(NextInst) - 1;
Changed = true;
} else {
RejectedTreesSize += R.getTreeSize();
R.analyzedBundle(Ops);
if (StandaloneSeeds)
I += ActualVF - 1;
}
}
}
if (!Changed && CandidateFound) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0)
<< "List vectorization was possible but not beneficial with cost "
<< ore::NV("Cost", MinCost)
<< " >= " << ore::NV("Treshold", -R.getCostThreshold());
});
} else if (!Changed) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0)
<< "Cannot SLP vectorize list: vectorization was impossible"
<< " with available vectorization factors";
});
}
return Changed;
}
namespace {
/// Model horizontal reductions.
///
/// A horizontal reduction is a tree of reduction instructions that has values
/// that can be put into a vector as its leaves. For example:
///
/// mul mul mul mul
/// \ / \ /
/// + +
/// \ /
/// +
/// This tree has "mul" as its leaf values and "+" as its reduction
/// instructions. A reduction can feed into a store or a binary operation
/// feeding a phi.
/// ...
/// \ /
/// +
/// |
/// phi +=
///
/// Or:
/// ...
/// \ /
/// +
/// |
/// *p =
///
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
const VectorizeOptions &Opts;
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
/// Reduced values that enter the reduction with a flipped sign (from an
/// fsub/fneg chain link), subtracted in the final combine.
SmallPtrSet<Value *, 8> NegatedReducedVals;
/// Maps reduced value to the corresponding reduction operation.
SmallDenseMap<Value *, SmallVector<Instruction *>, 16> ReducedValsToOps;
WeakTrackingVH ReductionRoot;
/// The narrowing cast of the cast round-trip following the reduction
/// operation that consumes the reduced value at the same index (in
/// collection order, before the reversal to accumulation order), or null.
/// Ordered reductions with cast-interleaved links only.
SmallVector<Instruction *> RoundedLinks;
/// The type of reduction operation.
RecurKind RdxKind;
/// Checks if the optimization of original scalar identity operations on
/// matched horizontal reductions is enabled and allowed.
bool IsSupportedHorRdxIdentityOp = false;
/// The minimum number of the reduced values.
const unsigned ReductionLimit = Opts.slp_vectorize_non_power_of_2 ? 3 : 4;
/// Contains vector values for reduction including their scale factor,
/// signedness and sign of their contribution to the reduction result.
SmallVector<ReductionVectorPart> VectorValuesAndScales;
/// Narrow reduced values mapped to the shift and the mask applied after
/// widening.
SmallDenseMap<Value *, NarrowedLeafInfo> NarrowedLeafShifts;
/// zext/shl/and/packed-binop chain dropped with the reduction.
SmallVector<Instruction *> NarrowedChainInsts;
static bool isCmpSelMinMax(Instruction *I) {
return match(I, m_Select(m_Cmp(), m_Value(), m_Value())) &&
RecurrenceDescriptor::isMinMaxRecurrenceKind(getRdxKind(I));
}
// And/or are potentially poison-safe logical patterns like:
// select x, y, false
// select x, true, y
static bool isBoolLogicOp(Instruction *I) {
return isa<SelectInst>(I) &&
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
static ReductionOrdering isVectorizable(RecurKind Kind, Instruction *I,
bool TwoElementReduction = false) {
if (Kind == RecurKind::None)
return ReductionOrdering::None;
// Integer ops that map to select instructions or intrinsics are fine.
if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind) ||
isBoolLogicOp(I))
return ReductionOrdering::Unordered;
// No need to check for associativity, if 2 reduced values.
if (TwoElementReduction)
return ReductionOrdering::Unordered;
if (Kind == RecurKind::FMax || Kind == RecurKind::FMin) {
// FP min/max are associative except for NaN and -0.0. We do not
// have to rule out -0.0 here because the intrinsic semantics do not
// specify a fixed result for it.
return I->getFastMathFlags().noNaNs() ? ReductionOrdering::Unordered
: ReductionOrdering::Ordered;
}
if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
return ReductionOrdering::Unordered;
// Reassociation alone cannot change the sign of a zero result, so nsz
// is not required for fadd reductions.
if (I->isAssociative() || (Kind == RecurKind::FAdd && I->hasAllowReassoc()))
return ReductionOrdering::Unordered;
return isCommutative(I) ? ReductionOrdering::Ordered
: ReductionOrdering::None;
}
static Value *getRdxOperand(Instruction *I, unsigned Index) {
// Poison-safe 'or' takes the form: select X, true, Y
// To make that work with the normal operand processing, we skip the
// true value operand.
// TODO: Change the code and data structures to handle this without a hack.
if (getRdxKind(I) == RecurKind::Or && isa<SelectInst>(I) && Index == 1)
return I->getOperand(2);
return I->getOperand(Index);
}
/// Creates reduction operation with the current opcode.
static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
Value *RHS, const Twine &Name, bool UseSelect) {
Type *OpTy = LHS->getType();
assert(OpTy == RHS->getType() && "Expected LHS and RHS of same type");
switch (Kind) {
case RecurKind::Or: {
if (UseSelect && OpTy == CmpInst::makeCmpResultType(OpTy))
return Builder.CreateSelectWithUnknownProfile(
LHS, ConstantInt::getAllOnesValue(CmpInst::makeCmpResultType(OpTy)),
RHS, DEBUG_TYPE, Name);
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
Name);
}
case RecurKind::And: {
if (UseSelect && OpTy == CmpInst::makeCmpResultType(OpTy))
return Builder.CreateSelectWithUnknownProfile(
LHS, RHS,
ConstantInt::getNullValue(CmpInst::makeCmpResultType(OpTy)),
DEBUG_TYPE, Name);
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
Name);
}
case RecurKind::Add:
case RecurKind::Mul:
case RecurKind::Xor:
case RecurKind::FAdd:
case RecurKind::FSub:
case RecurKind::FMul: {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
Name);
}
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
if (UseSelect) {
CmpInst::Predicate Pred = llvm::getMinMaxReductionPredicate(Kind);
Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS, Name);
return Builder.CreateSelectWithUnknownProfile(Cmp, LHS, RHS, DEBUG_TYPE,
Name);
}
[[fallthrough]];
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum: {
Intrinsic::ID Id = llvm::getMinMaxReductionIntrinsicOp(Kind);
return Builder.CreateBinaryIntrinsic(Id, LHS, RHS);
}
default:
llvm_unreachable("Unknown reduction operation.");
}
}
/// \returns true if the disjoint flag of the reduction ops is kept for the
/// emitted ops.
bool keepDisjoint() const {
if (getBoolReduxWideTy(RdxKind, ReductionRoot->getType(),
ReducedVals.front().front()->getType()))
return false;
return all_of(make_second_range(NarrowedLeafShifts),
[](const NarrowedLeafInfo &L) { return L.Disjoint; });
}
/// Creates reduction operation with the current opcode with the IR flags
/// from \p ReductionOps, dropping nuw/nsw flags, and disjoint flag if \p
/// KeepDisjoint is false.
static Value *createOpWithFlags(IRBuilderBase &Builder, RecurKind RdxKind,
Value *LHS, Value *RHS, const Twine &Name,
const ReductionOpsListType &ReductionOps,
bool KeepDisjoint) {
bool UseSelect = ReductionOps.size() == 2 ||
// Logical or/and.
(ReductionOps.size() == 1 &&
any_of(ReductionOps.front(), IsaPred<SelectInst>));
assert((!UseSelect || ReductionOps.size() != 2 ||
isa<SelectInst>(ReductionOps[1][0])) &&
"Expected cmp + select pairs for reduction");
Value *Op = createOp(Builder, RdxKind, LHS, RHS, Name, UseSelect);
if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind)) {
if (auto *Sel = dyn_cast<SelectInst>(Op)) {
propagateIRFlags(Sel->getCondition(), ReductionOps[0], nullptr,
/*IncludeWrapFlags=*/false);
propagateIRFlags(Op, ReductionOps[1], nullptr,
/*IncludeWrapFlags=*/false);
return Op;
}
}
propagateIRFlags(Op, ReductionOps[0], nullptr, /*IncludeWrapFlags=*/false);
if (!KeepDisjoint && match(Op, m_DisjointOr(m_Value(), m_Value())))
cast<PossiblyDisjointInst>(Op)->setIsDisjoint(false);
return Op;
}
/// Creates reduction operation with the current opcode with the IR flags
/// from \p ReductionOps, dropping nuw/nsw flags.
Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
Value *RHS, const Twine &Name,
const ReductionOpsListType &ReductionOps) const {
return createOpWithFlags(Builder, Kind, LHS, RHS, Name, ReductionOps,
keepDisjoint());
}
public:
/// \returns true if \p V has an opcode that could form an ordered reduction.
static bool isSupportedOrderedReductionOp(Instruction *I) {
return I->getOpcode() == Instruction::FAdd;
}
static RecurKind getRdxKind(Value *V) {
auto *I = dyn_cast<Instruction>(V);
if (!I)
return RecurKind::None;
if (match(I, m_Add(m_Value(), m_Value())))
return RecurKind::Add;
if (match(I, m_Mul(m_Value(), m_Value())))
return RecurKind::Mul;
if (match(I, m_And(m_Value(), m_Value())) ||
match(I, m_LogicalAnd(m_Value(), m_Value())))
return RecurKind::And;
if (match(I, m_Or(m_Value(), m_Value())) ||
match(I, m_LogicalOr(m_Value(), m_Value())))
return RecurKind::Or;
if (match(I, m_Xor(m_Value(), m_Value())))
return RecurKind::Xor;
if (match(I, m_FAdd(m_Value(), m_Value())))
return RecurKind::FAdd;
if (match(I, m_FMul(m_Value(), m_Value())))
return RecurKind::FMul;
if (match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
return RecurKind::FMax;
if (match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
return RecurKind::FMin;
if (match(I, m_FMaximum(m_Value(), m_Value())))
return RecurKind::FMaximum;
if (match(I, m_FMinimum(m_Value(), m_Value())))
return RecurKind::FMinimum;
// TODO: remove several special-case paths that deal with selects.
if (match(I, m_SMax(m_Value(), m_Value())))
return RecurKind::SMax;
if (match(I, m_SMin(m_Value(), m_Value())))
return RecurKind::SMin;
if (match(I, m_UMax(m_Value(), m_Value())))
return RecurKind::UMax;
if (match(I, m_UMin(m_Value(), m_Value())))
return RecurKind::UMin;
if (auto *Select = dyn_cast<SelectInst>(I)) {
// Try harder: look for min/max pattern based on instructions producing
// same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
// During the intermediate stages of SLP, it's very common to have
// pattern like this (since optimizeGatherSequence is run only once
// at the end):
// %1 = extractelement <2 x i32> %a, i32 0
// %2 = extractelement <2 x i32> %a, i32 1
// %cond = icmp sgt i32 %1, %2
// %3 = extractelement <2 x i32> %a, i32 0
// %4 = extractelement <2 x i32> %a, i32 1
// %select = select i1 %cond, i32 %3, i32 %4
CmpPredicate Pred;
Instruction *L1;
Instruction *L2;
Value *LHS = Select->getTrueValue();
Value *RHS = Select->getFalseValue();
Value *Cond = Select->getCondition();
// TODO: Support inverse predicates.
if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) {
if (!isa<ExtractElementInst>(RHS) ||
!L2->isIdenticalTo(cast<Instruction>(RHS)))
return RecurKind::None;
} else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) {
if (!isa<ExtractElementInst>(LHS) ||
!L1->isIdenticalTo(cast<Instruction>(LHS)))
return RecurKind::None;
} else {
if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS))
return RecurKind::None;
if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) ||
!L1->isIdenticalTo(cast<Instruction>(LHS)) ||
!L2->isIdenticalTo(cast<Instruction>(RHS)))
return RecurKind::None;
}
switch (Pred) {
default:
return RecurKind::None;
case CmpInst::ICMP_SGT:
case CmpInst::ICMP_SGE:
return RecurKind::SMax;
case CmpInst::ICMP_SLT:
case CmpInst::ICMP_SLE:
return RecurKind::SMin;
case CmpInst::ICMP_UGT:
case CmpInst::ICMP_UGE:
return RecurKind::UMax;
case CmpInst::ICMP_ULT:
case CmpInst::ICMP_ULE:
return RecurKind::UMin;
}
}
return RecurKind::None;
}
/// Get the index of the first operand.
static unsigned getFirstOperandIndex(Instruction *I) {
return isCmpSelMinMax(I) ? 1 : 0;
}
private:
/// Total number of operands in the reduction operation.
static unsigned getNumberOfOperands(Instruction *I) {
return isCmpSelMinMax(I) ? 3 : (I->isUnaryOp() ? 1 : 2);
}
/// Expected number of uses for reduction operations/reduced values.
static bool hasRequiredNumberOfUses(bool IsCmpSelMinMax, Instruction *I) {
if (IsCmpSelMinMax) {
// SelectInst must be used twice while the condition op must have single
// use only.
if (auto *Sel = dyn_cast<SelectInst>(I))
return Sel->hasNUses(2) && Sel->getCondition()->hasOneUse();
return I->hasNUses(2);
}
// Arithmetic reduction operation must be used once only.
return I->hasOneUse();
}
/// Initializes the list of reduction operations.
void initReductionOps(Instruction *I) {
if (isCmpSelMinMax(I))
ReductionOps.assign(2, ReductionOpsType());
else
ReductionOps.assign(1, ReductionOpsType());
}
/// Add all reduction operations for the reduction instruction \p I.
void addReductionOps(Instruction *I) {
if (isCmpSelMinMax(I)) {
ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
ReductionOps[1].emplace_back(I);
} else {
ReductionOps[0].emplace_back(I);
}
}
static bool isGoodForReduction(ArrayRef<Value *> Data) {
int Sz = Data.size();
auto *I = dyn_cast<Instruction>(Data.front());
return Sz > 1 || isConstant(Data.front()) ||
(I && !isa<LoadInst>(I) && isValidForAlternation(I->getOpcode()));
}
/// Optimizes original placement of the reduced values for the reduction tree.
/// For example, if there is a zext i1 + selects, we can merge select
/// into zext and improve emission of the reductions.
void optimizeReducedVals(BoUpSLP &R, DominatorTree &DT, const DataLayout &DL,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI) {
SmallDenseMap<unsigned, unsigned> UsedReductionOpIds;
for (const auto [Idx, Vals] : enumerate(ReducedVals)) {
if (auto *I = dyn_cast<Instruction>(Vals.front()))
UsedReductionOpIds.try_emplace(I->getOpcode(), Idx);
}
// Check if zext i1 can be merged with select.
auto ZExtIt = UsedReductionOpIds.find(Instruction::ZExt);
auto SelectIt = UsedReductionOpIds.find(Instruction::Select);
if (ZExtIt != UsedReductionOpIds.end() &&
SelectIt != UsedReductionOpIds.end()) {
unsigned ZExtIdx = ZExtIt->second;
unsigned SelectIdx = SelectIt->second;
auto *ZExt = cast<ZExtInst>(ReducedVals[ZExtIdx].front());
// ZExt is compatible with Select? Merge select to zext, if so.
if (ZExt->getSrcTy()->isIntegerTy(1) &&
ZExt->getType() == ReducedVals[SelectIdx].front()->getType()) {
ReducedVals[ZExtIdx].append(ReducedVals[SelectIdx]);
ReducedVals.erase(std::next(ReducedVals.begin(), SelectIdx));
}
}
// Merge 1 element reduced value groups into larger group of shl, if only 2
// groups available. May trigger extra vectorization with the copyables.
if (ReducedVals.size() == 2 &&
(ReducedVals.front().size() == 1 || ReducedVals.back().size() == 1)) {
SmallVector<Value *> Ops(ReducedVals.front().size() +
ReducedVals.back().size());
copy(ReducedVals.front(), Ops.begin());
copy(ReducedVals.back(),
std::next(Ops.begin(), ReducedVals.front().size()));
InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
InstructionsState OpS = Analysis.buildInstructionsState(
Ops, R, /*TryCopyableElementsVectorization=*/true,
/*WithProfitabilityCheck=*/true);
if (OpS && OpS.areInstructionsWithCopyableElements() &&
OpS.getOpcode() == Instruction::Shl) {
// The smallest reduced values group should be the first.
if (ReducedVals.back().size() == 1 && ReducedVals.front().size() != 1)
std::swap(ReducedVals.front(), ReducedVals.back());
// Check if the largest reduced values group are shl and sort them by
// the constant shift amount to improve chances of vectorization with
// the copyables.
auto Comparator = [](Value *V1, Value *V2) {
ConstantInt *C1, *C2;
if (!match(V1, m_Shl(m_Value(), m_ConstantInt(C1))))
return false;
if (!match(V2, m_Shl(m_Value(), m_ConstantInt(C2))))
return true;
return C1->getZExtValue() < C2->getZExtValue();
};
stable_sort(ReducedVals.back(), Comparator);
ReducedVals.front().append(ReducedVals.back());
ReducedVals.pop_back();
}
}
}
public:
explicit HorizontalReduction(const VectorizeOptions &Opts) : Opts(Opts) {}
HorizontalReduction(const VectorizeOptions &Opts, Instruction *I,
ArrayRef<Value *> Ops)
: Opts(Opts), ReductionRoot(I), ReductionLimit(2) {
RdxKind = HorizontalReduction::getRdxKind(I);
ReductionOps.emplace_back().push_back(I);
ReducedVals.emplace_back().assign(Ops.begin(), Ops.end());
for (Value *V : Ops)
ReducedValsToOps[V].push_back(I);
}
/// Returns true if the matched reduction chain has flattened fsub/fneg
/// links (the leaves behind them may all be positive, e.g. with double
/// negations).
bool hasFlattenedNegations() const {
return any_of(ReductionOps, [](ArrayRef<Value *> Ops) {
return any_of(Ops, [](Value *Op) {
return Op && isFlattenableNegation(cast<Instruction>(Op));
});
});
}
bool matchReductionForOperands() {
// Analyze "regular" integer/FP types for reductions - no target-specific
// types or pointers.
assert(ReductionRoot && "Reduction root is not set!");
RK = isVectorizable(RdxKind, cast<Instruction>(ReductionRoot),
all_of(ReducedVals, [](ArrayRef<Value *> Ops) {
return Ops.size() == 2;
}));
return RK != ReductionOrdering::None;
}
/// Analyze whether \p Root forms a linearized ordered reduction chain.
/// If \p MatchLHS is true, analyzes LHS-associated (left-linearized) chains
/// where the chain recurses on the LHS and the RHS at each level is a leaf:
/// ((((v0 op v1) op v2) op v3) op v4)
/// If \p MatchLHS is false, analyzes RHS-associated (right-linearized) chains
/// where the chain recurses on the RHS and the LHS at each level is a leaf:
/// (v0 op (v1 op (v2 op (v3 op v4))))
/// Leaf values are stored in ReducedVals.back() in accumulation order
/// (innermost pair first, outermost last), e.g. [v0,v1,v2,v3,v4] for
/// LHS-associated and [v4,v3,v2,v1,v0] for RHS-associated chains.
/// \returns true if the chain is a valid ordered reduction.
bool matchOrderedReduction(BoUpSLP &R, Instruction *Root, bool MatchLHS) {
ReducedVals.clear();
ReducedValsToOps.clear();
ReductionOps.clear();
RoundedLinks.clear();
NarrowedLeafShifts.clear();
NarrowedChainInsts.clear();
NegatedReducedVals.clear();
RdxKind = getRdxKind(Root);
// Currently, only ordered fadd reductions are supported.
if (RdxKind != RecurKind::FAdd)
return false;
// Reassociable chains with non-elidable cast round-trip links are
// handled here as well: the emission keeps the original order.
if (isVectorizable(RdxKind, Root) == ReductionOrdering::None)
return false;
// Ordered reductions only support simple binary ops, not min/max
// select(cmp) patterns or poison-safe bool logic ops.
if (isCmpSelMinMax(Root) || isBoolLogicOp(Root))
return false;
Type *Ty = Root->getType();
if (!isValidElementType(Ty, Opts.slp_revec) || Ty->isPointerTy())
return false;
// This is an ordered (linearized) reduction regardless of whether the
// individual operations are associative.
RK = ReductionOrdering::Ordered;
ReductionRoot = Root;
BasicBlock *BB = Root->getParent();
unsigned FirstOpIdx = getFirstOperandIndex(Root);
unsigned ChainOpIdx = MatchLHS ? FirstOpIdx : FirstOpIdx + 1;
unsigned LeafOpIdx = MatchLHS ? FirstOpIdx + 1 : FirstOpIdx;
initReductionOps(Root);
ReducedVals.emplace_back();
SmallPtrSet<Instruction *, 16> Visited;
Instruction *TreeN = Root;
unsigned Depth = 0;
bool ChainComplete = false;
constexpr unsigned MaxReducedVals = 1024;
Instruction *PendingCast = nullptr;
while (TreeN) {
if (Depth++ > Opts.slp_recursion_max_depth)
break;
if (ReducedVals.back().size() >= MaxReducedVals)
break;
if (!Visited.insert(TreeN).second)
break;
if (getRdxKind(TreeN) != RdxKind)
break;
if (TreeN->getParent() != BB)
break;
addReductionOps(TreeN);
Value *LeafVal = getRdxOperand(TreeN, LeafOpIdx);
Value *ChainVal = getRdxOperand(TreeN, ChainOpIdx);
ReducedValsToOps[LeafVal].push_back(TreeN);
ReducedValsToOps[ChainVal].push_back(TreeN);
ReducedVals.back().push_back(LeafVal);
RoundedLinks.push_back(PendingCast);
PendingCast = nullptr;
auto *ChainInst = dyn_cast<Instruction>(ChainVal);
// The link may round the accumulator to a narrower type and extend it
// back; look through the cast pair, the rounding is re-emitted later.
if (ChainInst && getRdxKind(ChainInst) != RdxKind) {
if (Instruction *NarrowCast =
lookThroughCastRoundTrip(ChainVal, /*MustBeElidable=*/false)) {
auto *I = cast<Instruction>(NarrowCast->getOperand(0));
if (getRdxKind(I) == RdxKind &&
hasRequiredNumberOfUses(/*IsCmpSelMinMax=*/false, I)) {
ReductionOps[0].push_back(ChainVal);
ReductionOps[0].push_back(NarrowCast);
PendingCast = NarrowCast;
ChainInst = I;
}
}
}
if (!ChainInst || getRdxKind(ChainInst) != RdxKind ||
!hasRequiredNumberOfUses(/*IsCmpSelMinMax=*/false, ChainInst)) {
ReducedVals.back().push_back(ChainVal);
RoundedLinks.push_back(nullptr);
ChainComplete = true;
break;
}
TreeN = ChainInst;
}
// Leaves are collected outer-to-inner (top-down). Reverse to get the
// accumulation order (innermost first) needed by ordered reduction
// intrinsics. This is correct for both LHS- and RHS-associated chains
// because fadd is commutative — each step only relies on a+b == b+a,
// never on associativity.
if (!ChainComplete || ReducedVals.back().size() < ReductionLimit) {
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
R.analyzedReductionRoot(cast<Instruction>(RdxOp));
ReducedVals.pop_back();
ReducedValsToOps.clear();
ReductionOps.clear();
RoundedLinks.clear();
return false;
}
std::reverse(ReducedVals.back().begin(), ReducedVals.back().end());
return true;
}
/// Checks if \p I is an fsub/fneg, which can be flattened into an fadd
/// reduction chain with a flipped sign of its subtracted operand. nsz is
/// required: subtracted leaves are regrouped and negated as a whole, and
/// -a + -b == -(a + b) may flip the sign of a zero result.
static bool isFlattenableNegation(Instruction *I) {
return (I->getOpcode() == Instruction::FSub ||
I->getOpcode() == Instruction::FNeg) &&
I->hasAllowReassoc() && I->hasNoSignedZeros();
}
/// A group of loads or non-instructions rarely amortizes on its own the
/// cost of the reduction operation, charged to the first vectorized group.
static bool isCheapReductionGroup(ArrayRef<Value *> P) {
return !isa<Instruction>(P.front()) || isa<LoadInst>(P.front());
}
/// Try to find a reduction tree. If \p FlattenNegations is false, the
/// reassociable fsub/fneg links of an fadd chain are not flattened (they are
/// leaves of the reduction).
bool matchAssociativeReduction(BoUpSLP &R, Instruction *Root,
ScalarEvolution &SE, DominatorTree &DT,
const DataLayout &DL,
const TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI,
bool FlattenNegations = true) {
RdxKind = HorizontalReduction::getRdxKind(Root);
// A reassociable fsub root is a flattened link of an fadd reduction: its
// subtracted operand enters with a flipped sign. Without the flattening
// there is no reduction to match. (An fneg is not a reduction candidate.)
const bool IsNegatedRoot = RdxKind == RecurKind::None &&
Root->getOpcode() == Instruction::FSub &&
isFlattenableNegation(Root);
if (IsNegatedRoot)
RdxKind = RecurKind::FAdd;
RK = isVectorizable(RdxKind, Root);
if (RK == ReductionOrdering::None)
return false;
// Analyze "regular" integer/FP types for reductions - no target-specific
// types or pointers.
Type *Ty = Root->getType();
if (!isValidElementType(Ty, Opts.slp_revec) || Ty->isPointerTy())
return false;
// Though the ultimate reduction may have multiple uses, its condition must
// have only single use.
if (auto *Sel = dyn_cast<SelectInst>(Root))
if (!Sel->getCondition()->hasOneUse())
RK = ReductionOrdering::Ordered;
ReductionRoot = Root;
// Iterate through all the operands of the possible reduction tree and
// gather all the reduced values, sorting them by their value id.
BasicBlock *BB = Root->getParent();
bool IsCmpSelMinMax = isCmpSelMinMax(Root);
const ReductionOrdering InitialRK = RK;
// For unordered fadd reductions, reassociable fsub/fneg chain links are
// flattened with a flipped sign on the subtracted operand: the leaves,
// reduced through such links, are subtracted in the final combine.
// Ordered reductions keep their accumulation order and are excluded.
bool TrackSign = FlattenNegations && RdxKind == RecurKind::FAdd &&
RK == ReductionOrdering::Unordered;
// The leaf signs are modeled per value. If they cannot be modeled this way
// (the same value occurs both added and subtracted, or the reduction
// switches to ordered after some fsub/fneg links have been flattened
// already), the analysis restarts without the fsub/fneg flattening.
bool Restart = false;
do {
if (Restart) {
assert(TrackSign && "Expected a sign-tracking analysis to restart");
TrackSign = false;
}
// The root itself is a flattened link - there is no reduction without
// the flattening.
if (IsNegatedRoot && !TrackSign)
return false;
Restart = false;
RK = InitialRK;
ReducedVals.clear();
ReducedValsToOps.clear();
NegatedReducedVals.clear();
bool FlattenedNegations = false;
initReductionOps(Root);
DenseMap<std::pair<size_t, Value *>, SmallVector<LoadInst *>> LoadsMap;
SmallSet<size_t, 2> LoadKeyUsed;
auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
Key = hash_combine(hash_value(LI->getParent()->getNumber()), Key);
Value *Ptr = getUnderlyingObject(LI->getPointerOperand(),
Opts.slp_recursion_max_depth);
if (!LoadKeyUsed.insert(Key).second) {
auto LIt = LoadsMap.find(std::make_pair(Key, Ptr));
if (LIt != LoadsMap.end()) {
for (LoadInst *RLI : LIt->second) {
if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(),
LI->getType(), LI->getPointerOperand(), DL,
SE, /*StrictCheck=*/true))
return hash_value(RLI->getPointerOperand());
}
for (LoadInst *RLI : LIt->second) {
if (arePointersCompatible(RLI->getPointerOperand(),
LI->getPointerOperand(), TLI,
Opts.slp_recursion_max_depth)) {
hash_code SubKey = hash_value(RLI->getPointerOperand());
return SubKey;
}
}
if (LIt->second.size() > 2) {
hash_code SubKey =
hash_value(LIt->second.back()->getPointerOperand());
return SubKey;
}
}
}
LoadsMap.try_emplace(std::make_pair(Key, Ptr))
.first->second.push_back(LI);
return hash_value(LI->getPointerOperand());
};
SmallVector<Value *> ReducedValsCandidates;
bool AdjustedToOrdered = false;
// True if the chain has an opaque fsub/fneg leaf (not reassociable, extra
// uses, too deep).
bool HasOpaqueNegationLeaf = false;
SmallVector<std::tuple<Instruction *, unsigned, bool>> Worklist(
1, std::make_tuple(Root, 0, /*Negated=*/false));
SmallPtrSet<Value *, 8> Operands;
SmallVector<std::pair<Instruction *, unsigned>>
PossibleOrderedReductionOps;
// Leaves that (also) occur with a non-flipped sign.
SmallPtrSet<Value *, 8> PositiveReducedVals;
// Checks if the instruction continues the reduction chain. An fsub/fneg
// chain link in an fadd reduction is recursed into like an fadd, but its
// subtracted operand enters with a flipped sign. Only reassociable links
// qualify.
auto IsChainLink = [&](Instruction *I) {
if (getRdxKind(I) == RdxKind)
return true;
return TrackSign && isFlattenableNegation(I);
};
// Checks if the operands of the \p TreeN instruction are also reduction
// operations or should be treated as reduced values or an extra
// argument, which is not part of the reduction.
auto CheckOperands =
[&](Instruction *TreeN, SmallVectorImpl<Value *> &PossibleReducedVals,
SmallVectorImpl<std::pair<Instruction *, bool>> &ReductionOps,
ReductionOpsType &AllReductionOps, unsigned Level, bool Negated) {
for (int I : reverse(seq<int>(getFirstOperandIndex(TreeN),
getNumberOfOperands(TreeN)))) {
Value *EdgeVal = getRdxOperand(TreeN, I);
auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
// The link may be an elidable cast round-trip; look through it to
// continue the chain. It is dropped when the reduction is folded.
if (EdgeInst && getRdxKind(EdgeInst) != RdxKind) {
if (Instruction *NarrowCast = lookThroughCastRoundTrip(
EdgeVal, /*MustBeElidable=*/true)) {
auto *SrcI = cast<Instruction>(NarrowCast->getOperand(0));
if (getRdxKind(SrcI) == RdxKind &&
hasRequiredNumberOfUses(IsCmpSelMinMax, SrcI)) {
AllReductionOps.push_back(EdgeVal);
AllReductionOps.push_back(NarrowCast);
EdgeVal = SrcI;
EdgeInst = SrcI;
}
}
}
ReducedValsToOps[EdgeVal].push_back(TreeN);
// fsub flips its second operand's sign, fneg its only
// operand's.
bool EdgeNegated =
Negated !=
(TreeN->getOpcode() == Instruction::FNeg ||
(TreeN->getOpcode() == Instruction::FSub && I == 1));
// If the edge is not an instruction, or it is different from
// the main reduction opcode or has too many uses - possible
// reduced value. Also, do not try to reduce const values, if
// the operation is not foldable.
bool IsReducedVal = !EdgeInst ||
Level > Opts.slp_recursion_max_depth ||
!IsChainLink(EdgeInst) ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst);
ReductionOrdering CurrentRK =
IsReducedVal ? ReductionOrdering::None
: isVectorizable(RdxKind, EdgeInst);
if (!IsReducedVal && CurrentRK == ReductionOrdering::Unordered &&
RK == ReductionOrdering::Unordered &&
!hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst)) {
IsReducedVal = true;
CurrentRK = ReductionOrdering::None;
if (PossibleReducedVals.size() < ReductionLimit &&
!Operands.contains(EdgeInst))
PossibleOrderedReductionOps.emplace_back(EdgeInst, Level);
}
if (CurrentRK == ReductionOrdering::None ||
Operands.contains(EdgeInst) ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
// Keep the sign of the leaf. A leaf, occurring with both
// signs, cannot be modeled with a per-value sign - restart
// without sign tracking.
if (TrackSign) {
auto &Same =
EdgeNegated ? NegatedReducedVals : PositiveReducedVals;
auto &Other =
EdgeNegated ? PositiveReducedVals : NegatedReducedVals;
Same.insert(EdgeVal);
Restart |= Other.contains(EdgeVal);
HasOpaqueNegationLeaf |=
EdgeInst && (EdgeInst->getOpcode() == Instruction::FSub ||
EdgeInst->getOpcode() == Instruction::FNeg);
}
PossibleReducedVals.push_back(EdgeVal);
if (EdgeInst && !isCmpSelMinMax(EdgeInst))
Operands.insert_range(EdgeInst->operands());
continue;
}
if (CurrentRK == ReductionOrdering::Ordered)
RK = ReductionOrdering::Ordered;
ReductionOps.emplace_back(EdgeInst, EdgeNegated);
}
};
SmallPtrSet<Instruction *, 16> Visited;
while (!Worklist.empty()) {
auto [TreeN, Level, Negated] = Worklist.pop_back_val();
if (!Visited.insert(TreeN).second)
continue;
// A chain link other than the reduction operation is a flattened
// fsub/fneg.
if (getRdxKind(TreeN) != RdxKind)
FlattenedNegations = true;
SmallVector<Value *> PossibleRedVals;
SmallVector<std::pair<Instruction *, bool>> PossibleReductionOps;
CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps,
ReductionOps[0], Level, Negated);
// A leaf occurred with both signs - stop, the analysis restarts
// without the fsub/fneg flattening.
if (Restart)
break;
addReductionOps(TreeN);
ReducedValsCandidates.append(PossibleRedVals.begin(),
PossibleRedVals.end());
for (auto [I, OpNegated] : reverse(PossibleReductionOps))
Worklist.emplace_back(I, I->getParent() == BB ? 0 : Level + 1,
OpNegated);
// If not enough elements for unordered vectorization, check if there
// are potential candidates for the ordered vectorization and try to
// add them to the worklist.
if (Worklist.empty() && ReducedValsCandidates.size() < ReductionLimit &&
!PossibleOrderedReductionOps.empty() &&
RK == ReductionOrdering::Unordered) {
RK = ReductionOrdering::Ordered;
AdjustedToOrdered = true;
SmallPtrSet<const Instruction *, 4> Ops;
// The ordered switch discards the tracked signs (the analysis
// restarts without the fsub/fneg flattening, if any occurred), so
// the candidates re-enter the walk with the non-flipped sign.
for (const auto &[I, Level] : PossibleOrderedReductionOps) {
Ops.insert(I);
Worklist.emplace_back(I, Level, /*Negated=*/false);
}
erase_if(ReducedValsCandidates, [&](Value *V) {
auto *I = dyn_cast<Instruction>(V);
return I && Ops.contains(I);
});
PossibleOrderedReductionOps.clear();
}
}
// The flattening reassociates the chain, which the ordered reduction
// must not do, and negating the regrouped subtracted leaves as a whole
// is not sign-of-zero-safe unless the whole flattened chain is nsz (this
// conservatively includes the elidable cast round-trips, collected into
// the reduction operations) - restart without the fsub/fneg flattening,
// if any occurred. Also, an opaque fsub/fneg leaf forms a homogeneous
// group with the flattened links when not flattened, which the
// sign-uniform regrouping of their operands may only fragment - keep the
// unflattened form.
if (FlattenedNegations &&
(RK == ReductionOrdering::Ordered || HasOpaqueNegationLeaf ||
(!NegatedReducedVals.empty() &&
any_of(ReductionOps.front(), [](Value *Op) {
return !cast<Instruction>(Op)->hasNoSignedZeros();
}))))
Restart = true;
if (Restart)
continue;
// InstCombine may pack same-kind binops under a zext, hiding the
// leaves. Analyze them in the narrow type; the vectorized result is
// widened back, shifted and masked per lane before the reduction.
if (Instruction::isBitwiseLogicOp(
RecurrenceDescriptor::getOpcode(RdxKind)) &&
RK == ReductionOrdering::Unordered && !IsCmpSelMinMax &&
Ty->isIntegerTy() && !Ty->isIntegerTy(1) &&
any_of(ReducedValsCandidates, [](Value *V) {
return isa<ZExtInst>(V) ||
match(V, m_Shl(m_ZExt(m_Value()), m_ConstantInt()));
})) {
unsigned WideBW = Ty->getIntegerBitWidth();
SmallVector<SmallVector<NarrowedLeafInfo>> AllLeaves;
SmallVector<Instruction *> ChainInsts;
Type *NarrowTy = nullptr;
bool Bail = false;
unsigned NumLeaves = 0;
SmallPtrSet<Value *, 8> UniqueLeaves;
for (Value *Cand : ReducedValsCandidates) {
SmallVector<NarrowedLeafInfo> &Leaves = AllLeaves.emplace_back();
collectNarrowedLeaves(Cand, RecurrenceDescriptor::getOpcode(RdxKind),
WideBW, Opts.slp_recursion_max_depth, Leaves,
ChainInsts);
for (const NarrowedLeafInfo &L : Leaves) {
if (isa<Constant>(L.V) || !L.V->getType()->isIntegerTy() ||
(NarrowTy && L.V->getType() != NarrowTy) ||
!UniqueLeaves.insert(L.V).second) {
Bail = true;
break;
}
if (!NarrowTy)
NarrowTy = L.V->getType();
}
if (Bail)
break;
NumLeaves += Leaves.size();
}
if (!Bail && NumLeaves > ReducedValsCandidates.size() &&
NarrowTy != Ty) {
// Collected user-first; dies with the reduction iff all users die.
SmallPtrSet<Value *, 8> Ignorable;
for (ArrayRef<Value *> RdxOps : ReductionOps)
Ignorable.insert_range(RdxOps);
for (Instruction *I : ChainInsts)
if (all_of(I->users(),
[&](User *U) { return Ignorable.contains(U); })) {
Ignorable.insert(I);
NarrowedChainInsts.push_back(I);
}
const bool AllDisjoint =
all_of(make_isa_range<PossiblyDisjointInst>(ChainInsts),
[](PossiblyDisjointInst *I) { return I->isDisjoint(); });
SmallVector<Value *> NewCandidates;
for (const auto &[Cand, Leaves] :
zip(ReducedValsCandidates, AllLeaves)) {
SmallVector<Instruction *> RdxOps = ReducedValsToOps.lookup(Cand);
for (NarrowedLeafInfo &L : Leaves) {
L.Disjoint = AllDisjoint;
NarrowedLeafShifts.try_emplace(L.V, L);
NewCandidates.push_back(L.V);
ReducedValsToOps[L.V].append(RdxOps);
}
}
ReducedValsCandidates = std::move(NewCandidates);
}
}
// Cap on the number of reduced values, including booleanized leaves.
constexpr unsigned ReducedValsLimit = 1024;
// Logical and/or reductions on i1 may have booleanized wide leaves:
// truncs of same-op chains or zero-tests of values from [0, 1].
// Reducing in the wide type and taking bit 0 of the result is
// equivalent, since trunc of a bitwise and/or is the bitwise and/or of
// the truncs, and it exposes the underlying (usually consecutive) wide
// values to the tree. All final leaves must be non-poison: the logical
// select/and/or ops may mask poison, which the wide reduction would
// propagate instead.
if ((RdxKind == RecurKind::And || RdxKind == RecurKind::Or) &&
RK == ReductionOrdering::Unordered && !IsCmpSelMinMax &&
Ty->isIntegerTy(1) && any_of(ReducedValsCandidates, [](Value *V) {
return isa<TruncInst, ICmpInst>(V);
})) {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
auto BooleanizeLeaves = [&]() {
Type *WideTy = nullptr;
SmallVector<Value *> NewCandidates;
SmallVector<Instruction *> ChainInsts;
for (Value *Cand : ReducedValsCandidates) {
SmallVector<Instruction *> RdxOps = ReducedValsToOps.lookup(Cand);
SmallVector<Value *> Worklist(1, Cand);
while (!Worklist.empty()) {
Value *V = Worklist.pop_back_val();
if (V->getType()->isIntegerTy(1)) {
Value *X;
// trunc iN to i1: bit 0 of the wide value.
if (match(V, m_OneUse(m_Trunc(m_Value(X))))) {
ChainInsts.push_back(cast<Instruction>(V));
Worklist.push_back(X);
continue;
}
// Zero-test of a value from [0, 1]: same as bit 0 of the
// value.
CmpPredicate Pred;
if (match(V, m_OneUse(m_c_ICmp(Pred, m_Value(X), m_Zero()))) &&
Pred == ICmpInst::ICMP_NE && X->getType()->isIntegerTy() &&
computeKnownBits(X, DL).getMaxValue() == 1) {
ChainInsts.push_back(cast<Instruction>(V));
Worklist.push_back(X);
continue;
}
return;
}
if (!V->getType()->isIntegerTy())
return;
// Same-op chain node: bit 0 of a bitwise and/or is the and/or
// of the operands' bit 0.
if (auto *BO = dyn_cast<BinaryOperator>(V);
BO && BO->getOpcode() == RdxOpcode && BO->hasOneUse() &&
!BO->hasPoisonGeneratingFlags()) {
ChainInsts.push_back(BO);
Worklist.push_back(BO->getOperand(1));
Worklist.push_back(BO->getOperand(0));
continue;
}
if (!isGuaranteedNotToBePoison(V) ||
(WideTy && V->getType() != WideTy))
return;
if (!WideTy)
WideTy = V->getType();
NewCandidates.push_back(V);
ReducedValsToOps[V].append(RdxOps);
if (NewCandidates.size() > ReducedValsLimit)
return;
}
}
if (WideTy) {
NarrowedChainInsts.append(ChainInsts);
ReducedValsCandidates = std::move(NewCandidates);
}
};
BooleanizeLeaves();
}
// Too many integer reduced values candidates for the ordered
// reductions after adjustements - try to switch to unordered
// reductions instead.
if (ReducedValsCandidates.size() > ReducedValsLimit &&
AdjustedToOrdered &&
ReducedValsCandidates.front()->getType()->isIntOrIntVectorTy())
return false;
// Try to regroup reduced values so that it gets more profitable to try
// to reduce them. Values are grouped by their value ids, instructions -
// by instruction op id and/or alternate op id, plus do extra analysis
// for loads (grouping them by the distance between pointers) and cmp
// instructions (grouping them by the predicate).
SmallMapVector<
size_t,
SmallMapVector<size_t, SmallMapVector<Value *, unsigned, 2>, 2>, 8>
PossibleReducedVals;
// Add reduction values. The values are sorted for better vectorization
// results.
for (Value *V : make_filter_range(ReducedValsCandidates, [&](Value *V) {
return RK != ReductionOrdering::Ordered || isa<Instruction>(V);
})) {
size_t Key, Idx;
std::tie(Key, Idx) = generateKeySubkey(V, &TLI, GenerateLoadsSubkey,
/*AllowAlternate=*/false);
// The sign of the contribution is a part of the grouping key: a
// mixed-sign group cannot be negated as a whole.
Key = hash_combine(Key, NegatedReducedVals.contains(V));
++PossibleReducedVals[Key][Idx].try_emplace(V, 0).first->second;
}
auto PossibleReducedValsVect = PossibleReducedVals.takeVector();
// Sort values by the total number of values kinds to start the
// reduction from the longest possible reduced values sequences.
for (auto &PossibleReducedVals : PossibleReducedValsVect) {
auto PossibleRedVals = PossibleReducedVals.second.takeVector();
SmallVector<SmallVector<Value *>> PossibleRedValsVect;
for (auto &Slice : PossibleRedVals) {
PossibleRedValsVect.emplace_back();
auto RedValsVect = Slice.second.takeVector();
stable_sort(RedValsVect, llvm::less_second());
for (const std::pair<Value *, unsigned> &Data : RedValsVect)
PossibleRedValsVect.back().append(Data.second, Data.first);
}
stable_sort(PossibleRedValsVect, [](const auto &P1, const auto &P2) {
return P1.size() > P2.size();
});
bool First = true;
for (ArrayRef<Value *> Data : PossibleRedValsVect) {
if (First) {
First = false;
ReducedVals.emplace_back();
} else if (!isGoodForReduction(Data)) {
auto *LI = dyn_cast<LoadInst>(Data.front());
auto *LastLI = dyn_cast<LoadInst>(ReducedVals.back().front());
if (!LI || !LastLI ||
getUnderlyingObject(LI->getPointerOperand()) !=
getUnderlyingObject(LastLI->getPointerOperand()))
ReducedVals.emplace_back();
}
ReducedVals.back().append(Data.rbegin(), Data.rend());
}
}
// Post optimize reduced values to get better reduction sequences and
// sort them by size.
optimizeReducedVals(R, DT, DL, TTI, TLI);
// Sort the reduced values by number of same/alternate opcode and/or
// pointer operand. For the sign-aware reductions, same-size groups are
// ordered by their expected profitability: the first vectorized group
// is charged the cost of the reduction operation itself, which a group
// of loads or non-instructions rarely amortizes on its own, while it is
// a cheap addition (a single vector operation) to an already
// vectorized group - such groups go last. Among the rest, non-negated
// groups go first.
stable_sort(ReducedVals, [&](ArrayRef<Value *> P1, ArrayRef<Value *> P2) {
if (P1.size() != P2.size())
return P1.size() > P2.size();
if (NegatedReducedVals.empty())
return false;
bool Cheap1 = isCheapReductionGroup(P1);
bool Cheap2 = isCheapReductionGroup(P2);
if (Cheap1 != Cheap2)
return Cheap2;
return NegatedReducedVals.contains(P1.front()) <
NegatedReducedVals.contains(P2.front());
});
// A group of subtracted loads, which are not consecutive (a gather):
// the flattening splits the fsub leaves over them into separate load
// groups of their operands, which the unflattened form vectorizes as a
// whole (a vector fsub with shuffled operands).
auto IsGatheredNegatedLoads = [&](ArrayRef<Value *> Vals) {
auto *LI0 = dyn_cast<LoadInst>(Vals.front());
if (!LI0 || Vals.size() < 2 || !NegatedReducedVals.contains(LI0))
return false;
SmallVector<int64_t> Diffs;
for (Value *V : Vals) {
auto *LI = dyn_cast<LoadInst>(V);
if (!LI)
return false;
std::optional<int64_t> Diff = getPointersDiff(
LI0->getType(), LI0->getPointerOperand(), LI->getType(),
LI->getPointerOperand(), DL, SE, /*StrictCheck=*/true);
if (!Diff)
return true;
Diffs.push_back(*Diff);
}
sort(Diffs);
Diffs.erase(llvm::unique(Diffs), Diffs.end());
return Diffs.back() - Diffs.front() + 1 !=
static_cast<int64_t>(Diffs.size());
};
// The flattening must not make the reduction unviable: the leaves,
// hidden behind the flattened fsub/fneg operations, may split into
// too small sign-uniform groups, while the unflattened form may
// still be vectorizable (e.g. with alternate fadd/fsub opcodes).
if (FlattenedNegations &&
(accumulate(ReducedVals, 0u,
[](unsigned Num, ArrayRef<Value *> Vals) -> unsigned {
if (!isGoodForReduction(Vals))
return Num;
return Num + Vals.size();
}) < ReductionLimit ||
any_of(ReducedVals, IsGatheredNegatedLoads)))
Restart = true;
} while (Restart);
return true;
}
/// Loop accumulator emission mode of the reduction: a reduction that
/// accumulates a loop phi (the phi is one of the reduced values and the
/// root is its backedge value) can be emitted as a vector accumulator in
/// the loop plus a single reduction after the loop instead of a horizontal
/// reduction on every iteration.
class LoopAccumulator {
/// The reduction root and the reduction operation kind and ordering.
Instruction *Root = nullptr;
RecurKind RdxKind = RecurKind::None;
ReductionOrdering RK = ReductionOrdering::None;
/// The vectorizer state and the analyses the accumulator cost and
/// emission are computed with.
BoUpSLP &R;
const TargetTransformInfo &TTI;
LoopInfo &LI;
/// The accumulated phi and the identity constant that replaced it in the
/// reduced values. The identity is never folded as a leftover: it does
/// not change the result.
PHINode *Phi = nullptr;
Constant *Identity = nullptr;
/// The number of calls in the loop, across which the vector accumulator
/// is kept live.
unsigned NumCalls = 0;
/// Exit phis consuming the reduction result outside the loop.
SmallSetVector<PHINode *, 2> ExitPhis;
/// \returns the identity constant of the reduction operation for \p Ty
/// under the fast-math flags \p FMF.
Constant *getIdentity(Type *Ty, FastMathFlags FMF) const {
Intrinsic::ID Id;
switch (RdxKind) {
case RecurKind::FMax:
case RecurKind::FMaxNum:
Id = Intrinsic::vector_reduce_fmax;
break;
case RecurKind::FMin:
case RecurKind::FMinNum:
Id = Intrinsic::vector_reduce_fmin;
break;
case RecurKind::FMaximum:
Id = Intrinsic::vector_reduce_fmaximum;
break;
case RecurKind::FMinimum:
Id = Intrinsic::vector_reduce_fminimum;
break;
default:
Id = RecurrenceDescriptor::isIntMinMaxRecurrenceKind(RdxKind)
? getMinMaxReductionIntrinsicID(
getMinMaxReductionIntrinsicOp(RdxKind))
: getReductionForBinop(static_cast<Instruction::BinaryOps>(
RecurrenceDescriptor::getOpcode(RdxKind)));
break;
}
return cast<Constant>(getReductionIdentity(Id, Ty, FMF));
}
/// \returns the extra cost of accumulating the slice lane-wise into the
/// vector accumulator instead of reducing it on every iteration: the
/// lane-wise operation, the spill and reload on every iteration of the
/// values live beyond the register file and keeping the vector
/// accumulator live across the calls of the loop.
InstructionCost getAccumulationCost(FastMathFlags FMF, VectorType *VectorTy,
Type *ScalarTy) const {
const TTI::TargetCostKind CostKind = R.getCostKind();
auto GetOpCost = [&](Type *OpTy) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind)) {
IntrinsicCostAttributes ICA(getMinMaxReductionIntrinsicOp(RdxKind),
OpTy, {OpTy, OpTy}, FMF);
return TTI.getIntrinsicInstrCost(ICA, CostKind);
}
return TTI.getArithmeticInstrCost(
RecurrenceDescriptor::getOpcode(RdxKind), OpTy, CostKind);
};
// The lane-wise operation; also replaces the scalar operation folding
// the accumulator phi on top of the horizontal reduction (the scalar
// cost of the slice credits it neither, the slice lacks the phi).
InstructionCost Cost = GetOpCost(VectorTy) - GetOpCost(ScalarTy);
// The vector accumulator and the reduced vector are live at the same
// time; the registers they need beyond the register file are spilled
// and reloaded on every iteration.
constexpr unsigned NumLiveVectors = 2;
unsigned Regs = TTI.getRegUsageForType(VectorTy);
unsigned RC = TTI.getRegisterClassForType(/*Vector=*/true, VectorTy);
unsigned NumRegs = TTI.getNumberOfRegisters(RC);
InstructionCost SpillReload =
TTI.getRegisterClassSpillCost(RC, CostKind) +
TTI.getRegisterClassReloadCost(RC, CostKind);
InstructionCost Spilled = 0;
if (NumRegs != 0 && NumLiveVectors * Regs > NumRegs)
Spilled = SpillReload * (NumLiveVectors * Regs - NumRegs);
Cost += Spilled;
// The vector accumulator replaces the scalar one across the calls of
// the loop: without callee-saved registers of the class it is spilled
// and reloaded around every call, like the scalar one. The accumulator
// registers already spilled on every iteration stay in memory across
// the calls and are not spilled again.
if (NumCalls != 0) {
InstructionCost VecLive =
std::max(TTI.getCostOfKeepingLiveOverCall(VectorTy),
SpillReload * Regs) -
std::min(Spilled, SpillReload * Regs);
unsigned SRC = TTI.getRegisterClassForType(/*Vector=*/false, ScalarTy);
InstructionCost ScalarLive =
std::max(TTI.getCostOfKeepingLiveOverCall(ScalarTy),
TTI.getRegisterClassSpillCost(SRC, CostKind) +
TTI.getRegisterClassReloadCost(SRC, CostKind));
Cost += (VecLive - ScalarLive) * NumCalls;
}
return Cost;
}
public:
LoopAccumulator(Instruction *Root, RecurKind RdxKind, ReductionOrdering RK,
BoUpSLP &R, const TargetTransformInfo &TTI, LoopInfo &LI)
: Root(Root), RdxKind(RdxKind), RK(RK), R(R), TTI(TTI), LI(LI) {}
/// If the reduction accumulates a loop phi (the phi is one of the reduced
/// values and the root is its backedge value), replaces the phi by the
/// reduction identity in the reduced values, so that the emitted tree
/// does not reference the phi. The identity does not change the result,
/// so if the accumulator emission does not apply, the phi is just folded
/// back on top of the emitted reduction. All unordered reduction kinds
/// have an identity constant.
void prepare(FastMathFlags RdxFMF,
SmallVectorImpl<SmallVector<Value *>> &ReducedVals,
SmallDenseMap<Value *, SmallVector<Instruction *>, 16>
&ReducedValsToOps,
const SmallPtrSetImpl<Value *> &NegatedReducedVals,
bool HasNarrowedLeafShifts) {
// A two-value reduction is unordered by commutativity alone; the
// accumulator reassociates the reduction across the loop iterations,
// which requires a genuinely associative operation.
if (!R.Opts.slp_vectorize_loop_acc_rdx ||
RK != ReductionOrdering::Unordered ||
isVectorizable(RdxKind, Root) != ReductionOrdering::Unordered ||
Root->getType()->isIntegerTy(1) || Root->getType()->isVectorTy() ||
HasNarrowedLeafShifts || isCmpSelMinMax(Root) ||
Root->hasNUsesOrMore(UsesLimit))
return;
Loop *L = LI.getLoopFor(Root->getParent());
if (!L)
return;
BasicBlock *Latch = L->getLoopLatch();
if (!Latch)
return;
// Only a single accumulated phi, used by the reduction operations
// only, is supported.
auto FindAccPhi = [&]() -> PHINode * {
PHINode *AccPhi = nullptr;
for (ArrayRef<Value *> Candidates : ReducedVals)
for (PHINode *P : make_isa_range<PHINode>(Candidates)) {
if (P->getParent() != L->getHeader() ||
P->getNumIncomingValues() > MaxPHINumOperands ||
P->getIncomingValueForBlock(Latch) != Root)
continue;
if (AccPhi || P->hasNUsesOrMore(UsesLimit) ||
P->getNumUses() != ReducedValsToOps.at(P).size())
return nullptr;
AccPhi = P;
}
return AccPhi;
};
PHINode *AccPhi = FindAccPhi();
// A negated phi is subtracted from the reduction result, not
// accumulated.
if (!AccPhi || NegatedReducedVals.contains(AccPhi))
return;
Constant *IdC = getIdentity(AccPhi->getType(), RdxFMF);
if (ReducedValsToOps.contains(IdC))
return;
// The root may be used only by the accumulator phi and by phis outside
// the loop (the exit phis). The initial values are inserted into the
// identity vector at the end of their incoming blocks, so they must not
// be defined by the terminators; the final reduction is emitted at the
// beginning of the exit blocks, which must not be EH pads. Neither may
// land in a loop not containing the reduction loop, which would execute
// them repeatedly.
auto IsExecutedOncePerLoop = [&](BasicBlock *BB) {
Loop *BBL = LI.getLoopFor(BB);
return !BBL || BBL->contains(L);
};
if (any_of(zip(AccPhi->incoming_values(), AccPhi->blocks()),
[&](const auto &P) {
auto [IncV, B] = P;
return IncV != Root && (IncV == B->getTerminator() ||
!IsExecutedOncePerLoop(B));
}))
return;
auto IsValidExitPhi = [&](User *U) {
auto *ExitPhi = dyn_cast<PHINode>(U);
return ExitPhi &&
ExitPhi->getNumIncomingValues() <= MaxPHINumOperands &&
!ExitPhi->getParent()->isEHPad() &&
!L->contains(ExitPhi->getParent()) &&
IsExecutedOncePerLoop(ExitPhi->getParent());
};
auto CollectExitPhis = [&] {
for (User *U : Root->users()) {
if (U == AccPhi)
continue;
if (!IsValidExitPhi(U))
return false;
ExitPhis.insert(cast<PHINode>(U));
}
return true;
};
if (!CollectExitPhis())
return;
// The final reduction and the insert of the initial value would be on
// the loop-carried chain of an enclosing loop if the reduction result
// feeds the initial value of the accumulator through it: keep the
// horizontal reduction, whose chain is the scalar accumulator only.
auto IsFedByExitPhi = [&](Value *InitV) {
auto *InitPhi = dyn_cast<PHINode>(InitV);
if (InitV == Root || !InitPhi)
return false;
// Too many incoming values to scan: keep the horizontal reduction.
if (InitPhi->getNumIncomingValues() > MaxPHINumOperands)
return true;
return ExitPhis.contains(InitPhi) ||
any_of(make_isa_range<PHINode>(InitPhi->incoming_values()),
[&](PHINode *P) { return ExitPhis.contains(P); });
};
if (any_of(AccPhi->incoming_values(), IsFedByExitPhi))
return;
// Calls clobber the vector registers: the vector accumulator has to be
// kept live across them. Intrinsics cheaper than a call are not calls.
for (BasicBlock *BB : L->blocks())
NumCalls += count_if(*BB, [&](const Instruction &I) {
auto *CB = dyn_cast<CallBase>(&I);
if (!CB || CB->doesNotReturn())
return false;
auto *II = dyn_cast<IntrinsicInst>(CB);
if (!II)
return true;
if (II->isAssumeLikeIntrinsic())
return false;
IntrinsicCostAttributes ICA(II->getIntrinsicID(), *II);
return TTI.getIntrinsicInstrCost(ICA, R.getCostKind()) >=
TTI.getCallInstrCost(nullptr, II->getType(), ICA.getArgTypes(),
R.getCostKind());
});
for (SmallVector<Value *> &Candidates : ReducedVals)
for (Value *&RdxVal : Candidates)
if (RdxVal == AccPhi)
RdxVal = IdC;
ReducedValsToOps.try_emplace(IdC, ReducedValsToOps.lookup(AccPhi));
ReducedValsToOps.erase(AccPhi);
Phi = AccPhi;
Identity = IdC;
}
/// Collects the reduced values that were not vectorized; they are folded
/// into the reduction on top of the vectorized part. The accumulator
/// identity constant is never folded. The operation list of a value has
/// one entry per occurrence of the value in the reduction, all reducing
/// the same value, so the vectorized occurrences are accounted for by
/// count only.
void collectLeftovers(
const DenseMap<Value *, unsigned> &VectorizedVals,
ArrayRef<SmallVector<Value *>> ReducedVals,
const SmallDenseMap<Value *, SmallVector<Instruction *>, 16>
&ReducedValsToOps,
SmallVectorImpl<std::pair<Instruction *, Value *>> &LeftoverReductions)
const {
SmallPtrSet<Value *, 8> Visited;
for (ArrayRef<Value *> Candidates : ReducedVals)
for (Value *RdxVal : make_filter_range(Candidates, [&](Value *RdxVal) {
return RdxVal != Identity && Visited.insert(RdxVal).second;
})) {
unsigned NumOps = VectorizedVals.lookup(RdxVal);
for (Instruction *RedOp :
ArrayRef(ReducedValsToOps.at(RdxVal)).drop_back(NumOps))
LeftoverReductions.emplace_back(RedOp, RdxVal);
}
}
/// \returns the cost of the code the vector accumulator emits outside the
/// loop: the final reductions in the exit blocks and the insert of the
/// initial value, scaled by the enclosing loop nest.
InstructionCost getExitCost(FastMathFlags FMF, const Loop *L) const {
assert(
(RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind) ||
Instruction::isBinaryOp(RecurrenceDescriptor::getOpcode(RdxKind))) &&
"Expected arithmetic or min/max reduction operation");
FixedVectorType *VecTy = R.getReductionType();
TTI::TargetCostKind CostKind = R.getCostKind();
InstructionCost RdxCost =
RecurrenceDescriptor::isMinMaxRecurrenceKind(RdxKind)
? TTI.getMinMaxReductionCost(
getMinMaxReductionIntrinsicOp(RdxKind), VecTy, FMF,
CostKind)
: TTI.getArithmeticReductionCost(
RecurrenceDescriptor::getOpcode(RdxKind), VecTy, FMF,
CostKind);
// One exit block is executed per loop execution: charge the final
// reductions of the largest one. Code size counts all of them.
unsigned MaxExitRdx = 0;
SmallDenseMap<BasicBlock *, unsigned> ExitBlockPhis;
for (PHINode *ExitPhi : ExitPhis)
MaxExitRdx =
std::max(MaxExitRdx, ++ExitBlockPhis[ExitPhi->getParent()]);
InstructionCost Cost =
RdxCost *
(CostKind == TTI::TCK_CodeSize ? ExitPhis.size() : MaxExitRdx) +
TTI.getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind,
/*Index=*/0);
return Cost * R.getLoopNestScale(L->getParentLoop());
}
/// Checks that the slice can be emitted as the vector accumulator: it is
/// the only one (all other reduced values are the identity), not scaled,
/// not reduced in-tree, of the root type and fully vectorized. An integer
/// accumulator stays in a callee-saved register across the calls of the
/// loop, the vector one would be spilled around every call on the
/// loop-carried chain.
bool isCandidate(const Value *VectorizedTree, bool HasVectorizedSlices,
bool IsSupportedHorRdxIdentityOp,
ArrayRef<SmallVector<Value *>> ReducedVals,
ArrayRef<Value *> Candidates, unsigned CandidatesIdx,
unsigned Pos, unsigned ReduxWidth, bool OptReusedScalars,
bool SameScaleFactor) const {
if (!Phi || VectorizedTree || HasVectorizedSlices || Pos != 0 ||
(OptReusedScalars && SameScaleFactor) || R.isReducedBitcastRoot() ||
R.isReducedCmpBitcastRoot() ||
(NumCalls != 0 && !Root->getType()->isFloatingPointTy()))
return false;
if (!all_of(ArrayRef(Candidates).drop_front(ReduxWidth),
[&](Value *RdxVal) { return RdxVal == Identity; }))
return false;
if (!all_of(enumerate(ReducedVals), [&](const auto &P) {
auto [Idx, RV] = P;
return Idx == CandidatesIdx ||
(RV.size() == 1 && RV.front() == Identity);
}))
return false;
if (R.getReductionType()->getElementType() != Root->getType())
return false;
return IsSupportedHorRdxIdentityOp ||
all_of(Candidates.slice(Pos, ReduxWidth),
[&](Value *RdxVal) { return R.isVectorized(RdxVal); });
}
/// \returns the cost delta of the vector accumulator form vs the
/// horizontal reduction on every iteration. Both remove the same scalar
/// operations, so only the vector code is compared: the lane-wise
/// operation is executed on every iteration of the loop, the horizontal
/// reduction as often as the tree cost model assumes (once, if the
/// reduced values are loop-invariant), the final reduction and the
/// initial insert once, outside the loop.
InstructionCost getCostDelta(InstructionCost HorVecCost,
InstructionCost RdxOpCost, FastMathFlags FMF,
VectorType *VectorTy, Type *ScalarTy) const {
// The lane-wise accumulation replaces the reduction operation in the
// vector code.
InstructionCost AccVecCost =
HorVecCost - RdxOpCost + getAccumulationCost(FMF, VectorTy, ScalarTy);
Loop *L = LI.getLoopFor(Phi->getParent());
InstructionCost Delta = AccVecCost * R.getLoopNestScale(L) -
HorVecCost * R.getScaleToLoopIterations(
R.getRootNode(), nullptr, Root) +
getExitCost(FMF, L);
LLVM_DEBUG(dbgs() << "SLP: Loop accumulator cost delta " << Delta
<< "\n");
return Delta;
}
/// Emits the reduction as a vector accumulator if the accumulator form
/// applies; otherwise folds the accumulator phi back on top of the
/// reduction like a leftover and returns null.
Value *tryEmit(
IRBuilderBase &Builder, FastMathFlags RdxFMF, bool Vectorized,
const Value *VectorizedTree,
SmallVectorImpl<std::pair<Instruction *, Value *>> &LeftoverReductions,
ArrayRef<ReductionVectorPart> VectorValuesAndScales,
const SmallPtrSetImpl<Value *> &RequiredExtract,
const SmallDenseMap<Value *, SmallVector<Instruction *>, 16>
&ReducedValsToOps,
const ReductionOpsListType &ReductionOps,
function_ref<Value *(Value *, IRBuilderBase &, Type *)> EmitReduction) {
if (!shouldEmit(Vectorized, VectorizedTree, LeftoverReductions,
VectorValuesAndScales, RequiredExtract)) {
// The accumulator phi is folded back on top of the reduction like a
// leftover: the identity replaced it in the reduced values and does
// not change the result.
if (Phi)
LeftoverReductions.emplace_back(ReducedValsToOps.at(Identity).front(),
Phi);
return nullptr;
}
return emit(Builder, RdxFMF, VectorValuesAndScales.front().Vec,
ReductionOps, EmitReduction);
}
private:
/// Checks that the accumulator form applies: the slice was vectorized as
/// the accumulator and no other vectorized value or leftover reduced
/// values remain to fold; the slice is of the root type, not scaled, not
/// reduced in-tree and without extracts.
bool
shouldEmit(bool Vectorized, const Value *VectorizedTree,
ArrayRef<std::pair<Instruction *, Value *>> LeftoverReductions,
ArrayRef<ReductionVectorPart> VectorValuesAndScales,
const SmallPtrSetImpl<Value *> &RequiredExtract) const {
if (!Vectorized || VectorizedTree || !LeftoverReductions.empty() ||
VectorValuesAndScales.size() != 1 || !RequiredExtract.empty())
return false;
const ReductionVectorPart &P = VectorValuesAndScales.front();
return P.Scale == 1 && !P.ReducedInTree &&
cast<VectorType>(P.Vec->getType())->getElementType() ==
Root->getType();
}
/// Emits the reduction as a vector accumulator: the vectorized reduced
/// values are accumulated lane-wise into a vector phi in the loop and
/// reduced once in the exit blocks.
Value *emit(
IRBuilderBase &Builder, FastMathFlags RdxFMF, Value *Vec,
const ReductionOpsListType &ReductionOps,
function_ref<Value *(Value *, IRBuilderBase &, Type *)> EmitReduction) {
auto *VecTy = cast<FixedVectorType>(Vec->getType());
// The initial values are placed in lane 0 of the identity splat at the
// end of their incoming blocks; multiple edges from a block carry the
// same value and share the insert.
Constant *IdVec =
ConstantVector::getSplat(VecTy->getElementCount(), Identity);
auto *VAcc = PHINode::Create(VecTy, Phi->getNumIncomingValues(),
"slprdx.acc", Phi->getIterator());
SmallDenseMap<BasicBlock *, Value *> InitVecs;
for (auto [IncV, B] : make_filter_range(
zip(Phi->incoming_values(), Phi->blocks()),
[&](const auto &P) { return std::get<0>(P) != Root; })) {
Value *&InitVec = InitVecs[B];
if (!InitVec) {
IRBuilder<> EB(B->getTerminator());
InitVec = EB.CreateInsertElement(IdVec, IncV, EB.getInt32(0),
"slprdx.init");
}
VAcc->addIncoming(InitVec, B);
}
Builder.SetInsertPoint(Root);
Value *VAdd = createOpWithFlags(Builder, RdxKind, VAcc, Vec, "slprdx.acc",
ReductionOps, /*KeepDisjoint=*/true);
// The accumulation is on the loop-carried chain: do not let it be
// contracted into an FMA with the reduced multiplication, which would
// lengthen the chain.
if (auto *FPOp = dyn_cast<FPMathOperator>(VAdd)) {
FastMathFlags FMF = FPOp->getFastMathFlags();
FMF.setAllowContract(false);
cast<Instruction>(VAdd)->copyFastMathFlags(FMF);
}
// The accumulator is the backedge value.
for (auto [IncV, B] : zip(Phi->incoming_values(), Phi->blocks()))
if (IncV == Root)
VAcc->addIncoming(VAdd, B);
LLVM_DEBUG(dbgs() << "SLP: Emitting loop accumulator reduction for "
"reduction with root "
<< *Root << "\n");
// The root is tracked by a weak handle: break its uses on the phi side.
Value *Poison = PoisonValue::get(Root->getType());
for (PHINode *ExitPhi : ExitPhis) {
// The accumulator replaces the root on the edges carrying it and is
// reduced once in the exit block. The values bypassing the loop never
// went through the reduction operations and must stay exact: the
// scalar phi keeps them and they are selected past the reduction
// without its fast-math flags.
unsigned NumIncoming = ExitPhi->getNumIncomingValues();
auto *VExit = PHINode::Create(VecTy, NumIncoming, "slprdx.exit",
ExitPhi->getIterator());
PHINode *FromLoop = nullptr;
if (any_of(ExitPhi->incoming_values(),
[this](Value *V) { return V != Root; }))
FromLoop = PHINode::Create(Builder.getInt1Ty(), NumIncoming,
"slprdx.fromloop", ExitPhi->getIterator());
Value *VecPoison = PoisonValue::get(VecTy);
for (auto [V, B] : zip(ExitPhi->incoming_values(), ExitPhi->blocks())) {
bool IsRoot = V == Root;
VExit->addIncoming(IsRoot ? VAdd : VecPoison, B);
if (FromLoop)
FromLoop->addIncoming(Builder.getInt1(IsRoot), B);
}
ExitPhi->replaceUsesOfWith(Root, Poison);
IRBuilder<> XB(&*ExitPhi->getParent()->getFirstInsertionPt());
XB.SetCurrentDebugLocation(ExitPhi->getDebugLoc());
XB.setFastMathFlags(RdxFMF);
Value *Res = EmitReduction(VExit, XB, ExitPhi->getType());
if (!FromLoop) {
ExitPhi->replaceAllUsesWith(Res);
// The erasure is deferred: the block iteration is still in progress.
R.eraseInstruction(ExitPhi);
continue;
}
Value *Sel = XB.CreateSelectFMFWithUnknownProfile(
FromLoop, Res, ExitPhi, FastMathFlags(), DEBUG_TYPE, "slprdx.sel");
ExitPhi->replaceUsesWithIf(
Sel, [Sel](Use &U) { return U.getUser() != Sel; });
}
Phi->replaceUsesOfWith(Root, Poison);
R.eraseInstruction(Phi);
// Do not let the caller re-analyze the emitted vector operation as a new
// reduction seed.
R.analyzedReductionRoot(cast<Instruction>(VAdd));
return VAdd;
}
};
/// Attempt to vectorize the tree found by matchAssociativeReduction.
/// \p IsSeedRoot allows vectorizing the groups at the minimum vector
/// factor. Set for single-use seed-level roots only.
Value *tryToReduce(BoUpSLP &V, const DataLayout &DL, TargetTransformInfo *TTI,
const TargetLibraryInfo &TLI, AssumptionCache *AC,
DominatorTree &DT, LoopInfo &LI, bool IsSeedRoot = false) {
constexpr unsigned RegMaxNumber = 4;
const unsigned RedValsMaxNumber =
(RK == ReductionOrdering::Ordered &&
ReductionRoot->getType()->isIntOrIntVectorTy())
? 48
: 128;
// If there are a sufficient number of reduction values, reduce
// to a nearby power-of-2. We can safely generate oversized
// vectors and rely on the backend to split them to legal sizes.
if (unsigned NumReducedVals =
accumulate(ReducedVals, 0,
[](unsigned Num, ArrayRef<Value *> Vals) -> unsigned {
if (!isGoodForReduction(Vals))
return Num;
return Num + Vals.size();
});
NumReducedVals < ReductionLimit &&
all_of(
ReducedVals,
[](ArrayRef<Value *> RedV) {
return RedV.size() < 2 || !allConstant(RedV) || !isSplat(RedV);
})) {
// For ordered reductions the leaves are pulled in via the fallback in
// matchAssociativeReduction and may still be valid reduction roots on
// their own; only the root is a dead end. For unordered reductions keep
// marking every reduction op as analyzed.
if (RK == ReductionOrdering::Ordered) {
V.analyzedReductionRoot(cast<Instruction>(ReductionRoot));
} else {
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
V.analyzedReductionRoot(cast<Instruction>(RdxOp));
}
return nullptr;
}
IRBuilder<TargetFolder> Builder(
cast<Instruction>(ReductionRoot)->getIterator(), TargetFolder(DL));
// The scalar parts (scaled reused values, their combines) are emitted right
// before the root; the instruction preceding them is remembered to drop
// them, if the sign-aware reduction is abandoned.
Instruction *InstBeforeRoot =
cast<Instruction>(ReductionRoot)->getPrevNode();
// Track the reduced values in case if they are replaced by extractelement
// because of the vectorization.
DenseMap<Value *, WeakTrackingVH> TrackedVals(ReducedVals.size() *
ReducedVals.front().size());
// The compare instruction of a min/max is the insertion point for new
// instructions and may be replaced with a new compare instruction.
auto &&GetCmpForMinMaxReduction = [](Instruction *RdxRootInst) {
assert(isa<SelectInst>(RdxRootInst) &&
"Expected min/max reduction to have select root instruction");
Value *ScalarCond = cast<SelectInst>(RdxRootInst)->getCondition();
assert(isa<Instruction>(ScalarCond) &&
"Expected min/max reduction to have compare condition");
return cast<Instruction>(ScalarCond);
};
bool AnyBoolLogicOp = any_of(ReductionOps.back(), [](Value *V) {
return isBoolLogicOp(cast<Instruction>(V));
});
// Return new VectorizedTree, based on previous value.
auto GetNewVectorizedTree = [&](Value *VectorizedTree, Value *Res) {
if (VectorizedTree) {
// Update the final value in the reduction.
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionOps.front().front())->getDebugLoc());
if (AnyBoolLogicOp) {
auto It = ReducedValsToOps.find(VectorizedTree);
auto It1 = ReducedValsToOps.find(Res);
if ((It == ReducedValsToOps.end() && It1 == ReducedValsToOps.end()) ||
isGuaranteedNotToBePoison(VectorizedTree, AC) ||
(It != ReducedValsToOps.end() &&
any_of(It->getSecond(), [&](Instruction *I) {
return isBoolLogicOp(I) &&
getRdxOperand(I, 0) == VectorizedTree;
}))) {
;
} else if (isGuaranteedNotToBePoison(Res, AC) ||
(It1 != ReducedValsToOps.end() &&
any_of(It1->getSecond(), [&](Instruction *I) {
return isBoolLogicOp(I) && getRdxOperand(I, 0) == Res;
}))) {
std::swap(VectorizedTree, Res);
} else {
VectorizedTree = Builder.CreateFreeze(VectorizedTree);
}
}
return createOp(Builder, RdxKind, VectorizedTree, Res, "op.rdx",
ReductionOps);
}
// Initialize the final value in the reduction.
return Res;
};
SmallDenseSet<Value *> IgnoreList(ReductionOps.size() *
ReductionOps.front().size());
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp :
make_filter_range(RdxOps, [](Value *RdxOp) { return RdxOp; }))
IgnoreList.insert(RdxOp);
// Dropped with the reduction; do not treat as external uses.
IgnoreList.insert_range(NarrowedChainInsts);
// Intersect the fast-math-flags from all reduction operations.
FastMathFlags RdxFMF;
RdxFMF.set();
for (FPMathOperator *FPMO : make_isa_range<FPMathOperator>(IgnoreList))
RdxFMF &= FPMO->getFastMathFlags();
// Returns true if the original reduced value \p V is subtracted from the
// reduction result.
auto IsNegated = [&](Value *V) { return NegatedReducedVals.contains(V); };
// Scalar values subtracted from the reduction result: combined pairwise
// like the positive ones and subtracted in the final combine, so no
// extra negations are emitted.
SmallVector<std::pair<Instruction *, Value *>> NegExtraReductions;
// For ordered reductions here we need to generate extractelement
// instructions, so clear IgnoreList.
if (RK == ReductionOrdering::Ordered)
IgnoreList.clear();
bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
// A reduction that accumulates a loop phi can be emitted as a vector
// accumulator in the loop plus a single reduction after it.
LoopAccumulator LoopAcc(cast<Instruction>(ReductionRoot), RdxKind, RK, V,
*TTI, LI);
LoopAcc.prepare(RdxFMF, ReducedVals, ReducedValsToOps, NegatedReducedVals,
!NarrowedLeafShifts.empty());
// Set when the slice costed as the vector accumulator was vectorized.
bool LoopAccVectorized = false;
// Need to track reduced vals, they may be changed during vectorization of
// subvectors.
for (ArrayRef<Value *> Candidates : ReducedVals)
for (Value *V : Candidates)
TrackedVals.try_emplace(V, V);
auto At = [](SmallMapVector<Value *, unsigned, 16> &MV,
Value *V) -> unsigned & {
auto *It = MV.find(V);
assert(It != MV.end() && "Unable to find given key.");
return It->second;
};
DenseMap<Value *, unsigned> VectorizedVals(ReducedVals.size());
// List of the values that were reduced in other trees as part of gather
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
// The flattened reduction may be abandoned and retried without the
// fsub/fneg flattening, where the same lists of reduced values may be
// profitable - they are not marked as analyzed.
auto MarkAnalyzedVals = [&](ArrayRef<Value *> VL) {
if (!hasFlattenedNegations())
V.analyzedReductionVals(VL);
};
// Routes the emitted value by the sign of its group: negated parts are
// combined separately and subtracted in the final combine. \p OrigV is
// the original reduced value (null for the combined vector reduction
// result); its reduction operation provides the debug location.
auto AddReducedPart = [&](Value *OrigV, Value *Res, bool Negated) {
if (Negated) {
NegExtraReductions.emplace_back(OrigV
? ReducedValsToOps.at(OrigV).front()
: cast<Instruction>(ReductionRoot),
Res);
return;
}
VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
};
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
SmallVector<SmallVector<Value *>> LocalReducedVals;
// Try merge consecutive reduced values into a single vectorizable group and
// check, if they can be vectorized as copyables.
const bool TwoGroupsOnly = ReducedVals.size() == 2;
const bool TwoGroupsOfSameSmallSize =
TwoGroupsOnly &&
ReducedVals.front().size() == ReducedVals.back().size() &&
ReducedVals.front().size() < ReductionLimit;
for (ArrayRef<Value *> RV : ReducedVals) {
// Groups with different signs of their contribution cannot be merged:
// the mixed group cannot be added/subtracted as a whole. The merge
// candidate is the last group of the same sign.
int MergeIdx = LocalReducedVals.size() - 1;
while (MergeIdx >= 0 && IsNegated(LocalReducedVals[MergeIdx].front()) !=
IsNegated(RV.front()))
--MergeIdx;
const bool SameSign = LocalReducedVals.empty() || MergeIdx >= 0;
// Loads are not very compatible with undefs.
if (isa<UndefValue>(RV.front()) &&
(States.empty() || !States.back() ||
States.back().getOpcode() == Instruction::Load)) {
LocalReducedVals.emplace_back().append(RV.begin(), RV.end());
States.push_back(InstructionsState::invalid());
continue;
}
if (!LocalReducedVals.empty() &&
isa<UndefValue>(LocalReducedVals.back().front()) &&
isa<LoadInst>(RV.front())) {
LocalReducedVals.emplace_back().append(RV.begin(), RV.end());
States.push_back(getSameOpcode(RV, TLI));
continue;
}
// Do some copyables analysis only if more than 2 groups exist or they
// are large enough.
if (SameSign && !TwoGroupsOfSameSmallSize) {
SmallVector<Value *> Ops;
if (!LocalReducedVals.empty())
Ops = LocalReducedVals[MergeIdx];
Ops.append(RV.begin(), RV.end());
InstructionsCompatibilityAnalysis Analysis(DT, DL, *TTI, TLI);
InstructionsState OpS = Analysis.buildInstructionsState(
Ops, V, /*WithProfitabilityCheck=*/true);
if (OpS) {
if (LocalReducedVals.empty()) {
LocalReducedVals.push_back(Ops);
States.push_back(OpS);
continue;
}
LocalReducedVals[MergeIdx].swap(Ops);
States[MergeIdx] = OpS;
continue;
}
// For safety, allow split vectorization only if 2 groups are available
// overall.
if (TwoGroupsOnly) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(Ops);
OpS = InstructionsState(MainOp, AltOp);
// Last chance to try to vectorize alternate node.
SmallVector<Value *> Op1, Op2;
BoUpSLP::OrdersType ReorderIndices;
// canBuildSplitNode() relies on VectorizableTree(), make sure to
// clear here since it may contain leftover state from prior attempts.
V.deleteTree();
if (MainOp && AltOp &&
V.canBuildSplitNode(Ops, OpS, Op1, Op2, ReorderIndices)) {
if (LocalReducedVals.empty()) {
LocalReducedVals.push_back(Ops);
States.push_back(OpS);
continue;
}
LocalReducedVals[MergeIdx].swap(Ops);
States[MergeIdx] = OpS;
continue;
}
}
}
LocalReducedVals.emplace_back().append(RV.begin(), RV.end());
States.push_back(getSameOpcode(RV, TLI));
}
ReducedVals.swap(LocalReducedVals);
FastMathFlags GroupRdxFMF = RdxFMF;
// The minimum vector factor pays off only when it covers the whole
// reduction: every group must be either a pair of distinct values or
// large enough for the regular vector factor.
const bool NoScalarLeftovers =
all_of(ReducedVals, [this](ArrayRef<Value *> Vals) {
return Vals.size() >= ReductionLimit ||
(Vals.size() == 2 && Vals.front() != Vals.back());
});
// Small sign-aware groups pay off only when paired via the vector fsub:
// each sign must have a group of at least 2 values.
auto HasGroupOfSign = [&](bool Negated) {
return any_of(ReducedVals, [&](ArrayRef<Value *> Vals) {
return Vals.size() >= 2 && IsNegated(Vals.front()) == Negated;
});
};
const bool CanPairSigns =
HasGroupOfSign(/*Negated=*/false) && HasGroupOfSign(/*Negated=*/true);
// Same-size groups of the seed-level reduction get the profitability
// ordering (cheap groups go last) when the minimum vector factor covers
// the whole reduction.
if (IsSeedRoot && NegatedReducedVals.empty() && NoScalarLeftovers &&
any_of(ReducedVals,
[](ArrayRef<Value *> Vals) { return Vals.size() == 2; })) {
SmallVector<unsigned> GroupOrder(ReducedVals.size());
std::iota(GroupOrder.begin(), GroupOrder.end(), 0);
stable_sort(GroupOrder, [&](unsigned L, unsigned R) {
ArrayRef<Value *> P1 = ReducedVals[L], P2 = ReducedVals[R];
if (P1.size() != P2.size())
return P1.size() > P2.size();
return !isCheapReductionGroup(P1) && isCheapReductionGroup(P2);
});
SmallVector<SmallVector<Value *>> SortedVals;
SmallVector<InstructionsState> SortedStates;
SortedVals.reserve(ReducedVals.size());
SortedStates.reserve(States.size());
for (unsigned Idx : GroupOrder) {
SortedVals.emplace_back(std::move(ReducedVals[Idx]));
SortedStates.push_back(States[Idx]);
}
ReducedVals = std::move(SortedVals);
States = std::move(SortedStates);
}
for (unsigned I = 0, E = ReducedVals.size(); I < E; ++I) {
ArrayRef<Value *> OrigReducedVals = ReducedVals[I];
InstructionsState S = States[I];
// The main operation of a state with copyable elements may be deleted
// during previous vectorization attempts, the state is not usable then.
if (S && S.areInstructionsWithCopyableElements() &&
V.isDeleted(S.getMainOp()))
S = InstructionsState::invalid();
SmallVector<Value *> Candidates;
Candidates.reserve(2 * OrigReducedVals.size());
SmallVector<Value *> TrackedToOrig;
// Reduced values that were replaced by an extractelement (because they
// were vectorized in another reduction subset or operand cone) and are
// not compatible with this group. They are not vectorized again here, but
// they are still live and consumed by the final reduction, so they must
// be kept externally used to avoid erasing them while vectorizing this
// group, which may use them as operands.
SmallVector<Value *> ExcludedVectorizedReducedVals;
for (Value *ReducedVal : OrigReducedVals) {
Value *RdxVal = TrackedVals.at(ReducedVal);
// Check if the reduction value was not overriden by the extractelement
// instruction because of the vectorization and exclude it, if it is not
// compatible with other values.
// Also check if the instruction was folded to constant/other value.
auto *Inst = dyn_cast<Instruction>(RdxVal);
if (Inst && V.isDeleted(Inst))
continue;
if (Inst && isVectorLikeInstWithConstOps(Inst) &&
(!S || (!S.getMatchingMainOpOrAltOp(Inst) &&
!S.isCopyableElement(Inst)))) {
ExcludedVectorizedReducedVals.push_back(RdxVal);
continue;
}
if (S && !Inst && !isa<UndefValue>(RdxVal) &&
!S.isCopyableElement(RdxVal))
continue;
Candidates.push_back(RdxVal);
TrackedToOrig.push_back(ReducedVal);
}
// The groups are sign-uniform by matching, so the first original
// value's sign applies to all of them.
const bool GroupNegated =
!TrackedToOrig.empty() && IsNegated(TrackedToOrig.front());
bool ShuffledExtracts = false;
// Try to handle shuffled extractelements. Only pure extractelement
// groups can be merged: merged groups are skipped for external uses,
// and other values would be erased while still used by reduction ops.
if (S && S.getOpcode() == Instruction::ExtractElement &&
!S.isAltShuffle() && I + 1 < E &&
all_of(ReducedVals[I + 1],
[&](Value *RV) {
return isa<ExtractElementInst>(TrackedVals.at(RV));
}) &&
GroupNegated == IsNegated(ReducedVals[I + 1].front())) {
SmallVector<Value *> CommonCandidates(Candidates);
for (Value *RV : ReducedVals[I + 1]) {
Value *RdxVal = TrackedVals.at(RV);
// Check if the reduction value was not overriden by the
// extractelement instruction because of the vectorization and
// exclude it, if it is not compatible with other values.
auto *Inst = dyn_cast<ExtractElementInst>(RdxVal);
if (!Inst)
continue;
CommonCandidates.push_back(RdxVal);
TrackedToOrig.push_back(RV);
}
SmallVector<int> Mask;
if (isFixedVectorShuffle(CommonCandidates, Mask, AC,
V.getAnalysisCache())) {
++I;
Candidates.swap(CommonCandidates);
ShuffledExtracts = true;
}
}
assert(all_of(TrackedToOrig,
[&](Value *V) { return IsNegated(V) == GroupNegated; }) &&
"Expected sign-uniform group of reduced values");
// Emit code for constant values.
if (Candidates.size() > 1 && allConstant(Candidates)) {
if (RK == ReductionOrdering::Ordered)
continue;
Value *Res = Candidates.front();
Value *OrigV = TrackedToOrig.front();
++VectorizedVals.try_emplace(OrigV).first->getSecond();
for (const auto [Idx, VC] :
enumerate(ArrayRef(Candidates).drop_front())) {
Res = createOp(Builder, RdxKind, Res, VC, "const.rdx", ReductionOps);
Value *OrigV = TrackedToOrig[Idx + 1];
++VectorizedVals.try_emplace(OrigV).first->getSecond();
if (auto *ResI = dyn_cast<Instruction>(Res))
V.analyzedReductionRoot(ResI);
}
AddReducedPart(TrackedToOrig.front(), Res, GroupNegated);
continue;
}
unsigned NumReducedVals = Candidates.size();
auto UsedByReductionOnly = [&](Value *V) {
if (!V->hasUseList())
return true;
// Bail out if we have too many uses to save compilation time.
if (V->hasNUsesOrMore(UsesLimit))
return false;
return all_of(V->users(),
[&](User *U) { return IgnoreList.contains(U); });
};
// Sign-aware reductions pair small positive/negative groups: allow
// non-splat groups down to 2 elements. Seed-level reductions get the
// same for groups whose values are used by the reduction operations
// only, if the minimum vector factor covers the whole reduction.
const bool MinVFAllowed = IsSeedRoot && NoScalarLeftovers &&
all_of(Candidates, UsedByReductionOnly);
if (NumReducedVals < ReductionLimit &&
(NumReducedVals < 2 ||
(!isSplat(Candidates) && !CanPairSigns && !MinVFAllowed)))
continue;
// Check if we support repeated scalar values processing (optimization of
// original scalar identity operations on matched horizontal reductions).
IsSupportedHorRdxIdentityOp =
RK == ReductionOrdering::Unordered && RdxKind != RecurKind::Mul &&
RdxKind != RecurKind::FMul && RdxKind != RecurKind::FMulAdd &&
(!Opts.slp_revec || !Candidates.front()->getType()->isVectorTy());
// Gather same values.
SmallMapVector<Value *, unsigned, 16> SameValuesCounter;
if (IsSupportedHorRdxIdentityOp)
for (const auto [Idx, V] : enumerate(Candidates)) {
Value *OrigV = TrackedToOrig[Idx];
++SameValuesCounter.try_emplace(OrigV).first->second;
}
// Used to check if the reduced values used same number of times. In this
// case the compiler may produce better code. E.g. if reduced values are
// aabbccdd (8 x values), then the first node of the tree will have a node
// for 4 x abcd + shuffle <4 x abcd>, <0, 0, 1, 1, 2, 2, 3, 3>.
// Plus, the final reduction will be performed on <8 x aabbccdd>.
// Instead compiler may build <4 x abcd> tree immediately, + reduction (4
// x abcd) * 2.
// Currently it only handles add/fadd/xor. and/or/min/max do not require
// this analysis, other operations may require an extra estimation of
// the profitability.
bool SameScaleFactor = false;
bool OptReusedScalars = IsSupportedHorRdxIdentityOp &&
SameValuesCounter.size() != Candidates.size();
BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
for (Value *RdxVal : ExcludedVectorizedReducedVals)
ExternallyUsedValues.insert(RdxVal);
if (OptReusedScalars) {
SameScaleFactor =
(RdxKind == RecurKind::Add || RdxKind == RecurKind::FAdd ||
RdxKind == RecurKind::Xor) &&
all_of(drop_begin(SameValuesCounter),
[&SameValuesCounter](const std::pair<Value *, unsigned> &P) {
return P.second == SameValuesCounter.front().second;
});
Candidates.resize(SameValuesCounter.size());
TrackedToOrig.resize(SameValuesCounter.size());
transform(SameValuesCounter, Candidates.begin(),
[&](const auto &P) { return TrackedVals.at(P.first); });
transform(SameValuesCounter, TrackedToOrig.begin(),
[](const auto &P) { return P.first; });
NumReducedVals = Candidates.size();
// Have a reduction of the same element.
if (NumReducedVals == 1) {
Value *OrigV = TrackedToOrig.front();
unsigned Cnt = At(SameValuesCounter, OrigV);
// A negated repeated value joins an already vectorized part of the
// same width as a splat operand of the vector fsub (a per-lane fnma
// with a multiply part) instead of the scalar scale and subtract.
if (GroupNegated &&
any_of(VectorValuesAndScales, [&](const ReductionVectorPart &P) {
return P.Vec && !P.ReducedInTree &&
getNumElements(P.Vec->getType()) == Cnt;
})) {
Value *Splat = Builder.CreateVectorSplat(Cnt, Candidates.front());
VectorValuesAndScales.push_back({Splat, /*Scale=*/1,
/*IsSigned=*/true,
/*ReducedInTree=*/false,
GroupNegated});
} else {
Value *RedVal =
emitScaleForReusedOps(Candidates.front(), Builder, Cnt);
AddReducedPart(OrigV, RedVal, GroupNegated);
}
VectorizedVals.try_emplace(OrigV, Cnt);
ExternallyUsedValues.insert(OrigV);
continue;
}
}
GroupRdxFMF = RdxFMF;
// No need to check for associativity, if 2 reduced values.
if (NumReducedVals == 2 &&
(RdxKind == RecurKind::FAdd || RdxKind == RecurKind::FMul))
GroupRdxFMF.setAllowReassoc(true);
unsigned MaxVecRegSize = V.getMaxVecRegSize();
unsigned EltSize = V.getVectorElementSize(Candidates[0]);
const unsigned MaxElts = std::clamp<unsigned>(
llvm::bit_floor(MaxVecRegSize / EltSize), RedValsMaxNumber,
RegMaxNumber * RedValsMaxNumber);
unsigned ReduxWidth = NumReducedVals;
auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) {
unsigned NumRegs, NumAvailRegs;
Type *ScalarTy = Candidates.front()->getType();
ReduxWidth = getFloorFullVectorNumberOfElements(
TTI, ScalarTy, ReduxWidth, Opts.slp_revec);
VectorType *Tp = cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth));
NumRegs = V.getRegUsageForType(Tp, ScalarTy);
NumAvailRegs =
TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
while (NumRegs > NumAvailRegs) {
assert(ReduxWidth > 0 && "ReduxWidth is unexpectedly 0.");
ReduxWidth = bit_floor(ReduxWidth - 1);
VectorType *Tp =
cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth));
NumRegs = V.getRegUsageForType(Tp, ScalarTy);
NumAvailRegs =
TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
}
if (NumRegs > NumAvailRegs / 2)
ReduxWidth = bit_floor(ReduxWidth);
return ReduxWidth;
};
if (!isAllowedNonPowerOf2VF(ReduxWidth,
Opts.slp_vectorize_non_power_of_2))
ReduxWidth = GetVectorFactor(ReduxWidth);
ReduxWidth = std::min(ReduxWidth, MaxElts);
unsigned Start = 0;
unsigned Pos = Start;
// Restarts vectorization attempt with lower vector factor.
unsigned PrevReduxWidth = ReduxWidth;
bool CheckForReusedReductionOpsLocal = false;
auto AdjustReducedVals = [&](bool IgnoreVL = false) {
bool IsAnyRedOpGathered =
!IgnoreVL &&
(RK == ReductionOrdering::Ordered || V.isAnyGathered(IgnoreList));
if (!CheckForReusedReductionOpsLocal && PrevReduxWidth == ReduxWidth) {
// Check if any of the reduction ops are gathered. If so, worth
// trying again with less number of reduction ops.
CheckForReusedReductionOpsLocal |= IsAnyRedOpGathered;
}
++Pos;
if (Pos < NumReducedVals - ReduxWidth + 1)
return IsAnyRedOpGathered;
Pos = Start;
--ReduxWidth;
if (ReduxWidth > 1)
ReduxWidth = GetVectorFactor(ReduxWidth);
return IsAnyRedOpGathered;
};
bool AnyVectorized = false;
SmallDenseSet<std::pair<unsigned, unsigned>, 8> IgnoredCandidates;
// The overlapping windows rebuild almost the same trees. Like the
// scheduling region, the size of the next tree, estimated by the last
// rejected one, is limited by the budget, reduced by the sizes of the
// rejected trees, but not below the minimal size. The non-power-of-2
// widths get only the minimal size for the rejected trees. The budget
// limits only the series of the rejected trees, the vectorized tree
// starts a new series.
unsigned ReductionBudget = Opts.slp_reduction_budget;
unsigned LastRejectedTreeSize = 1;
unsigned NonPowerOf2RejectedSize = 0;
auto RejectTree = [&]() {
LastRejectedTreeSize = V.getTreeSize();
const unsigned Size = ReduxWidth * LastRejectedTreeSize;
ReductionBudget -= std::min(ReductionBudget, Size);
ReductionBudget = std::max(ReductionBudget, MinReductionTreeSize);
if (!has_single_bit(ReduxWidth))
NonPowerOf2RejectedSize += Size;
};
// Same small-group allowance for the vector width, if it covers the
// whole group.
const unsigned MinReduxWidth =
CanPairSigns || (MinVFAllowed && NumReducedVals == 2)
? 2
: ReductionLimit;
while (Pos < NumReducedVals - ReduxWidth + 1 &&
ReduxWidth >= MinReduxWidth) {
// Dependency in tree of the reduction ops - drop this attempt, try
// later.
if (CheckForReusedReductionOpsLocal && PrevReduxWidth != ReduxWidth &&
Start == 0) {
CheckForReusedReductionOps = true;
break;
}
PrevReduxWidth = ReduxWidth;
ArrayRef<Value *> VL(std::next(Candidates.begin(), Pos), ReduxWidth);
// Been analyzed already or out of budget - skip.
if (ReduxWidth * LastRejectedTreeSize > ReductionBudget ||
IgnoredCandidates.contains(std::make_pair(Pos, ReduxWidth)) ||
(!has_single_bit(ReduxWidth) &&
(NonPowerOf2RejectedSize > MinReductionTreeSize ||
IgnoredCandidates.contains(
std::make_pair(Pos, bit_floor(ReduxWidth))) ||
IgnoredCandidates.contains(
std::make_pair(Pos + (ReduxWidth - bit_floor(ReduxWidth)),
bit_floor(ReduxWidth))))) ||
V.areAnalyzedReductionVals(VL)) {
(void)AdjustReducedVals(/*IgnoreVL=*/true);
continue;
}
// Early exit if any of the reduction values were deleted during
// previous vectorization attempts.
if (any_of(make_isa_range<Instruction>(VL),
[&V](Instruction *RedValI) { return V.isDeleted(RedValI); }))
break;
if (RK == ReductionOrdering::Ordered) {
V.buildTree(VL);
} else {
V.buildTree(VL, IgnoreList);
V.setNarrowedChainInsts(NarrowedChainInsts);
}
if (V.isTreeTinyAndNotFullyVectorizable(RK ==
ReductionOrdering::Unordered)) {
RejectTree();
constexpr unsigned CandidatesLimit = 64;
if (!AdjustReducedVals(RK == ReductionOrdering::Ordered &&
Candidates.size() >= CandidatesLimit))
MarkAnalyzedVals(VL);
continue;
}
V.reorderTopToBottom();
// No need to reorder the root node at all for reassociative reduction.
V.reorderBottomToTop(/*IgnoreReorder=*/GroupRdxFMF.allowReassoc() ||
VL.front()->getType()->isIntOrIntVectorTy() ||
ReductionLimit > 2 ||
RK == ReductionOrdering::Ordered);
// Keep extracted other reduction values, if they are used in the
// vectorization trees.
BoUpSLP::ExtraValueToDebugLocsMap LocalExternallyUsedValues(
ExternallyUsedValues);
// The reduction root is used as the insertion point for new
// instructions, so set it as externally used to prevent it from being
// deleted.
LocalExternallyUsedValues.insert(ReductionRoot);
for (unsigned Cnt = 0, Sz = ReducedVals.size(); Cnt < Sz; ++Cnt) {
if (Cnt == I || (ShuffledExtracts && Cnt == I - 1))
continue;
for (Instruction *Inst :
make_isa_range<Instruction>(ReducedVals[Cnt]))
LocalExternallyUsedValues.insert(TrackedVals[Inst]);
}
if (!IsSupportedHorRdxIdentityOp) {
// Number of uses of the candidates in the vector of values.
assert(SameValuesCounter.empty() &&
"Reused values counter map is not empty");
for (unsigned Cnt = 0; Cnt < NumReducedVals; ++Cnt) {
if (Cnt >= Pos && Cnt < Pos + ReduxWidth)
continue;
Value *OrigV = TrackedToOrig[Cnt];
++SameValuesCounter.try_emplace(OrigV).first->second;
}
}
V.transformNodes();
V.computeMinimumValueSizes();
InstructionCost TreeCost = V.calculateTreeCostAndTrimNonProfitable(
VL, cast<Instruction>(ReductionRoot));
// A negated slice is subtracted in the final combine, it cannot be
// accumulated lane-wise.
const bool LoopAccCandidate =
!GroupNegated &&
LoopAcc.isCandidate(VectorizedTree, !VectorValuesAndScales.empty(),
IsSupportedHorRdxIdentityOp, ReducedVals,
Candidates, I, Pos, ReduxWidth,
OptReusedScalars, SameScaleFactor);
SmallPtrSet<Value *, 4> VLScalars(llvm::from_range, VL);
// Gather externally used values.
SmallPtrSet<Value *, 4> Visited;
for (unsigned Cnt = 0; Cnt < NumReducedVals; ++Cnt) {
if (Cnt >= Pos && Cnt < Pos + ReduxWidth)
continue;
// The candidate may be replaced by the previous vectorization
// attempts, use the currently tracked value.
Value *RdxVal = TrackedVals.at(TrackedToOrig[Cnt]);
if (!Visited.insert(RdxVal).second)
continue;
// Scalars not reduced by the top node may be used by the reduction
// epilogue, even if not vectorized as part of the tree.
if (!VLScalars.contains(RdxVal)) {
LocalExternallyUsedValues.insert(RdxVal);
continue;
}
Value *OrigV = TrackedToOrig[Cnt];
unsigned NumOps =
VectorizedVals.lookup(OrigV) + At(SameValuesCounter, OrigV);
if (NumOps != ReducedValsToOps.at(OrigV).size())
LocalExternallyUsedValues.insert(RdxVal);
}
// Do not need the list of reused scalars in regular mode anymore.
if (!IsSupportedHorRdxIdentityOp)
SameValuesCounter.clear();
for (Value *RdxVal : VL)
if (RequiredExtract.contains(RdxVal))
LocalExternallyUsedValues.insert(RdxVal);
V.buildExternalUses(LocalExternallyUsedValues);
// Estimate cost.
InstructionCost ReductionCost, HorVecCost = 0, RdxOpCost = 0;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
V.isReducedCmpBitcastRoot())
ReductionCost = 0;
else
ReductionCost = getReductionCost(
TTI, VL, TrackedToOrig, Pos, SameValuesCounter, IsCmpSelMinMax,
GroupRdxFMF, V, DT, DL, TLI, HorVecCost, RdxOpCost);
// The vector accumulator form is used if it is not more expensive than
// the horizontal reduction on every iteration.
InstructionCost LoopAccDelta = 0;
if (LoopAccCandidate && ReductionCost.isValid())
LoopAccDelta =
LoopAcc.getCostDelta(HorVecCost, RdxOpCost, RdxFMF,
V.getReductionType(), VL.front()->getType());
// On a tie the accumulator wins: it also replaces the loop-carried
// scalar dependency through the horizontal reduction by a lane-wise
// one. An invalid reduction cost rejects the tree below anyway.
const bool UseLoopAccForm =
LoopAccCandidate && ReductionCost.isValid() && LoopAccDelta <= 0;
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
Instruction *InsertPt = RdxRootInst;
if (IsCmpSelMinMax)
InsertPt = GetCmpForMinMaxReduction(RdxRootInst);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, InsertPt);
if (UseLoopAccForm)
Cost += LoopAccDelta;
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
<< " for reduction\n");
if (!Cost.isValid())
break;
if (Cost >= -V.getCostThreshold()) {
V.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "HorSLPNotBeneficial",
ReducedValsToOps.at(VL[0]).front())
<< "Vectorizing horizontal reduction is possible "
<< "but not beneficial with cost " << ore::NV("Cost", Cost)
<< " and threshold "
<< ore::NV("Threshold", -V.getCostThreshold());
});
RejectTree();
if (!AdjustReducedVals()) {
MarkAnalyzedVals(VL);
unsigned Offset = Pos == Start ? Pos : Pos - 1;
if (ReduxWidth > ReductionLimit && V.isTreeNotExtendable()) {
// Add subvectors of VL to the list of the analyzed values.
for (unsigned VF = getFloorFullVectorNumberOfElements(
*TTI, VL.front()->getType(), ReduxWidth - 1,
Opts.slp_revec);
VF >= ReductionLimit;
VF = getFloorFullVectorNumberOfElements(
*TTI, VL.front()->getType(), VF - 1, Opts.slp_revec)) {
if (has_single_bit(VF) &&
V.getCanonicalGraphSize() != V.getTreeSize())
continue;
for (unsigned Idx : seq<unsigned>(ReduxWidth - VF))
IgnoredCandidates.insert(std::make_pair(Offset + Idx, VF));
}
}
}
continue;
}
LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n");
ReductionBudget = Opts.slp_reduction_budget;
LastRejectedTreeSize = 1;
NonPowerOf2RejectedSize = 0;
V.getORE()->emit([&]() {
return OptimizationRemark(SV_NAME, "VectorizedHorizontalReduction",
ReducedValsToOps.at(VL[0]).front())
<< "Vectorized horizontal reduction with cost "
<< ore::NV("Cost", Cost) << " and with tree size "
<< ore::NV("TreeSize", V.getTreeSize());
});
Builder.setFastMathFlags(RdxFMF);
// Match the bitmask form of the first vector part before the
// vectorization, which drops the operands of the vectorized leaves.
BoolBitmask BitmaskMatch =
VectorValuesAndScales.empty()
? isBoolBitmaskRdx(RdxKind, NarrowedLeafShifts, DL)
: BoolBitmask::None;
// Vectorize a tree.
Value *VectorizedRoot = V.vectorizeTree(
LocalExternallyUsedValues, InsertPt, VectorValuesAndScales);
if (RK == ReductionOrdering::Ordered) {
// No need to generate reduction here, emit extractelements instead in
// the tree vectorizer.
assert(VectorizedRoot && "Expected vectorized tree");
// Count vectorized reduced values to exclude them from final
// reduction.
for (Value *RdxVal : VL)
++VectorizedVals.try_emplace(RdxVal).first->getSecond();
Pos += ReduxWidth;
Start = Pos;
ReduxWidth = NumReducedVals - Pos;
if (ReduxWidth > 1)
ReduxWidth = GetVectorFactor(NumReducedVals - Pos);
AnyVectorized = true;
VectorizedTree = ReductionRoot;
continue;
}
Builder.SetInsertPoint(InsertPt);
// To prevent poison from leaking across what used to be sequential,
// safe, scalar boolean logic operations, the reduction operand must be
// frozen.
if (AnyBoolLogicOp && !isGuaranteedNotToBePoison(VectorizedRoot, AC))
VectorizedRoot = Builder.CreateFreeze(VectorizedRoot);
// Emit code to correctly handle reused reduced values, if required.
if (OptReusedScalars && !SameScaleFactor) {
// The reuse counters are stored in the deduplicated candidates
// order, but the emitted reduction vector lane order is defined by
// the root node, which may be reordered, split or have copyable
// elements. Place each counter at the lane the matching candidate is
// vectorized to, so the counter is applied to the correct lane.
ArrayRef<Value *> RootVL = V.getRootNodeScalars();
ArrayRef<Value *> CandSlice(Candidates.begin() + Pos, ReduxWidth);
SmallVector<Value *> RootTrackedToOrig(RootVL.size());
for (auto [Idx, Val] : enumerate(CandSlice))
RootTrackedToOrig[V.findRootLaneForValue(Val)] =
TrackedToOrig[Pos + Idx];
VectorizedRoot = emitReusedOps(VectorizedRoot, Builder, V,
SameValuesCounter, RootTrackedToOrig);
}
// Mask in the narrow type, then widen and shift per lane. Map back
// from extractelements to the original reduced values for the shift
// amounts. The absorbed narrow and-masks and the bits truncated by
// the absorbed narrow shls are masked off before the widening.
if (!NarrowedLeafShifts.empty()) {
Type *WideTy = ReductionRoot->getType();
Type *NarrowTy = VL.front()->getType();
unsigned VF = getNumElements(VectorizedRoot->getType());
// The root may be demoted and resized to the reduction bitwidth,
// which is computed for the wide reduction operations.
if (VectorizedRoot->getType()->getScalarType() != NarrowTy) {
VectorizedRoot = Builder.CreateIntCast(
VectorizedRoot, getWidenedType(NarrowTy, VF),
V.isSignedMinBitwidthRootNode());
++NumVectorInstructions;
}
SmallVector<Constant *> ShiftConsts(VF, ConstantInt::get(WideTy, 0));
SmallVector<Constant *> MaskConsts(
VF, Constant::getAllOnesValue(NarrowTy));
bool AnyShift = false;
bool AnyMask = false;
for (auto [Idx, Val] : enumerate(VL)) {
const NarrowedLeafInfo &L =
NarrowedLeafShifts.at(TrackedToOrig[Pos + Idx]);
unsigned Lane = V.findRootLaneForValue(Val);
ShiftConsts[Lane] = ConstantInt::get(WideTy, L.Shift);
AnyShift |= L.Shift != 0;
if (!L.Mask.isAllOnes()) {
MaskConsts[Lane] = ConstantInt::get(NarrowTy, L.Mask);
AnyMask = true;
}
}
if (Value *Bitmask = tryEmitBoolBitmaskRdx(
Builder, V, *TTI, BitmaskMatch, VectorizedRoot, VL,
TrackedToOrig, Pos, MaskConsts, GroupRdxFMF)) {
VectorizedRoot = Bitmask;
} else {
if (AnyMask) {
VectorizedRoot = Builder.CreateAnd(
VectorizedRoot, ConstantVector::get(MaskConsts));
++NumVectorInstructions;
}
VectorizedRoot =
Builder.CreateZExt(VectorizedRoot, getWidenedType(WideTy, VF));
++NumVectorInstructions;
if (AnyShift) {
VectorizedRoot = Builder.CreateShl(
VectorizedRoot, ConstantVector::get(ShiftConsts));
++NumVectorInstructions;
}
}
}
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
// A group of negated reduced values (from an fsub/fneg in the chain)
// is subtracted in the final combine.
VectorValuesAndScales.push_back(
{VectorizedRoot,
OptReusedScalars && SameScaleFactor
? SameValuesCounter.front().second
: 1,
RedScalarTy != ScalarTy->getScalarType()
? NarrowedLeafShifts.empty() && V.isSignedMinBitwidthRootNode()
: true,
V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot() ||
!VectorizedRoot->getType()->isVectorTy(),
GroupNegated});
LoopAccVectorized = UseLoopAccForm;
// Count vectorized reduced values to exclude them from final reduction.
for (const auto [Idx, RdxVal] : enumerate(VL)) {
Value *OrigV = TrackedToOrig[Pos + Idx];
if (IsSupportedHorRdxIdentityOp) {
VectorizedVals.try_emplace(OrigV, At(SameValuesCounter, OrigV));
continue;
}
++VectorizedVals.try_emplace(OrigV).first->getSecond();
if (!V.isVectorized(RdxVal))
RequiredExtract.insert(RdxVal);
}
Pos += ReduxWidth;
Start = Pos;
ReduxWidth = NumReducedVals - Pos;
if (ReduxWidth > 1)
ReduxWidth = GetVectorFactor(NumReducedVals - Pos);
AnyVectorized = true;
// All remaining reduced values are the identity: nothing left to
// vectorize for the accumulator form.
if (UseLoopAccForm)
break;
}
if (OptReusedScalars && !AnyVectorized) {
for (const std::pair<Value *, unsigned> &P : SameValuesCounter) {
Value *RdxVal = TrackedVals.at(P.first);
Value *RedVal = emitScaleForReusedOps(RdxVal, Builder, P.second);
AddReducedPart(P.first, RedVal, GroupNegated);
VectorizedVals.try_emplace(P.first, P.second);
}
continue;
}
}
// Early exit for the ordered reductions.
// No need to do anything else here, so we can just exit.
if (RK == ReductionOrdering::Ordered) {
assert(NegatedReducedVals.empty() &&
"Unexpected negated reduced values in the ordered reduction");
// Only the root is a dead end, the leaves may still be the roots of the
// other reductions.
if (!VectorizedTree)
V.analyzedReductionRoot(cast<Instruction>(ReductionRoot));
return VectorizedTree;
}
// A booleanized logical and/or reduction has an i1 root and wide integer
// reduced values; it is performed in the wide type and bit 0 of the
// result is the final value.
Type *BoolReduxWideTy =
getBoolReduxWideTy(RdxKind, ReductionRoot->getType(),
ReducedVals.front().front()->getType());
SmallVector<std::pair<Instruction *, Value *>> LeftoverReductions;
LoopAcc.collectLeftovers(VectorizedVals, ReducedVals, ReducedValsToOps,
LeftoverReductions);
// The accumulator form requires a single vectorized slice and no
// leftover reduced values; the latter would have to be folded into the
// scalar accumulator on every iteration. Otherwise the accumulator phi is
// folded back on top of the reduction like a leftover.
Value *AccV = LoopAcc.tryEmit(
Builder, RdxFMF, LoopAccVectorized, VectorizedTree, LeftoverReductions,
VectorValuesAndScales, RequiredExtract, ReducedValsToOps, ReductionOps,
[this, TTI = TTI, &V](Value *Vec, IRBuilderBase &B, Type *Ty) {
return emitReduction(Vec, B, TTI,
V.getCastContextHint(V.getRootNode()), Ty);
});
if (AccV)
VectorizedTree = AccV;
else if (!VectorValuesAndScales.empty()) {
Builder.setFastMathFlags(GroupRdxFMF);
Value *Res = nullptr;
bool ResNegated = false;
const ReductionVectorPart &P = VectorValuesAndScales.front();
// A single fully vectorized booleanized reduction without scalar
// remainders can be emitted as trunc+bitcast+cmp, if cheaper.
if (BoolReduxWideTy && !VectorizedTree &&
VectorValuesAndScales.size() == 1 && !P.ReducedInTree && !P.Negated &&
isa<FixedVectorType>(P.Vec->getType()) &&
none_of(ReducedVals, [&](ArrayRef<Value *> Candidates) {
return any_of(Candidates, [&](Value *RdxVal) {
return VectorizedVals.lookup(RdxVal) <
ReducedValsToOps.at(RdxVal).size();
});
}))
Res = tryEmitBoolReduxBitcastCmp(Builder, *TTI, RdxKind, P.Vec,
ReductionRoot, GroupRdxFMF,
V.getCostKind());
if (!Res)
std::tie(Res, ResNegated) = emitReduction(
Builder, *TTI, V.getCastContextHint(V.getRootNode()),
BoolReduxWideTy ? BoolReduxWideTy : ReductionRoot->getType());
Builder.setFastMathFlags(RdxFMF);
// The reduction result of the all-negated parts is subtracted in the
// final combine.
AddReducedPart(/*OrigV=*/nullptr, Res, ResNegated);
} else if (hasFlattenedNegations()) {
// The flattened reduction pays off only with a vector part. Without one
// the scalar chain is merely reshuffled (which may also break the scalar
// fma formation), while the original code may still be vectorizable as
// an operand tree - drop the scalar parts emitted so far and give up.
auto *Root = cast<Instruction>(ReductionRoot);
for (Instruction *I = Root->getPrevNode(); I && I != InstBeforeRoot;) {
Instruction *Prev = I->getPrevNode();
if (I->use_empty() && !V.isAnalyzedReductionRoot(I))
I->eraseFromParent();
I = Prev;
}
// The fsub/fneg leaves themselves may still form vectorizable groups
// and the chains behind them are matched as separate reductions - leave
// the reduction operations unmarked for the retry.
return nullptr;
}
if (!VectorizedTree && NegExtraReductions.empty()) {
if (!CheckForReusedReductionOps) {
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
V.analyzedReductionRoot(cast<Instruction>(RdxOp));
}
return nullptr;
}
// Reorder operands of bool logical op in the natural order to avoid
// possible problem with poison propagation. If not possible to reorder
// (both operands are originally RHS), emit an extra freeze instruction
// for the LHS operand.
// I.e., if we have original code like this:
// RedOp1 = select i1 ?, i1 LHS, i1 false
// RedOp2 = select i1 RHS, i1 ?, i1 false
// Then, we swap LHS/RHS to create a new op that matches the poison
// semantics of the original code.
// If we have original code like this and both values could be poison:
// RedOp1 = select i1 ?, i1 LHS, i1 false
// RedOp2 = select i1 ?, i1 RHS, i1 false
// Then, we must freeze LHS in the new op.
auto FixBoolLogicalOps =
[&, VectorizedTree](Value *&LHS, Value *&RHS, Instruction *RedOp1,
Instruction *RedOp2, bool InitStep) {
if (!AnyBoolLogicOp)
return;
if (isBoolLogicOp(RedOp1) && ((!InitStep && LHS == VectorizedTree) ||
getRdxOperand(RedOp1, 0) == LHS ||
isGuaranteedNotToBePoison(LHS, AC)))
return;
bool NeedFreeze = LHS != VectorizedTree;
if (isBoolLogicOp(RedOp2) && ((!InitStep && RHS == VectorizedTree) ||
getRdxOperand(RedOp2, 0) == RHS ||
isGuaranteedNotToBePoison(RHS, AC))) {
// If RedOp2 was used as a second operand - do not swap.
if ((InitStep || RHS != VectorizedTree) &&
getRdxOperand(RedOp2, 0) == RHS &&
((isBoolLogicOp(RedOp1) &&
getRdxOperand(RedOp1, 1) == RedOp2) ||
any_of(ReductionOps, [&](ArrayRef<Value *> Ops) {
return any_of(make_isa_range<Instruction>(Ops),
[&](Instruction *OpI) {
return isBoolLogicOp(OpI) &&
getRdxOperand(OpI, 1) == RedOp2;
});
}))) {
NeedFreeze = false;
} else {
std::swap(LHS, RHS);
return;
}
}
if (NeedFreeze)
LHS = Builder.CreateFreeze(LHS);
};
// Leftover narrow values need the same mask/zext/shift as the vectorized
// ones.
auto WidenNarrowedVal = [&](Value *OrigV, Value *&V) {
auto It = NarrowedLeafShifts.find(OrigV);
if (It == NarrowedLeafShifts.end())
return;
if (!It->second.Mask.isAllOnes())
V = Builder.CreateAnd(V, It->second.Mask);
V = Builder.CreateZExt(V, ReductionRoot->getType());
if (It->second.Shift != 0)
V = Builder.CreateShl(V, It->second.Shift);
};
// Finish the reduction.
// Need to add extra arguments and not vectorized possible reduction values.
// Try to avoid dependencies between the scalar remainders after reductions.
auto FinalGen = [&](ArrayRef<std::pair<Instruction *, Value *>> InstVals,
bool InitStep) {
unsigned Sz = InstVals.size();
SmallVector<std::pair<Instruction *, Value *>> ExtraReds(Sz / 2 + Sz % 2);
for (unsigned I = 0, E = (Sz / 2) * 2; I < E; I += 2) {
Instruction *RedOp = InstVals[I + 1].first;
Builder.SetCurrentDebugLocation(RedOp->getDebugLoc());
Value *RdxVal1 = InstVals[I].second;
Value *StableRdxVal1 = RdxVal1;
auto It1 = TrackedVals.find(RdxVal1);
if (It1 != TrackedVals.end())
StableRdxVal1 = It1->second;
Value *RdxVal2 = InstVals[I + 1].second;
Value *StableRdxVal2 = RdxVal2;
auto It2 = TrackedVals.find(RdxVal2);
if (It2 != TrackedVals.end())
StableRdxVal2 = It2->second;
WidenNarrowedVal(RdxVal1, StableRdxVal1);
WidenNarrowedVal(RdxVal2, StableRdxVal2);
// To prevent poison from leaking across what used to be sequential,
// safe, scalar boolean logic operations, the reduction operand must be
// frozen.
FixBoolLogicalOps(StableRdxVal1, StableRdxVal2, InstVals[I].first,
RedOp, InitStep);
Value *ExtraRed = createOp(Builder, RdxKind, StableRdxVal1,
StableRdxVal2, "op.rdx", ReductionOps);
ExtraReds[I / 2] = std::make_pair(InstVals[I].first, ExtraRed);
}
if (Sz % 2 == 1)
ExtraReds[Sz / 2] = InstVals.back();
return ExtraReds;
};
SmallVector<std::pair<Instruction *, Value *>> ExtraReductions;
if (VectorizedTree)
ExtraReductions.emplace_back(cast<Instruction>(ReductionRoot),
VectorizedTree);
for (const auto &[RedOp, RdxVal] : LeftoverReductions)
(IsNegated(RdxVal) ? NegExtraReductions : ExtraReductions)
.emplace_back(RedOp, RdxVal);
// Iterate through all not-vectorized reduction values/extra arguments,
// combining them pairwise.
auto CombineReds =
[&](SmallVectorImpl<std::pair<Instruction *, Value *>> &Reds) {
bool InitStep = true;
while (Reds.size() > 1) {
SmallVector<std::pair<Instruction *, Value *>> NewReds =
FinalGen(Reds, InitStep);
Reds.swap(NewReds);
InitStep = false;
}
Value *RdxVal = Reds.front().second;
auto It = TrackedVals.find(RdxVal);
if (It != TrackedVals.end())
RdxVal = It->second;
return RdxVal;
};
VectorizedTree = nullptr;
if (!ExtraReductions.empty())
VectorizedTree = CombineReds(ExtraReductions);
// Combine the subtracted values pairwise as well and subtract the combined
// value from the positive part of the reduction (or just negate it, if
// there is no positive part).
if (!NegExtraReductions.empty()) {
Value *NegTree = CombineReds(NegExtraReductions);
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionOps.front().front())->getDebugLoc());
Value *Op = VectorizedTree
? Builder.CreateFSub(VectorizedTree, NegTree, "op.rdx")
: Builder.CreateFNeg(NegTree, "op.rdx");
propagateIRFlags(Op, ReductionOps.front(), nullptr,
/*IncludeWrapFlags=*/false);
VectorizedTree = Op;
}
if (BoolReduxWideTy &&
VectorizedTree->getType() != ReductionRoot->getType())
VectorizedTree =
Builder.CreateTrunc(VectorizedTree, ReductionRoot->getType());
// The accumulator emission already replaced all uses of the root.
if (!AccV)
ReductionRoot->replaceAllUsesWith(VectorizedTree);
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
// correct, replace internal uses with undef, and mark for eventual
// deletion.
#ifndef NDEBUG
SmallPtrSet<Value *, 4> IgnoreSet;
for (ArrayRef<Value *> RdxOps : ReductionOps)
IgnoreSet.insert_range(RdxOps);
#endif
for (ArrayRef<Value *> RdxOps : ReductionOps) {
for (Value *Ignore :
make_filter_range(RdxOps, [](Value *Ignore) { return Ignore; })) {
#ifndef NDEBUG
for (auto *U : Ignore->users()) {
assert(IgnoreSet.count(U) &&
"All users must be either in the reduction ops list.");
}
#endif
if (!Ignore->use_empty()) {
Value *P = PoisonValue::get(Ignore->getType());
Ignore->replaceAllUsesWith(P);
}
}
V.removeInstructionsAndOperands(RdxOps, VectorValuesAndScales);
}
return VectorizedTree;
}
/// Attempt to vectorize an ordered (linearized) reduction chain.
/// Reduced values from matchOrderedReduction() are in accumulation order.
/// Vectorized subsets are immediately reduced via ordered reduction
/// intrinsics; non-vectorized values are folded linearly.
Value *tryToReduceOrdered(BoUpSLP &V, const DataLayout &DL,
TargetTransformInfo *TTI,
const TargetLibraryInfo &TLI, AssumptionCache *AC,
DominatorTree &DT) {
constexpr unsigned RegMaxNumber = 4;
constexpr unsigned RedValsMaxNumber = 128;
assert(RK == ReductionOrdering::Ordered && "Expected ordered reduction");
assert(ReducedVals.size() == 1 &&
"Expected single group from matchOrderedReduction");
assert(NegatedReducedVals.empty() &&
"Unexpected negated reduced values in the ordered reduction");
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
IRBuilder<TargetFolder> Builder(RdxRootInst->getIterator(),
TargetFolder(DL));
// Track the reduced values in case if they are replaced by extractelement
// because of the vectorization.
SmallDenseMap<Value *, WeakTrackingVH> TrackedVals(
ReducedVals.size() * ReducedVals.front().size());
// Need to track reduced vals, they may be changed during vectorization of
// subvectors.
for (ArrayRef<Value *> Candidates : ReducedVals)
for (Value *V : Candidates)
TrackedVals.try_emplace(V, V);
SmallVector<Value *> Candidates(ReducedVals.back());
// Intersect the fast-math-flags from all reduction operations.
FastMathFlags RdxFMF;
RdxFMF.set();
for (Value *RdxVal : Candidates)
for (FPMathOperator *FPMO :
make_isa_range<FPMathOperator>(ReducedValsToOps.at(RdxVal)))
RdxFMF &= FPMO->getFastMathFlags();
// This is an ordered reduction, selected because the reduction operations
// are not associative (e.g. an fadd without nsz). The accumulation order
// must be preserved, so the generated llvm.vector.reduce.fadd has to stay
// ordered. Drop 'reassoc' from the flags used to emit and cost the
// reduction.
RdxFMF.setAllowReassoc(/*B=*/false);
unsigned MaxVecRegSize = V.getMaxVecRegSize();
unsigned EltSize = V.getVectorElementSize(Candidates[0]);
const unsigned MaxElts =
std::clamp<unsigned>(llvm::bit_floor(MaxVecRegSize / EltSize),
RedValsMaxNumber, RegMaxNumber * RedValsMaxNumber);
unsigned ReduxWidth = 0;
auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) {
Type *ScalarTy = Candidates.front()->getType();
ReduxWidth = getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth,
Opts.slp_revec);
Type *Tp = getWidenedType(ScalarTy, ReduxWidth);
unsigned NumParts = V.getNumberOfParts(Tp, ScalarTy);
unsigned NumRegs =
TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
while (NumParts > NumRegs) {
assert(ReduxWidth > 0 && "ReduxWidth is unexpectedly 0.");
ReduxWidth = bit_floor(ReduxWidth - 1);
Type *Tp = getWidenedType(ScalarTy, ReduxWidth);
NumParts = V.getNumberOfParts(Tp, ScalarTy);
NumRegs =
TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
}
if (NumParts > NumRegs / 2)
ReduxWidth = bit_floor(ReduxWidth);
return ReduxWidth;
};
auto ShrinkReduxWidth = [&]() {
--ReduxWidth;
if (ReduxWidth > 1)
ReduxWidth = GetVectorFactor(ReduxWidth);
};
// Try to build, cost-check, and vectorize a window [Start, Start+Width).
unsigned SuccessStart = 0, SuccessWidth = 0;
Value *SuccessRoot = nullptr;
SmallMapVector<Value *, unsigned, 16> EmptySameValuesCounter;
auto TryWindow = [&](unsigned Start, unsigned Width) -> bool {
ArrayRef<Value *> VL = ArrayRef(Candidates).slice(Start, Width);
if (V.areAnalyzedReductionVals(VL))
return false;
if (any_of(make_isa_range<Instruction>(VL),
[&V](Instruction *RedValI) { return V.isDeleted(RedValI); }))
return false;
// Ignore the whole reduction operation chain, not only the ops tied to
// the current window.
SmallDenseSet<Value *> IgnoreList;
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
if (RdxOp)
IgnoreList.insert(RdxOp);
V.buildTree(VL, IgnoreList);
if (V.isTreeTinyAndNotFullyVectorizable(/*ForReduction=*/true)) {
V.analyzedReductionVals(VL);
return false;
}
// Do not reorder whole tree to preserve the original order of the ordered
// reduction.
V.reorderBottomToTop();
BoUpSLP::ExtraValueToDebugLocsMap LocalExternallyUsedValues;
LocalExternallyUsedValues.insert(ReductionRoot);
for (unsigned Cnt : seq<unsigned>(Candidates.size())) {
if (Cnt >= Start && Cnt < Start + Width)
continue;
if (isa<Instruction>(Candidates[Cnt]))
LocalExternallyUsedValues.insert(Candidates[Cnt]);
}
V.transformNodes();
V.computeMinimumValueSizes();
InstructionCost TreeCost =
V.calculateTreeCostAndTrimNonProfitable(VL, RdxRootInst);
V.buildExternalUses(LocalExternallyUsedValues);
InstructionCost VectorCost, RdxOpCost;
InstructionCost ReductionCost = getReductionCost(
TTI, VL, /*TrackedToOrig=*/{}, /*Pos=*/0, EmptySameValuesCounter,
/*IsCmpSelMinMax=*/false, RdxFMF, V, DT, DL, TLI, VectorCost,
RdxOpCost);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, RdxRootInst);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
<< " for ordered reduction\n");
if (Cost > -V.getCostThreshold() ||
(Cost == -V.getCostThreshold() && V.getTreeSize() > 1)) {
if (Cost.isValid())
V.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "HorSLPNotBeneficial",
ReducedValsToOps.at(VL[0]).front())
<< "Vectorizing ordered reduction is possible "
<< "but not beneficial with cost " << ore::NV("Cost", Cost)
<< " and threshold "
<< ore::NV("Threshold", -V.getCostThreshold());
});
V.analyzedReductionVals(VL);
return false;
}
LLVM_DEBUG(dbgs() << "SLP: Vectorizing ordered reduction at cost:" << Cost
<< ". (HorRdx)\n");
V.getORE()->emit([&]() {
return OptimizationRemark(SV_NAME, "VectorizedHorizontalReduction",
ReducedValsToOps.at(VL[0]).front())
<< "Vectorized ordered reduction with cost "
<< ore::NV("Cost", Cost) << " and with tree size "
<< ore::NV("TreeSize", V.getTreeSize());
});
Builder.setFastMathFlags(RdxFMF);
SuccessRoot = V.vectorizeTree(LocalExternallyUsedValues, RdxRootInst);
assert(SuccessRoot && "Expected vectorized tree");
SuccessStart = Start;
SuccessWidth = Width;
return true;
};
// Interleaved window search - for each width W (shrinking),
// try front-anchored [0, W) then back-anchored [N-W, N).
unsigned N = Candidates.size();
ReduxWidth = N;
if (!Opts.slp_vectorize_non_power_of_2 || !has_single_bit(ReduxWidth + 1))
ReduxWidth = GetVectorFactor(ReduxWidth);
ReduxWidth = std::min(ReduxWidth, MaxElts);
while (ReduxWidth >= ReductionLimit) {
if (TryWindow(0, ReduxWidth))
break;
unsigned BackStart = N - ReduxWidth;
if (BackStart != 0 && TryWindow(BackStart, ReduxWidth))
break;
ShrinkReduxWidth();
}
if (!SuccessRoot) {
for (ReductionOpsType &RdxOps : ReductionOps)
for (Value *RdxOp : RdxOps)
V.analyzedReductionRoot(cast<Instruction>(RdxOp));
return nullptr;
}
// Fold leading scalars [0, SuccessStart) into an accumulator.
Type *DestTy = ReductionRoot->getType();
WeakTrackingVH VectorizedTree = nullptr;
if (!all_of(RoundedLinks, equal_to(nullptr))) {
// The round-trip links cannot be expressed by the reduction intrinsic:
// fold the values into a scalar chain in the original order, extracting
// the vectorized ones, and re-emit the round-trips between the folded
// operations. The emitted chain matches the same pattern again, so its
// operations are marked as analyzed to prevent repeated vectorization.
for (unsigned Idx : seq<unsigned>(N)) {
Value *RdxVal = Candidates[Idx];
Builder.SetCurrentDebugLocation(
ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
Value *NextVal = TrackedVals.at(RdxVal);
if (Idx >= SuccessStart && Idx < SuccessStart + SuccessWidth)
NextVal =
Builder.CreateExtractElement(SuccessRoot, Idx - SuccessStart);
if (!VectorizedTree) {
VectorizedTree = NextVal;
continue;
}
auto *Op = cast<Instruction>(createOp(Builder, RdxKind, VectorizedTree,
NextVal, "op.rdx", ReductionOps));
V.analyzedReductionRoot(Op);
VectorizedTree = Op;
Instruction *Narrow = RoundedLinks[N - 1 - Idx];
if (!Narrow)
continue;
Instruction *NewNarrow = Narrow->clone();
NewNarrow->setOperand(0, VectorizedTree);
Builder.Insert(NewNarrow);
Instruction *NewWide =
cast<Instruction>(*Narrow->user_begin())->clone();
NewWide->setOperand(0, NewNarrow);
VectorizedTree = Builder.Insert(NewWide);
}
} else {
for (Value *RdxVal : ArrayRef(Candidates).take_front(SuccessStart)) {
Builder.SetCurrentDebugLocation(
ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
if (!VectorizedTree)
VectorizedTree = TrackedVals.at(RdxVal);
else
VectorizedTree =
createOp(Builder, RdxKind, VectorizedTree, TrackedVals.at(RdxVal),
"op.rdx", ReductionOps);
}
// Emit ordered reduction for the vectorized window. The reduction only
// applies to floating point types.
assert(DestTy->isFPOrFPVectorTy() &&
SuccessRoot->getType()->isFPOrFPVectorTy() &&
"Expected floating point types for ordered reduction");
Builder.SetCurrentDebugLocation(
cast<Instruction>(ReductionRoot)->getDebugLoc());
VectorizedTree =
createSingleOp(Builder, *TTI, V.getCastContextHint(V.getRootNode()),
SuccessRoot, /*Scale=*/1,
/*IsSigned=*/false, DestTy,
/*ReducedInTree=*/false, VectorizedTree);
// Fold trailing scalars [SuccessStart+SuccessWidth, N).
for (Value *RdxVal :
ArrayRef(Candidates).drop_front(SuccessStart + SuccessWidth)) {
Builder.SetCurrentDebugLocation(
ReducedValsToOps.at(RdxVal).front()->getDebugLoc());
VectorizedTree =
createOp(Builder, RdxKind, VectorizedTree, TrackedVals.at(RdxVal),
"op.rdx", ReductionOps);
}
}
ReductionRoot->replaceAllUsesWith(VectorizedTree);
#ifndef NDEBUG
SmallPtrSet<Value *, 4> IgnoreSet;
for (ArrayRef<Value *> RdxOps : ReductionOps)
IgnoreSet.insert_range(RdxOps);
#endif
for (ArrayRef<Value *> RdxOps : ReductionOps) {
for (Value *Ignore :
make_filter_range(RdxOps, [](Value *Ignore) { return Ignore; })) {
#ifndef NDEBUG
for (auto *U : Ignore->users()) {
assert(IgnoreSet.count(U) &&
"All users must be either in the reduction ops list.");
}
#endif
if (!Ignore->use_empty()) {
Value *P = PoisonValue::get(Ignore->getType());
Ignore->replaceAllUsesWith(P);
}
}
V.removeInstructionsAndOperands(RdxOps, {});
}
return VectorizedTree;
}
private:
/// Creates the reduction from the given \p Vec vector value with the given
/// scale \p Scale and signedness \p IsSigned.
Value *createSingleOp(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
TTI::CastContextHint Ctx, Value *Vec, unsigned Scale,
bool IsSigned, Type *DestTy, bool ReducedInTree,
Value *Start = nullptr) {
Value *Rdx;
if (ReducedInTree) {
Rdx = Vec;
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy);
VecTy && Opts.slp_revec) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
assert(getNumElements(Vec->getType()) % DestTyNumElements == 0 &&
"Vec element count must be a multiple of DestTy element count");
// e.g. Consider vector reduce add.
// Initial reduction is
// clang-format off
// %add0 = add <4 x i32> zeroinitializer, %A
// %add1 = add <4 x i32> %add0, %B
//
// where,
// %A = <a, b, c, d>
// %B = <e, f, g, h>
// %add1 = A + B = <a + e, b + f, c + g, d + h>
//
// After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
// the reduction can be expressed as:
// %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
// %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
// %add0 = add <4 x i32> zeroinitializer, %A
// %add1 = add <4 x i32> %add0, %B
// clang-format on
Rdx = Start;
for (auto I : seq<unsigned>(VF)) {
auto Position = I * DestTyNumElements;
Value *SubVec =
createExtractVector(Builder, Vec, DestTyNumElements, Position);
if (!Rdx)
Rdx = SubVec;
else
Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, Ctx, DestTy, Start);
}
if (Rdx->getType() != DestTy)
Rdx = Builder.CreateIntCast(Rdx, DestTy, IsSigned);
// Improved analysis for add/fadd/xor reductions with same scale
// factor for all operands of reductions. We can emit scalar ops for
// them instead.
if (Scale > 1)
Rdx = emitScaleForReusedOps(Rdx, Builder, Scale);
return Rdx;
}
/// The lane order of the zero tests by their bit position, shared between
/// the cost evaluation and the emission of the boolean bitmask reduction.
SmallVector<int> getBoolBitmaskPermMask(ArrayRef<Value *> VL,
ArrayRef<Value *> TrackedToOrig,
unsigned Pos,
const BoUpSLP &R) const {
// Narrowed leaves exist for unordered reductions only, where the tracked
// original values are always available.
assert(TrackedToOrig.size() >= Pos + VL.size() &&
"tracked original values must be available for all reduced values");
SmallVector<int> PermMask(VL.size(), PoisonMaskElem);
for (auto [Idx, Val] : enumerate(VL))
PermMask[NarrowedLeafShifts.at(TrackedToOrig[Pos + Idx]).Shift] =
R.findRootLaneForValue(Val);
return PermMask;
}
/// Emits the boolean bitmask form of the narrowed-leaf or-reduction (a
/// bitcast of the per-lane zero tests to an integer) if it covers the whole
/// reduction in a single vector part and is cheaper than the shift and
/// reduction sequence. Returns nullptr otherwise.
Value *tryEmitBoolBitmaskRdx(IRBuilderBase &Builder, const BoUpSLP &R,
const TargetTransformInfo &TTI,
BoolBitmask Match, Value *VectorizedRoot,
ArrayRef<Value *> VL,
ArrayRef<Value *> TrackedToOrig, unsigned Pos,
ArrayRef<Constant *> MaskConsts,
FastMathFlags FMF) const {
Type *WideTy = ReductionRoot->getType();
auto *NarrowVecTy = dyn_cast<VectorType>(VectorizedRoot->getType());
if (!NarrowVecTy)
return nullptr;
Type *NarrowTy = NarrowVecTy->getScalarType();
unsigned VF = getNumElements(NarrowVecTy);
// Applies only when the whole reduction is a single vector part.
if (NarrowedLeafShifts.size() != VL.size() || VF != VL.size() ||
!VectorValuesAndScales.empty() ||
!R.getRootNode().ReuseShuffleIndices.empty())
return nullptr;
if (Match == BoolBitmask::None)
return nullptr;
SmallVector<int> PermMask =
getBoolBitmaskPermMask(VL, TrackedToOrig, Pos, R);
// Emit the bitmask form only if it is cheaper than the shift and
// reduction sequence.
const TTI::TargetCostKind CostKind = R.getCostKind();
auto *WideVecTy = cast<VectorType>(getWidenedType(WideTy, VF));
InstructionCost GenericCost =
TTI.getExtendedReductionCost(Instruction::Or, /*IsUnsigned=*/true,
WideTy, NarrowVecTy, FMF, CostKind) +
getNarrowedLeafOpsCost(TTI, NarrowedLeafShifts, NarrowVecTy, WideVecTy,
cast<Instruction>(ReductionRoot), CostKind);
if (getBoolBitmaskCost(TTI, Match == BoolBitmask::NeedMask, NarrowTy,
WideTy, VF, PermMask, ReductionRoot,
CostKind) >= GenericCost)
return nullptr;
Value *Vec = VectorizedRoot;
if (Match == BoolBitmask::NeedMask) {
Vec = Builder.CreateAnd(Vec, ConstantVector::get(MaskConsts));
++NumVectorInstructions;
}
// Reorder the lanes by their bit position and pack the zero tests into an
// integer.
if (!ShuffleVectorInst::isIdentityMask(PermMask, VF)) {
Vec = Builder.CreateShuffleVector(Vec, PermMask);
++NumVectorInstructions;
}
if (!NarrowTy->isIntegerTy(1)) {
Vec = Builder.CreateICmpNE(Vec, Constant::getNullValue(Vec->getType()));
++NumVectorInstructions;
}
Vec = Builder.CreateBitCast(Vec, Builder.getIntNTy(VF));
++NumVectorInstructions;
if (Vec->getType() != WideTy) {
Vec = Builder.CreateIntCast(Vec, WideTy, /*isSigned=*/false);
++NumVectorInstructions;
}
return Vec;
}
/// Calculate the cost of a reduction.
/// \p VectorCost receives the cost of the emitted vector code alone and
/// \p RdxOpCost the cost of the reduction operation in it.
InstructionCost getReductionCost(
TargetTransformInfo *TTI, ArrayRef<Value *> ReducedVals,
ArrayRef<Value *> TrackedToOrig, unsigned Pos,
const SmallMapVector<Value *, unsigned, 16> SameValuesCounter,
bool IsCmpSelMinMax, FastMathFlags FMF, const BoUpSLP &R,
DominatorTree &DT, const DataLayout &DL, const TargetLibraryInfo &TLI,
InstructionCost &VectorCost, InstructionCost &RdxOpCost) {
const TTI::TargetCostKind CostKind = R.getCostKind();
Type *ScalarTy = ReducedVals.front()->getType();
unsigned ReduxWidth = ReducedVals.size();
FixedVectorType *VectorTy = R.getReductionType();
InstructionCost ScalarCost;
VectorCost = 0;
RdxOpCost = 0;
// If all of the reduced values are constant, the vector cost is 0, since
// the reduction value can be calculated at the compile time.
bool AllConsts = allConstant(ReducedVals);
// A reduced value may have been replaced by an extractelement while
// vectorizing a previous subvector of the reduction. Such tracked values
// are not present in ReducedValsToOps, so look through them to the
// reduction operation that uses the replaced value.
auto GetReductionOp = [&](Value *RdxVal) -> Instruction * {
auto It = ReducedValsToOps.find(RdxVal);
if (It != ReducedValsToOps.end())
return It->second.front();
for (User *U : RdxVal->users())
// For flattened fadd reductions the reduced value may be consumed by
// an fsub/fneg chain link instead of an fadd. No reassoc/nsz gating
// is needed: the result is only a cost context.
if (getRdxKind(U) == RdxKind ||
(RdxKind == RecurKind::FAdd &&
(match(U, m_FSub(m_Value(), m_Value())) ||
match(U, m_FNeg(m_Value())))))
return cast<Instruction>(U);
return cast<Instruction>(RdxVal);
};
// The root is accumulated through the phi of its loop.
const bool IsLoopCarriedRoot =
RdxKind == RecurKind::FAdd &&
any_of(ReductionRoot->users(), [&](User *U) {
auto *Phi = dyn_cast<PHINode>(U);
return Phi &&
DT.dominates(Phi->getParent(),
cast<Instruction>(ReductionRoot)->getParent());
});
auto EvaluateScalarCost =
[&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
InstructionCost Cost = 0;
// Scalar cost is repeated for N-1 elements.
int Cnt = ReducedVals.size();
for (Instruction *RdxVal : make_isa_range<Instruction>(ReducedVals)) {
Instruction *RdxOp = GetReductionOp(RdxVal);
if (Cnt == 1) {
unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
// A loop-carried chain removes one operation per reduced value;
// its last operation, the reduction root, folds the accumulator
// phi back and is credited by the accumulation cost instead.
unsigned NumRemoved = SameValueCount ? SameValueCount - 1 : 0;
if (RdxKind == RecurKind::FAdd && IsLoopCarriedRoot &&
RdxOp != ReductionRoot)
NumRemoved = SameValueCount ? SameValueCount : 1;
Cost += NumRemoved * GenCostFn(RdxOp);
break;
}
--Cnt;
if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn(RdxOp);
continue;
}
InstructionCost ScalarCost = 0;
for (User *U : RdxVal->users()) {
auto *RdxOp = cast<Instruction>(U);
// The loop-carried root is not removed: the horizontal form
// folds the accumulator phi back through it, the accumulator
// form replaces it with the lane-wise operation, costed by
// the accumulation cost.
if (RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot &&
IsLoopCarriedRoot)
continue;
// The reduction root is used outside of the reduction any
// number of times, its fmul operand is still fused into it.
if ((RdxKind == RecurKind::FAdd && RdxOp == ReductionRoot) ||
hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
InstructionsState RdxOpS = RdxKind == RecurKind::FAdd
? getSameOpcode(RdxOp, TLI)
: InstructionsState::invalid();
// The reduced value itself must be a fusable fmul, whose cost
// is excluded.
if (RdxOpS && RdxOpS.isAddSubOrFNegLikeOp() &&
isFusableFMulOperand(RdxVal, RdxOp, *TTI)) {
InstructionCost FMACost =
canConvertToFMA(RdxOp, RdxOpS, DT, DL, *TTI, TLI, R);
if (FMACost.isValid()) {
LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
// Also, exclude scalar fmul cost.
InstructionCost FMulCost =
TTI->getInstructionCost(RdxVal, CostKind);
LLVM_DEBUG(dbgs()
<< "Minus FMul cost: " << FMulCost << "\n");
FMACost -= FMulCost;
ScalarCost += FMACost;
continue;
}
}
ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
continue;
}
ScalarCost = InstructionCost::getInvalid();
break;
}
if (ScalarCost.isValid())
Cost += ScalarCost;
else
Cost += GenCostFn(RdxOp);
}
return Cost;
};
// Require reduction cost if:
// 1. This type is not a full register type and no other vectors with the
// same type in the storage (first vector with small type).
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
switch (RdxKind) {
case RecurKind::Add:
case RecurKind::Mul:
case RecurKind::Or:
case RecurKind::And:
case RecurKind::Xor:
case RecurKind::FAdd:
case RecurKind::FMul: {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
Type *BoolReduxWideTy =
getBoolReduxWideTy(RdxKind, ReductionRoot->getType(), ScalarTy);
if (!AllConsts) {
if (!NarrowedLeafShifts.empty()) {
Type *WideTy = ReductionRoot->getType();
auto *NarrowVecTy =
cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth));
auto *WideVecTy =
cast<VectorType>(getWidenedType(WideTy, ReduxWidth));
if (DoesRequireReductionOp) {
VectorCost = TTI->getExtendedReductionCost(
RdxOpcode, /*IsUnsigned=*/true, WideTy, NarrowVecTy, FMF,
CostKind);
} else {
VectorCost =
TTI->getCastInstrCost(Instruction::ZExt, WideVecTy, NarrowVecTy,
TTI::CastContextHint::None, CostKind) +
TTI->getArithmeticInstrCost(RdxOpcode, WideVecTy, CostKind);
}
for (Instruction *I : NarrowedChainInsts)
VectorCost -= TTI->getInstructionCost(I, CostKind);
VectorCost += getNarrowedLeafOpsCost(
*TTI, NarrowedLeafShifts, NarrowVecTy, WideVecTy,
cast<Instruction>(ReductionRoot), CostKind);
// The boolean bitmask emission may be cheaper. Restricted to the
// first vector part, same as in the emission.
BoolBitmask Match =
VectorValuesAndScales.empty() &&
NarrowedLeafShifts.size() == ReduxWidth &&
R.getRootNode().getVectorFactor() == ReduxWidth &&
R.getRootNode().ReuseShuffleIndices.empty()
? isBoolBitmaskRdx(RdxKind, NarrowedLeafShifts, DL)
: BoolBitmask::None;
if (Match != BoolBitmask::None) {
SmallVector<int> PermMask =
getBoolBitmaskPermMask(ReducedVals, TrackedToOrig, Pos, R);
InstructionCost BitmaskCost = getBoolBitmaskCost(
*TTI, Match == BoolBitmask::NeedMask, ScalarTy, WideTy,
ReduxWidth, PermMask, ReductionRoot, CostKind);
for (Instruction *I : NarrowedChainInsts)
BitmaskCost -= TTI->getInstructionCost(I, CostKind);
VectorCost = std::min(VectorCost, BitmaskCost);
}
} else if (DoesRequireReductionOp) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(Opts.slp_revec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
VectorCost +=
getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy,
CostKind, {}, I * ScalarTyNumElements, VecTy);
}
// We get one less arithmetic instruction compared to number of
// reduced values. We are also passing CtxI so that the backend can
// synthesize FMF from the respective reduction instruction.
for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
VectorCost += TTI->getArithmeticInstrCost(
RdxOpcode, VecTy, CostKind,
TTI::getOperandInfo(ReducedVals[I]),
TTI::getOperandInfo(ReducedVals[I + 1]), {},
GetReductionOp(ReducedVals[I]));
}
} else {
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
std::make_pair(RedTy, true));
if (BoolReduxWideTy) {
auto *WideVecTy = cast<FixedVectorType>(
getWidenedType(BoolReduxWideTy, ReduxWidth));
VectorCost = getBoolReduxWideRdxCost(
*TTI, RdxKind, WideVecTy, ReductionRoot, FMF, CostKind);
// The trunc+bitcast+cmp form applies only when the whole
// reduction is a single vector part, without scalar remainders.
if (this->ReducedVals.size() == 1 &&
ReducedVals.size() == this->ReducedVals.front().size())
VectorCost = std::min(
VectorCost, getBoolReduxBitcastCmpCost(
*TTI, RdxKind, WideVecTy, ReductionRoot,
NarrowedChainInsts, CostKind));
} else if (RType == RedTy) {
if (VectorTy->getElementType()->isIntegerTy(1) &&
((ScalarTy->isIntegerTy(1) &&
(RdxKind == RecurKind::And || RdxKind == RecurKind::Or)) ||
(RdxKind == RecurKind::Add && !ScalarTy->isIntegerTy(1)))) {
VectorCost =
getI1ReductionCost(RdxKind, *TTI, VectorTy, ScalarTy,
R.getCastContextHint(R.getRootNode()),
CostKind)
.first;
} else {
VectorCost = TTI->getArithmeticReductionCost(
RdxOpcode, VectorTy, FMF, CostKind);
}
} else {
VectorCost = TTI->getExtendedReductionCost(
RdxOpcode, !IsSigned, RedTy,
cast<VectorType>(getWidenedType(RType, ReduxWidth)), FMF,
CostKind);
}
RdxOpCost = VectorCost;
}
} else {
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
std::make_pair(RedTy, true));
VectorType *RVecTy =
cast<VectorType>(getWidenedType(RType, ReduxWidth));
InstructionCost FMACost = InstructionCost::getInvalid();
if (RdxKind == RecurKind::FAdd) {
// Check if the reduction operands can be converted to FMA.
SmallVector<Value *> Ops;
FastMathFlags FMF;
FMF.set();
for (Value *RdxVal : ReducedVals) {
// The vector fmul fuses only if all the reduced values are
// fusable fmuls.
if (!RdxVal->hasOneUse() ||
!isFusableFMulOperand(
RdxVal, cast<Instruction>(RdxVal->user_back()), *TTI)) {
Ops.clear();
break;
}
if (auto *FPCI = dyn_cast<FPMathOperator>(RdxVal))
FMF &= FPCI->getFastMathFlags();
Ops.push_back(RdxVal->user_back());
}
if (!Ops.empty()) {
InstructionsState S = getSameOpcode(Ops, TLI);
if (S && S.isAddSubOrFNegLikeOp())
FMACost = canConvertToFMA(Ops, S, DT, DL, *TTI, TLI, R);
if (FMACost.isValid()) {
// Calculate actual FMAD cost.
IntrinsicCostAttributes ICA(Intrinsic::fmuladd, RVecTy,
{RVecTy, RVecTy, RVecTy}, FMF);
FMACost = TTI->getIntrinsicInstrCost(ICA, CostKind);
LLVM_DEBUG(dbgs() << "Vector FMA cost: " << FMACost << "\n");
// Also, exclude vector fmul cost.
InstructionCost FMulCost = TTI->getArithmeticInstrCost(
Instruction::FMul, RVecTy, CostKind);
LLVM_DEBUG(dbgs()
<< "Minus vector FMul cost: " << FMulCost << "\n");
FMACost -= FMulCost;
}
}
}
if (FMACost.isValid())
VectorCost += FMACost;
else
VectorCost +=
TTI->getArithmeticInstrCost(RdxOpcode, RVecTy, CostKind);
if (RType != RedTy) {
unsigned Opcode = Instruction::Trunc;
if (RedTy->getScalarSizeInBits() > RType->getScalarSizeInBits())
Opcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
VectorCost += TTI->getCastInstrCost(
Opcode, VectorTy, RVecTy, TTI::CastContextHint::None, CostKind);
}
}
}
Type *ScalarCostTy =
!NarrowedLeafShifts.empty() ? ReductionRoot->getType() : ScalarTy;
ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
// A flattened fneg chain link is removed together with the fadd/fsub
// consuming it - cost both. Also, RdxOp may be the reduced value
// itself, if the actual reduction operation was not found - guard the
// second operand access.
if (RdxOp->isUnaryOp())
return TTI->getArithmeticInstrCost(
RdxOp->getOpcode(), ScalarCostTy, CostKind,
TTI::getOperandInfo(RdxOp->getOperand(0)), {}, {}, RdxOp) +
TTI->getArithmeticInstrCost(RdxOpcode, ScalarCostTy, CostKind);
TTI::OperandValueInfo Op2Info;
if (RdxOp->getNumOperands() > 1)
Op2Info = TTI::getOperandInfo(RdxOp->getOperand(1));
return TTI->getArithmeticInstrCost(
RdxOpcode, ScalarCostTy, CostKind,
TTI::getOperandInfo(RdxOp->getOperand(0)), Op2Info, {}, RdxOp);
});
// Sign-flipped combines are emitted as fsubs but costed as fadds:
// same op count as the unflattened chain, same cost.
break;
}
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin: {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
RdxOpCost = VectorCost;
} else {
// Check if the previous reduction already exists and account it as
// series of operations + single reduction.
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
std::make_pair(RedTy, true));
VectorType *RVecTy =
cast<VectorType>(getWidenedType(RType, ReduxWidth));
IntrinsicCostAttributes ICA(Id, RVecTy, {RVecTy, RVecTy}, FMF);
VectorCost += TTI->getIntrinsicInstrCost(ICA, CostKind);
if (RType != RedTy) {
unsigned Opcode = Instruction::Trunc;
if (RedTy->getScalarSizeInBits() > RType->getScalarSizeInBits())
Opcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
VectorCost += TTI->getCastInstrCost(
Opcode, VectorTy, RVecTy, TTI::CastContextHint::None, CostKind);
}
RdxOpCost = VectorCost;
}
}
ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
IntrinsicCostAttributes ICA(Id, ScalarTy, {ScalarTy, ScalarTy}, FMF);
return TTI->getIntrinsicInstrCost(ICA, CostKind);
});
break;
}
default:
llvm_unreachable("Expected arithmetic or min/max reduction operation");
}
LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VectorCost - ScalarCost
<< " for reduction of " << shortBundleName(ReducedVals)
<< " (It is a splitting reduction)\n");
return VectorCost - ScalarCost;
}
/// Splits the values, stored in VectorValuesAndScales, into registers/free
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
/// The parts, marked as negated, are combined with the vector fsub operation
/// (which may form per-lane fma with the multiplications in the reduced
/// values). \returns the combined value and a flag set if it represents a
/// negated result (all parts negated) and must be subtracted in the final
/// combine.
std::pair<Value *, bool> emitReduction(IRBuilderBase &Builder,
const TargetTransformInfo &TTI,
TTI::CastContextHint Ctx,
Type *DestTy) {
Value *ReducedSubTree = nullptr;
bool ResNegated = false;
// Combines the accumulated value \p Acc (with the sign \p AccNegated)
// with \p V (with the sign \p VNegated). Values with different signs are
// combined via fsub, subtracting the negated one from the positive one, so
// the result is positive. If \p KeepAccSign is set (the result is inserted
// back into the wider \p Acc), the result keeps the sign of \p Acc
// instead: with the negated Acc it is Acc - V == -(V - Acc).
// \returns the combined value and its sign.
auto CombineOps = [&](Value *Acc, bool AccNegated, Value *V, bool VNegated,
const Twine &Name,
bool KeepAccSign) -> std::pair<Value *, bool> {
if (AccNegated == VNegated)
return {createOp(Builder, RdxKind, Acc, V, Name, ReductionOps),
AccNegated};
assert(RdxKind == RecurKind::FAdd &&
"Negated parts expected in the fadd reductions only");
if (AccNegated && !KeepAccSign)
std::swap(Acc, V);
return {createOp(Builder, RecurKind::FSub, Acc, V, Name, ReductionOps),
AccNegated && KeepAccSign};
};
// Creates reduction and combines with the previous reduction, respecting
// the signs of the operands.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
bool ReducedInTree, bool Negated) {
Value *Rdx = createSingleOp(Builder, TTI, Ctx, Vec, Scale, IsSigned,
DestTy, ReducedInTree);
if (!ReducedSubTree) {
ReducedSubTree = Rdx;
ResNegated = Negated;
return;
}
std::tie(ReducedSubTree, ResNegated) =
CombineOps(ReducedSubTree, ResNegated, Rdx, Negated, "op.rdx",
/*KeepAccSign=*/false);
};
if (VectorValuesAndScales.size() == 1) {
const ReductionVectorPart &P = VectorValuesAndScales.front();
CreateSingleOp(P.Vec, P.Scale, P.IsSigned, P.ReducedInTree, P.Negated);
return {ReducedSubTree, ResNegated};
}
// Scales Vec using given Cnt scale factor and then performs vector combine
// with previous value of VecOp.
Value *VecRes = nullptr;
bool VecResSignedness = false;
bool VecResNegated = false;
auto CreateVecOp = [&](Value *Vec, unsigned Cnt, bool IsSigned,
bool ReducedInTree, bool Negated) {
if (ReducedInTree) {
CreateSingleOp(Vec, Cnt, IsSigned, ReducedInTree, Negated);
return;
}
Type *ScalarTy = Vec->getType()->getScalarType();
// Scale Vec using given Cnt scale factor.
if (Cnt > 1) {
ElementCount EC = cast<VectorType>(Vec->getType())->getElementCount();
switch (RdxKind) {
case RecurKind::Add: {
if (ScalarTy == Builder.getInt1Ty() && ScalarTy != DestTy) {
unsigned VF = getNumElements(Vec->getType());
LLVM_DEBUG(dbgs() << "SLP: ctpop " << Cnt << "of " << Vec
<< ". (HorRdx)\n");
SmallVector<int> Mask(Cnt * VF, PoisonMaskElem);
for (unsigned I : seq<unsigned>(Cnt))
std::iota(std::next(Mask.begin(), VF * I),
std::next(Mask.begin(), VF * (I + 1)), 0);
++NumVectorInstructions;
Vec = Builder.CreateShuffleVector(Vec, Mask);
break;
}
// res = mul vv, n
if (ScalarTy != DestTy->getScalarType())
Vec = Builder.CreateIntCast(
Vec, getWidenedType(DestTy, getNumElements(Vec->getType())),
IsSigned);
Value *Scale = ConstantVector::getSplat(
EC, ConstantInt::get(DestTy->getScalarType(), Cnt));
LLVM_DEBUG(dbgs() << "SLP: Add (to-mul) " << Cnt << "of " << Vec
<< ". (HorRdx)\n");
++NumVectorInstructions;
Vec = Builder.CreateMul(Vec, Scale);
break;
}
case RecurKind::Xor: {
// res = n % 2 ? 0 : vv
LLVM_DEBUG(dbgs()
<< "SLP: Xor " << Cnt << "of " << Vec << ". (HorRdx)\n");
if (Cnt % 2 == 0)
Vec = Constant::getNullValue(Vec->getType());
break;
}
case RecurKind::FAdd: {
// res = fmul v, n
Value *Scale =
ConstantVector::getSplat(EC, ConstantFP::get(ScalarTy, Cnt));
LLVM_DEBUG(dbgs() << "SLP: FAdd (to-fmul) " << Cnt << "of " << Vec
<< ". (HorRdx)\n");
++NumVectorInstructions;
Vec = Builder.CreateFMul(Vec, Scale);
break;
}
case RecurKind::And:
case RecurKind::Or:
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
// res = vv
break;
case RecurKind::Sub:
case RecurKind::FSub:
case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
}
// Combine Vec with the previous VecOp.
if (!VecRes) {
VecRes = Vec;
VecResSignedness = IsSigned;
VecResNegated = Negated;
} else {
++NumVectorInstructions;
if (ScalarTy == Builder.getInt1Ty() && ScalarTy != DestTy &&
VecRes->getType()->getScalarType() == Builder.getInt1Ty()) {
// Handle ctpop.
unsigned VecResVF = getNumElements(VecRes->getType());
unsigned VecVF = getNumElements(Vec->getType());
SmallVector<int> Mask(VecResVF + VecVF, PoisonMaskElem);
std::iota(Mask.begin(), Mask.end(), 0);
// Ensure that VecRes is always larger than Vec
if (VecResVF < VecVF) {
std::swap(VecRes, Vec);
std::swap(VecResVF, VecVF);
}
if (VecResVF != VecVF) {
SmallVector<int> ResizeMask(VecResVF, PoisonMaskElem);
std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0);
Vec = Builder.CreateShuffleVector(Vec, ResizeMask);
}
VecRes = Builder.CreateShuffleVector(VecRes, Vec, Mask, "rdx.op");
return;
}
if (VecRes->getType()->getScalarType() != DestTy->getScalarType()) {
assert(getNumElements(VecRes->getType()) % getNumElements(DestTy) ==
0 &&
"Expected the number of elements in VecRes to be a multiple "
"of the number of elements in DestTy");
VecRes = Builder.CreateIntCast(
VecRes,
getWidenedType(DestTy->getScalarType(),
getNumElements(VecRes->getType())),
VecResSignedness);
}
if (ScalarTy != DestTy->getScalarType())
Vec = Builder.CreateIntCast(
Vec,
getWidenedType(DestTy->getScalarType(),
getNumElements(Vec->getType())),
IsSigned);
unsigned VecResVF = getNumElements(VecRes->getType());
unsigned VecVF = getNumElements(Vec->getType());
// Ensure that VecRes is always larger than Vec
if (VecResVF < VecVF) {
std::swap(VecRes, Vec);
std::swap(VecResVF, VecVF);
std::swap(VecResNegated, Negated);
}
// extract + op + insert
Value *Op = VecRes;
if (VecResVF != VecVF)
Op = createExtractVector(Builder, VecRes, VecVF, /*Index=*/0);
std::tie(Op, VecResNegated) =
CombineOps(Op, VecResNegated, Vec, Negated, "rdx.op",
/*KeepAccSign=*/VecResVF != VecVF);
if (VecResVF != VecVF)
Op = createInsertVector(Builder, VecRes, Op, /*Index=*/0);
VecRes = Op;
}
};
// Emit the positive parts first, so the combined vector value stays
// positive when there is at least one positive part.
for (bool Negated : {false, true})
for (const ReductionVectorPart &P : VectorValuesAndScales)
if (P.Negated == Negated)
CreateVecOp(P.Vec, P.Scale, P.IsSigned, P.ReducedInTree, P.Negated);
// All parts may be reduced in the tree already, leaving no vector value
// to reduce.
if (VecRes)
CreateSingleOp(VecRes, /*Scale=*/1, /*IsSigned=*/false,
/*ReducedInTree=*/false, VecResNegated);
return {ReducedSubTree, ResNegated};
}
/// Emits an i1 reduction in the cheaper of the plain and the bitcast-based
/// form. Returns nullptr if not an i1 reduction handled here.
Value *emitI1Reduction(Value *VectorizedValue, IRBuilderBase &Builder,
const TargetTransformInfo &TTI,
TTI::CastContextHint Ctx, Type *DestTy, Value *Start) {
auto *FTy = cast<FixedVectorType>(VectorizedValue->getType());
bool IsAdd = RdxKind == RecurKind::Add &&
DestTy->getScalarType() != FTy->getScalarType();
bool IsBoolLogic =
(RdxKind == RecurKind::And || RdxKind == RecurKind::Or) &&
DestTy->getScalarType() == FTy->getScalarType();
if (Start || !FTy->getScalarType()->isIntegerTy(1) ||
(!IsAdd && !IsBoolLogic))
return nullptr;
if (getI1ReductionCost(
RdxKind, TTI, FTy, DestTy->getScalarType(), Ctx,
getSLPCostKind(Builder.GetInsertBlock()->getParent()))
.second) {
// Convert vector_reduce_and/or(<n x i1>) to icmp eq/ne(bitcast <n x i1>
// to in) and vector_reduce_add(ZExt(<n x i1>)) to
// ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
Value *V = Builder.CreateBitCast(
VectorizedValue, Builder.getIntNTy(FTy->getNumElements()));
++NumVectorInstructions;
switch (RdxKind) {
case RecurKind::And:
return Builder.CreateICmpEQ(V,
ConstantInt::getAllOnesValue(V->getType()));
case RecurKind::Or:
return Builder.CreateIsNotNull(V);
case RecurKind::Add:
return Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, V);
default:
llvm_unreachable("Unexpected reduction kind for the i1 bitcast form");
}
}
if (IsAdd) {
// Keep the extended vector_reduce_add form.
VectorizedValue = Builder.CreateZExt(
VectorizedValue,
getWidenedType(DestTy->getScalarType(), FTy->getNumElements()));
++NumVectorInstructions;
}
++NumVectorInstructions;
return createSimpleReduction(Builder, VectorizedValue, RdxKind);
}
/// Emit a horizontal reduction of the vectorized value.
/// If \p Start is non-null, emit an ordered reduction intrinsic that
/// sequentially accumulates into \p Start (only valid for FAdd/FMulAdd).
Value *emitReduction(Value *VectorizedValue, IRBuilderBase &Builder,
const TargetTransformInfo *TTI, TTI::CastContextHint Ctx,
Type *DestTy, Value *Start = nullptr) {
assert(VectorizedValue && "Need to have a vectorized tree node");
assert(RdxKind != RecurKind::FMulAdd &&
"A call to the llvm.fmuladd intrinsic is not handled yet");
if (Value *Rdx =
emitI1Reduction(VectorizedValue, Builder, *TTI, Ctx, DestTy, Start))
return Rdx;
++NumVectorInstructions;
if (Start)
return createOrderedReduction(Builder, RdxKind, VectorizedValue, Start);
return createSimpleReduction(Builder, VectorizedValue, RdxKind);
}
/// Emits optimized code for unique scalar value reused \p Cnt times.
Value *emitScaleForReusedOps(Value *VectorizedValue, IRBuilderBase &Builder,
unsigned Cnt) {
assert(IsSupportedHorRdxIdentityOp &&
"The optimization of matched scalar identity horizontal reductions "
"must be supported.");
if (Cnt == 1)
return VectorizedValue;
switch (RdxKind) {
case RecurKind::Add: {
// res = mul vv, n
Value *Scale =
ConstantInt::get(VectorizedValue->getType(), Cnt,
/*IsSigned=*/false, /*ImplicitTrunc=*/true);
LLVM_DEBUG(dbgs() << "SLP: Add (to-mul) " << Cnt << "of "
<< VectorizedValue << ". (HorRdx)\n");
return Builder.CreateMul(VectorizedValue, Scale);
}
case RecurKind::Xor: {
// res = n % 2 ? 0 : vv
LLVM_DEBUG(dbgs() << "SLP: Xor " << Cnt << "of " << VectorizedValue
<< ". (HorRdx)\n");
if (Cnt % 2 == 0)
return Constant::getNullValue(VectorizedValue->getType());
return VectorizedValue;
}
case RecurKind::FAdd: {
// res = fmul v, n
Value *Scale = ConstantFP::get(VectorizedValue->getType(), Cnt);
LLVM_DEBUG(dbgs() << "SLP: FAdd (to-fmul) " << Cnt << "of "
<< VectorizedValue << ". (HorRdx)\n");
// The mul stands in for a run of the chain operations and joins the
// same combines, so it carries the flags of the whole chain, except
// contract: an fma candidate would only block the later vectorization
// of the scaled value's operands (fma candidates are deferred).
Value *Mul = createOp(Builder, RecurKind::FMul, VectorizedValue, Scale,
"", ReductionOps);
if (auto *MulI = dyn_cast<Instruction>(Mul))
MulI->setHasAllowContract(false);
return Mul;
}
case RecurKind::And:
case RecurKind::Or:
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
// res = vv
return VectorizedValue;
case RecurKind::Sub:
case RecurKind::FSub:
case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
return nullptr;
}
/// Emits actual operation for the scalar identity values, found during
/// horizontal reduction analysis.
Value *
emitReusedOps(Value *VectorizedValue, IRBuilderBase &Builder, BoUpSLP &R,
const SmallMapVector<Value *, unsigned, 16> &SameValuesCounter,
ArrayRef<Value *> TrackedToOrig) {
assert(IsSupportedHorRdxIdentityOp &&
"The optimization of matched scalar identity horizontal reductions "
"must be supported.");
ArrayRef<Value *> VL = R.getRootNodeScalars();
auto *VTy = cast<FixedVectorType>(VectorizedValue->getType());
if (VTy->getElementType() != VL.front()->getType()) {
VectorizedValue = Builder.CreateIntCast(
VectorizedValue,
getWidenedType(VL.front()->getType(), VTy->getNumElements()),
R.isSignedMinBitwidthRootNode());
}
switch (RdxKind) {
case RecurKind::Add: {
// root = mul prev_root, <1, 1, n, 1>
SmallVector<Constant *> Vals;
for (const auto [Idx, V] : enumerate(VL)) {
unsigned Cnt = SameValuesCounter.lookup(TrackedToOrig[Idx]);
// The repeat count may not fit the element type (e.g. i1); the
// modular reduction arithmetic keeps the truncated count equivalent.
Vals.push_back(ConstantInt::get(
V->getType(), APInt(V->getType()->getIntegerBitWidth(), Cnt,
/*isSigned=*/false, /*implicitTrunc=*/true)));
}
auto *Scale = ConstantVector::get(Vals);
LLVM_DEBUG(dbgs() << "SLP: Add (to-mul) " << Scale << "of "
<< VectorizedValue << ". (HorRdx)\n");
return Builder.CreateMul(VectorizedValue, Scale);
}
case RecurKind::And:
case RecurKind::Or:
// No need for multiple or/and(s).
LLVM_DEBUG(dbgs() << "SLP: And/or of same " << VectorizedValue
<< ". (HorRdx)\n");
return VectorizedValue;
case RecurKind::SMax:
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
// No need for multiple min/max(s) of the same value.
LLVM_DEBUG(dbgs() << "SLP: Max/min of same " << VectorizedValue
<< ". (HorRdx)\n");
return VectorizedValue;
case RecurKind::Xor: {
// Replace values with even number of repeats with 0, since
// x xor x = 0.
// root = shuffle prev_root, zeroinitalizer, <0, 1, 2, vf, 4, vf, 5, 6,
// 7>, if elements 4th and 6th elements have even number of repeats.
SmallVector<int> Mask(
cast<FixedVectorType>(VectorizedValue->getType())->getNumElements(),
PoisonMaskElem);
std::iota(Mask.begin(), Mask.end(), 0);
bool NeedShuffle = false;
for (const auto [Idx, V] : enumerate(VL)) {
unsigned Cnt = SameValuesCounter.lookup(TrackedToOrig[Idx]);
if (Cnt % 2 == 0) {
Mask[Idx] = VL.size();
NeedShuffle = true;
}
}
LLVM_DEBUG(dbgs() << "SLP: Xor <"; for (int I
: Mask) dbgs()
<< I << " ";
dbgs() << "> of " << VectorizedValue << ". (HorRdx)\n");
if (NeedShuffle)
VectorizedValue = Builder.CreateShuffleVector(
VectorizedValue,
ConstantVector::getNullValue(VectorizedValue->getType()), Mask);
return VectorizedValue;
}
case RecurKind::FAdd: {
// root = fmul prev_root, <1.0, 1.0, n.0, 1.0>
SmallVector<Constant *> Vals;
for (const auto [Idx, V] : enumerate(VL)) {
unsigned Cnt = SameValuesCounter.lookup(TrackedToOrig[Idx]);
Vals.push_back(ConstantFP::get(V->getType(), Cnt));
}
auto *Scale = ConstantVector::get(Vals);
return Builder.CreateFMul(VectorizedValue, Scale);
}
case RecurKind::Sub:
case RecurKind::FSub:
case RecurKind::FAddChainWithSubs:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
case RecurKind::AnyOf:
case RecurKind::FindIV:
case RecurKind::FindLast:
case RecurKind::FMaxNum:
case RecurKind::FMinNum:
case RecurKind::FMaximumNum:
case RecurKind::FMinimumNum:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for reused scalars.");
}
return nullptr;
}
};
} // end anonymous namespace
/// Gets recurrence kind from the specified value.
static RecurKind getRdxKind(Value *V) {
return HorizontalReduction::getRdxKind(V);
}
static void findBuildAggregateRec(Instruction *LastInsertInst,
TargetTransformInfo *TTI,
SmallVectorImpl<Value *> &BuildVectorOpds,
SmallVectorImpl<Value *> &InsertElts,
unsigned OperandOffset, const BoUpSLP &R) {
do {
Value *InsertedOperand = LastInsertInst->getOperand(1);
std::optional<unsigned> OperandIndex =
getElementIndex(LastInsertInst, OperandOffset);
if (!OperandIndex || R.isDeleted(LastInsertInst))
return;
if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {
findBuildAggregateRec(cast<Instruction>(InsertedOperand), TTI,
BuildVectorOpds, InsertElts, *OperandIndex, R);
} else {
BuildVectorOpds[*OperandIndex] = InsertedOperand;
InsertElts[*OperandIndex] = LastInsertInst;
}
LastInsertInst = dyn_cast<Instruction>(LastInsertInst->getOperand(0));
} while (LastInsertInst != nullptr &&
isa<InsertValueInst, InsertElementInst>(LastInsertInst) &&
LastInsertInst->hasOneUse());
}
/// Recognize construction of vectors like
/// %ra = insertelement <4 x float> poison, float %s0, i32 0
/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
/// starting from the last insertelement or insertvalue instruction.
///
/// Also recognize homogeneous aggregates like {<2 x float>, <2 x float>},
/// {{float, float}, {float, float}}, [2 x {float, float}] and so on.
/// See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples.
///
/// Assume LastInsertInst is of InsertElementInst or InsertValueInst type.
///
/// \return true if it matches.
static bool findBuildAggregate(Instruction *LastInsertInst,
TargetTransformInfo *TTI,
SmallVectorImpl<Value *> &BuildVectorOpds,
SmallVectorImpl<Value *> &InsertElts,
const BoUpSLP &R) {
assert((isa<InsertElementInst>(LastInsertInst) ||
isa<InsertValueInst>(LastInsertInst)) &&
"Expected insertelement or insertvalue instruction!");
assert((BuildVectorOpds.empty() && InsertElts.empty()) &&
"Expected empty result vectors!");
std::optional<unsigned> AggregateSize = getAggregateSize(LastInsertInst);
if (!AggregateSize)
return false;
BuildVectorOpds.resize(*AggregateSize);
InsertElts.resize(*AggregateSize);
findBuildAggregateRec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0, R);
llvm::erase(BuildVectorOpds, nullptr);
llvm::erase(InsertElts, nullptr);
if (BuildVectorOpds.size() >= 2)
return true;
return false;
}
/// We could have an initial reduction that is not an add.
/// r *= v1 + v2 + v3 + v4
/// In such a case start looking for a tree rooted in the first '+'.
/// \Returns the new root if found, which may be nullptr if not an instruction.
static Instruction *tryGetSecondaryReductionRoot(PHINode *Phi,
Instruction *Root) {
assert((isa<BinaryOperator>(Root) || isa<SelectInst>(Root) ||
isa<IntrinsicInst>(Root)) &&
"Expected binop, select, or intrinsic for reduction matching");
Value *LHS =
Root->getOperand(HorizontalReduction::getFirstOperandIndex(Root));
Value *RHS =
Root->getOperand(HorizontalReduction::getFirstOperandIndex(Root) + 1);
if (LHS == Phi)
return dyn_cast<Instruction>(RHS);
if (RHS == Phi)
return dyn_cast<Instruction>(LHS);
return nullptr;
}
bool SLPVectorizerPass::vectorizeHorReduction(
PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
SmallVectorImpl<WeakTrackingVH> &PostponedInsts) {
if (!Opts->slp_vectorize_hor)
return false;
bool TryOperandsAsNewSeeds = P && isa<BinaryOperator>(Root);
if (Root->getParent() != BB || isa<PHINode>(Root))
return false;
// If we can find a secondary reduction root, use that instead.
auto SelectRoot = [&]() {
if (TryOperandsAsNewSeeds && isReductionCandidate(Root) &&
HorizontalReduction::getRdxKind(Root) != RecurKind::None)
if (Instruction *NewRoot = tryGetSecondaryReductionRoot(P, Root))
return NewRoot;
return Root;
};
// Start analysis starting from Root instruction. If horizontal reduction is
// found, try to vectorize it. If it is not a horizontal reduction or
// vectorization is not possible or not effective, and currently analyzed
// instruction is a binary operation, try to vectorize the operands, using
// pre-order DFS traversal order. If the operands were not vectorized, repeat
// the same procedure considering each operand as a possible root of the
// horizontal reduction.
// Interrupt the process if the Root instruction itself was vectorized or all
// sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
// If a horizintal reduction was not matched or vectorized we collect
// instructions for possible later attempts for vectorization.
std::queue<std::pair<Instruction *, unsigned>> Stack;
Stack.emplace(SelectRoot(), 0);
SmallPtrSet<Value *, 8> VisitedInstrs;
bool Res = false;
auto TryToReduce = [this, &R, TTI = TTI](Instruction *Inst,
bool IsSeedRoot) -> Value * {
if (R.isAnalyzedReductionRoot(Inst))
return nullptr;
if (!isReductionCandidate(Inst))
return nullptr;
HorizontalReduction HorRdx(*Opts);
Value *Res = nullptr;
if (HorRdx.matchAssociativeReduction(R, Inst, *SE, *DT, *DL, *TTI, *TLI)) {
Value *Red =
HorRdx.tryToReduce(R, *DL, TTI, *TLI, AC, *DT, *LI, IsSeedRoot);
// The chain with the flattened fsub/fneg links produced no vector part:
// retry with the fsub/fneg links as leaves, they may be vectorizable on
// their own.
if (!Red && HorRdx.hasFlattenedNegations()) {
HorizontalReduction UnflattenedHorRdx(*Opts);
if (UnflattenedHorRdx.matchAssociativeReduction(
R, Inst, *SE, *DT, *DL, *TTI, *TLI, /*FlattenNegations=*/false))
Red = UnflattenedHorRdx.tryToReduce(R, *DL, TTI, *TLI, AC, *DT, *LI,
IsSeedRoot);
}
if (Red) {
if (Red != Inst)
return Red;
Res = Red;
}
}
if (HorizontalReduction::isSupportedOrderedReductionOp(Inst)) {
if (HorRdx.matchOrderedReduction(R, Inst, /*MatchLHS=*/true))
if (Value *Red =
HorRdx.tryToReduceOrdered(R, *DL, TTI, *TLI, AC, *DT)) {
if (Red != Inst)
return Red;
Res = Red;
}
if (HorRdx.matchOrderedReduction(R, Inst, /*MatchLHS=*/false))
if (Value *Red = HorRdx.tryToReduceOrdered(R, *DL, TTI, *TLI, AC, *DT))
return Red;
}
return Res;
};
auto TryAppendToPostponedInsts = [&](Instruction *FutureSeed) {
if (TryOperandsAsNewSeeds && FutureSeed == Root) {
FutureSeed = getNonPhiOperand(Root, P);
if (!FutureSeed)
return false;
}
// Do not collect CmpInst or InsertElementInst/InsertValueInst as their
// analysis is done separately.
if (!isa<CmpInst, InsertElementInst, InsertValueInst>(FutureSeed))
PostponedInsts.push_back(FutureSeed);
return true;
};
while (!Stack.empty()) {
Instruction *Inst;
unsigned Level;
std::tie(Inst, Level) = Stack.front();
Stack.pop();
// Do not try to analyze instruction that has already been vectorized.
// This may happen when we vectorize instruction operands on a previous
// iteration while stack was populated before that happened.
if (R.isDeleted(Inst))
continue;
// The minimum-vector-factor allowance applies to single-use seed-level
// roots only.
if (Value *VectorizedV =
TryToReduce(Inst, /*IsSeedRoot=*/Level == 0 && Inst->hasOneUse())) {
Res = true;
if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
Stack.emplace(I, Level);
continue;
}
if (R.isDeleted(Inst))
continue;
} else {
// We could not vectorize `Inst` so try to use it as a future seed.
if (!TryAppendToPostponedInsts(Inst)) {
assert(Stack.empty() && "Expected empty stack");
break;
}
}
// Try to vectorize operands.
// Continue analysis for the instruction from the same basic block only to
// save compile time.
if (++Level < Opts->slp_recursion_max_depth)
for (auto *Op : Inst->operand_values())
if (VisitedInstrs.insert(Op).second)
if (auto *I = dyn_cast<Instruction>(Op))
// Do not try to vectorize CmpInst operands, this is done
// separately.
if (!isa<PHINode, CmpInst, InsertElementInst, InsertValueInst>(I) &&
!R.isDeleted(I) && I->getParent() == BB)
Stack.emplace(I, Level);
}
return Res;
}
bool SLPVectorizerPass::tryToVectorize(
Instruction *I, BoUpSLP &R, SmallSetVector<Instruction *, 8> &FMACandidates,
bool AllowFMACandidates) {
if (!I)
return false;
if (!isa<BinaryOperator, CmpInst>(I) || isa<VectorType>(I->getType()))
return false;
// Skip potential FMA candidates and collect them for a retry after all
// blocks of the function have been processed.
if (!AllowFMACandidates &&
(I->getOpcode() == Instruction::FAdd ||
I->getOpcode() == Instruction::FSub) &&
canConvertToFMA(I, getSameOpcode(I, *TLI), *DT, *DL, *TTI, *TLI, R)
.isValid()) {
FMACandidates.insert(I);
return false;
}
Value *P = I->getParent();
// Vectorize in current basic block only.
auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P ||
R.isDeleted(Op0) || R.isDeleted(Op1))
return false;
// First collect all possible candidates
SmallVector<std::pair<Value *, Value *>, 4> Candidates;
Candidates.emplace_back(Op0, Op1);
auto *A = dyn_cast<BinaryOperator>(Op0);
auto *B = dyn_cast<BinaryOperator>(Op1);
// Try to skip B.
if (A && B && B->hasOneUse()) {
auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
if (B0 && B0->getParent() == P && !R.isDeleted(B0))
Candidates.emplace_back(A, B0);
if (B1 && B1->getParent() == P && !R.isDeleted(B1))
Candidates.emplace_back(A, B1);
}
// Try to skip A.
if (B && A && A->hasOneUse()) {
auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
if (A0 && A0->getParent() == P && !R.isDeleted(A0))
Candidates.emplace_back(A0, B);
if (A1 && A1->getParent() == P && !R.isDeleted(A1))
Candidates.emplace_back(A1, B);
}
auto TryToReduce = [this, &R, &TTI = *TTI](Instruction *Inst,
ArrayRef<Value *> Ops) {
if (!isReductionCandidate(Inst))
return false;
// A 2-element reduction that is just a link in a larger reduction chain
// or feeds a buildvector is better handled by the whole-chain reduction
// or buildvector analysis.
RecurKind Kind = getRdxKind(Inst);
auto *FPMO = dyn_cast<FPMathOperator>(Inst);
if (FPMO && !FPMO->hasAllowReassoc() &&
(any_of(Inst->users(),
[Kind](User *U) {
return getRdxKind(U) == Kind || isa<InsertElementInst>(U);
}) ||
any_of(Inst->operands(),
[Kind](Value *Op) { return getRdxKind(Op) == Kind; })))
return false;
Type *Ty = Inst->getType();
if (!isValidElementType(Ty, Opts->slp_revec) || Ty->isPointerTy())
return false;
HorizontalReduction HorRdx(*Opts, Inst, Ops);
if (!HorRdx.matchReductionForOperands())
return false;
// Check the cost of operations.
auto *VecTy = cast<VectorType>(getWidenedType(Ty, Ops.size()));
const TTI::TargetCostKind CostKind = R.getCostKind();
InstructionCost ScalarCost =
TTI.getScalarizationOverhead(
VecTy, APInt::getAllOnes(getNumElements(VecTy)), /*Insert=*/false,
/*Extract=*/true, CostKind) +
TTI.getInstructionCost(Inst, CostKind);
// The reduction-only cost cannot price the compared-values tree of i1
// and/or reductions, so ties are left to the full analysis.
if (Ty->isIntegerTy(1) &&
(Kind == RecurKind::And || Kind == RecurKind::Or)) {
TTI::CastContextHint Ctx =
all_of(Ops, [](Value *Op) { return isa<LoadInst>(Op); })
? TTI::CastContextHint::Normal
: TTI::CastContextHint::None;
if (getI1ReductionCost(Kind, TTI, cast<FixedVectorType>(VecTy), Ty, Ctx,
CostKind)
.first > ScalarCost)
return false;
return HorRdx.tryToReduce(R, *DL, &TTI, *TLI, AC, *DT, *LI) != nullptr;
}
InstructionCost RedCost;
switch (::getRdxKind(Inst)) {
case RecurKind::Add:
case RecurKind::Mul:
case RecurKind::Or:
case RecurKind::And:
case RecurKind::Xor:
case RecurKind::FAdd:
case RecurKind::FMul: {
FastMathFlags FMF;
if (auto *FPCI = dyn_cast<FPMathOperator>(Inst)) {
FMF = FPCI->getFastMathFlags();
// No need to check for associativity, if 2 reduced values.
if (Ops.size() == 2)
FMF.setAllowReassoc(true);
}
RedCost = TTI.getArithmeticReductionCost(Inst->getOpcode(), VecTy, FMF,
CostKind);
break;
}
default:
return false;
}
if (RedCost >= ScalarCost)
return false;
return HorRdx.tryToReduce(R, *DL, &TTI, *TLI, AC, *DT, *LI) != nullptr;
};
if (Candidates.size() == 1)
return TryToReduce(I, {Op0, Op1}) ||
(!AllowFMACandidates && tryToVectorizeList({Op0, Op1}, R));
// We have multiple options. Try to pick the single best.
std::optional<int> BestCandidate = R.findBestRootPair(Candidates).first;
if (!BestCandidate)
return false;
return (*BestCandidate == 0 &&
TryToReduce(I, {Candidates[*BestCandidate].first,
Candidates[*BestCandidate].second})) ||
(!AllowFMACandidates &&
tryToVectorizeList({Candidates[*BestCandidate].first,
Candidates[*BestCandidate].second},
R));
}
bool SLPVectorizerPass::vectorizeRootInstruction(
PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
SmallVector<WeakTrackingVH> PostponedInsts;
bool Res = vectorizeHorReduction(P, Root, BB, R, PostponedInsts);
Res |= tryToVectorize(PostponedInsts, R, FMACandidates);
return Res;
}
bool SLPVectorizerPass::tryToVectorize(
ArrayRef<WeakTrackingVH> Insts, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
bool Res = false;
for (Value *V : Insts)
if (auto *Inst = dyn_cast<Instruction>(V); Inst && !R.isDeleted(Inst))
Res |= tryToVectorize(Inst, R, FMACandidates);
return Res;
}
/// Vectorizes the aggregate \p IVI, built of the packs of the same layout, as
/// the single vector of all the fields, bitcasted to the vector of the packs.
/// The packs vectorized alone would leave the vectors narrower than possible.
static bool vectorizePackedAggregate(InsertValueInst *IVI, BoUpSLP &R,
TargetTransformInfo &TTI,
const DataLayout &DL) {
if (!R.Opts.slp_vectorize_hor || R.Opts.slp_revec || DL.isBigEndian() ||
!R.canMapToVector(IVI->getType()))
return false;
SmallVector<Value *, 16> Packs;
SmallVector<Value *, 16> Inserts;
if (!findBuildAggregate(IVI, &TTI, Packs, Inserts, R))
return false;
// The inserts must form the plain chain, which is moved after the vector.
BasicBlock *BB = IVI->getParent();
if (!all_of(Inserts, [&](Value *V) {
auto *Insert = dyn_cast<InsertValueInst>(V);
if (!Insert || Insert->getParent() != BB ||
(Insert != IVI && !(Insert->hasOneUse() &&
is_contained(Inserts, Insert->user_back()))))
return false;
Value *Agg = Insert->getAggregateOperand();
return !isa<InsertValueInst>(Agg) || is_contained(Inserts, Agg);
}))
return false;
SmallVector<Value *> Fields;
SmallVector<Instruction *> Chain;
for (Value *Pack : Packs) {
auto *Root = dyn_cast<Instruction>(Pack);
if (!Root || Root->getParent() != BB ||
!matchPackedFields(Root, R.Opts.slp_recursion_max_depth, Fields, Chain))
return false;
}
// The pack instructions are dropped, so they must be used by the packs only.
SmallDenseSet<Value *> IgnoreList;
IgnoreList.insert_range(Chain);
if (!all_of(Chain, [&](Instruction *I) {
return I->getParent() == BB && all_of(I->users(), [&](User *U) {
return IgnoreList.contains(U) || is_contained(Inserts, U);
});
}))
return false;
// Merging the loads into the wider load is better done for each pack.
if (all_of(Fields, IsaPred<LoadInst>))
return false;
// The fields of different types are not vectorized, the tree is empty.
R.buildTree(Fields, IgnoreList);
if (R.isTreeTinyAndNotFullyVectorizable(/*ForReduction=*/true))
return false;
if (R.isProfitableToReorder()) {
R.reorderTopToBottom();
// The root is shuffled back to the order of the fields, its order can be
// ignored.
R.reorderBottomToTop(/*IgnoreReorder=*/true);
}
R.transformNodes();
// The fields are not demoted, the bitcast depends on their width.
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable(Fields);
R.buildExternalUses();
// The gathered operands of the fields are unpacked better for each pack.
if (R.hasNonConstantGathers())
return false;
// The packs take the lanes in the order of the fields.
SmallVector<int> OrderMask(Fields.size());
SmallBitVector Seen(Fields.size());
for (auto [Idx, F] : enumerate(Fields)) {
unsigned Lane = R.findRootLaneForValue(F);
if (Lane >= Fields.size() || Seen.test(Lane))
return false;
Seen.set(Lane);
OrderMask[Idx] = Lane;
}
const bool IsIdentity =
ShuffleVectorInst::isIdentityMask(OrderMask, OrderMask.size());
const TTI::TargetCostKind CostKind = R.getCostKind();
const unsigned NumPacks = Packs.size();
Type *FieldsTy = getWidenedType(Fields.front()->getType(), Fields.size());
Type *PacksTy = getWidenedType(Packs.front()->getType(), NumPacks);
InstructionCost PackCost =
TTI.getCastInstrCost(Instruction::BitCast, PacksTy, FieldsTy,
TTI::CastContextHint::None, CostKind);
if (!IsIdentity)
PackCost += getShuffleCost(TTI, TTI::SK_PermuteSingleSrc,
cast<VectorType>(FieldsTy), CostKind, OrderMask);
PackCost += TTI.getScalarizationOverhead(
cast<VectorType>(PacksTy), APInt::getAllOnes(NumPacks),
/*Insert=*/false, /*Extract=*/true, CostKind);
for (Instruction *I : Chain)
PackCost -= TTI.getInstructionCost(I, CostKind);
InstructionCost Cost = R.getTreeCost(TreeCost, Fields, PackCost, IVI);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
<< " for packed aggregate\n");
if (!Cost.isValid() || Cost >= -R.getCostThreshold())
return false;
Value *Vec = R.vectorizeTree({}, IVI);
assert(Vec->getType() == FieldsTy && "Expected the vector of the fields.");
IRBuilder<TargetFolder> Builder(IVI->getIterator(), TargetFolder(DL));
if (!IsIdentity)
Vec = Builder.CreateShuffleVector(Vec, OrderMask);
Value *Packed = Builder.CreateBitCast(Vec, PacksTy);
for (auto [Idx, Insert] : enumerate(Inserts))
cast<InsertValueInst>(Insert)->setOperand(
1, Builder.CreateExtractElement(Packed, Idx));
// The inserts may precede the vector, move them after the extracts.
InsertValueInst *Next = IVI;
while (auto *Prev = dyn_cast<InsertValueInst>(Next->getAggregateOperand())) {
if (!is_contained(Inserts, Prev))
break;
Prev->moveBefore(Next->getIterator());
Next = Prev;
}
R.removeInstructionsAndOperands(ArrayRef(Chain), {});
return true;
}
bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
BasicBlock *BB, BoUpSLP &R,
bool MaxVFOnly) {
if (!R.canMapToVector(IVI->getType()))
return false;
SmallVector<Value *, 16> BuildVectorOpds;
SmallVector<Value *, 16> BuildVectorInsts;
if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, BuildVectorInsts, R))
return false;
if (MaxVFOnly && BuildVectorOpds.size() == 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "NotPossible", IVI)
<< "Cannot SLP vectorize list: only 2 elements of buildvalue, "
"trying reduction first.";
});
return false;
}
LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
// Aggregate value is unlikely to be processed in vector register.
// Try to vectorize the insertvalue chain itself (rebuilding the aggregate
// from a vector) only for struct aggregates: getContainedTypes() returns the
// per-field types for a struct, but returns the aggregate type itself as a
// single element for arrays and other aggregates, so the size check below
// fails for them and they fall back to vectorizing the scalar operands.
// TODO: Extend the vectorized-store path to array aggregates.
Type *BVType = BuildVectorInsts.front()->getType();
ArrayRef<Type *> ContainedTypes = getContainedTypes(BVType);
if (ContainedTypes.size() == BuildVectorInsts.size() &&
isValidElementType(ContainedTypes.front(), Opts->slp_revec) &&
all_of(ContainedTypes,
[&](Type *Ty) { return Ty == ContainedTypes.front(); }))
if (tryToVectorizeList(BuildVectorInsts, R, MaxVFOnly))
return true;
return tryToVectorizeList(BuildVectorOpds, R, MaxVFOnly);
}
bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
BasicBlock *BB, BoUpSLP &R,
bool MaxVFOnly) {
SmallVector<Value *, 16> BuildVectorInsts;
SmallVector<Value *, 16> BuildVectorOpds;
SmallVector<int> Mask;
if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts, R) ||
(all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
isFixedVectorShuffle(BuildVectorOpds, Mask, AC, R.getAnalysisCache())))
return false;
if (MaxVFOnly && BuildVectorInsts.size() == 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "NotPossible", IEI)
<< "Cannot SLP vectorize list: only 2 elements of buildvector, "
"trying reduction first.";
});
return false;
}
LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IEI << "\n");
return tryToVectorizeList(BuildVectorInsts, R, MaxVFOnly);
}
template <typename T>
static bool tryToVectorizeSequence(
const VectorizeOptions &Opts, SmallVectorImpl<T *> &Incoming,
function_ref<bool(T *, T *)> Comparator,
function_ref<bool(ArrayRef<T *>, T *)> AreCompatible,
function_ref<bool(ArrayRef<T *>, bool)> TryToVectorizeHelper,
bool MaxVFOnly, BoUpSLP &R,
function_ref<bool(ArrayRef<T *>)> Filter = nullptr) {
bool Changed = false;
// Sort by type, parent, operands.
stable_sort(Incoming, Comparator);
// Try to vectorize elements base on their type.
SmallVector<T *> Candidates;
SmallVector<T *> VL;
for (auto *IncIt = Incoming.begin(), *E = Incoming.end(); IncIt != E;
VL.clear()) {
// Look for the next elements with the same type, parent and operand
// kinds.
auto *I = dyn_cast<Instruction>(*IncIt);
if (!I || R.isDeleted(I) ||
!isValidElementType(getValueType(I, Opts.slp_revec), Opts.slp_revec)) {
++IncIt;
continue;
}
auto *SameTypeIt = IncIt;
while (SameTypeIt != E && (!isa<Instruction>(*SameTypeIt) ||
R.isDeleted(cast<Instruction>(*SameTypeIt)) ||
AreCompatible(VL, *SameTypeIt))) {
auto *I = dyn_cast<Instruction>(*SameTypeIt);
++SameTypeIt;
if (I && !R.isDeleted(I))
VL.push_back(cast<T>(I));
}
// Rejected groups are dropped, not left among the candidates.
if (Filter && !Filter(ArrayRef(VL)))
VL.clear();
// Try to vectorize them.
unsigned NumElts = VL.size();
LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at nodes ("
<< NumElts << ")\n");
// The vectorization is a 3-state attempt:
// 1. Try to vectorize instructions with the same/alternate opcodes with the
// size of maximal register at first.
// 2. Try to vectorize remaining instructions with the same type, if
// possible. This may result in the better vectorization results rather than
// if we try just to vectorize instructions with the same/alternate opcodes.
// 3. Final attempt to try to vectorize all instructions with the
// same/alternate ops only, this may result in some extra final
// vectorization.
if (NumElts > 1 && TryToVectorizeHelper(ArrayRef(VL), MaxVFOnly)) {
// Success start over because instructions might have been changed.
Changed = true;
VL.swap(Candidates);
Candidates.clear();
for (T *V : VL) {
if (auto *I = dyn_cast<Instruction>(V); I && !R.isDeleted(I))
Candidates.push_back(V);
}
} else {
/// \Returns the minimum number of elements that we will attempt to
/// vectorize.
auto GetMinNumElements = [&R](Value *V) {
unsigned EltSize = R.getVectorElementSize(V);
return std::max(2U, R.getMaxVecRegSize() / EltSize);
};
if (NumElts < GetMinNumElements(*IncIt) &&
(Candidates.empty() ||
Candidates.front()->getType() == (*IncIt)->getType())) {
for (T *V : VL) {
if (auto *I = dyn_cast<Instruction>(V); I && !R.isDeleted(I))
Candidates.push_back(V);
}
}
}
// Final attempt to vectorize instructions with the same types.
if (Candidates.size() > 1 &&
(SameTypeIt == E || (*SameTypeIt)->getType() != (*IncIt)->getType())) {
if (TryToVectorizeHelper(Candidates, /*MaxVFOnly=*/false)) {
// Success start over because instructions might have been changed.
Changed = true;
} else if (MaxVFOnly) {
// Try to vectorize using small vectors.
SmallVector<T *> VL;
for (auto *It = Candidates.begin(), *End = Candidates.end(); It != End;
VL.clear()) {
auto *I = dyn_cast<Instruction>(*It);
if (!I || R.isDeleted(I)) {
++It;
continue;
}
auto *SameTypeIt = It;
while (SameTypeIt != End &&
(!isa<Instruction>(*SameTypeIt) ||
R.isDeleted(cast<Instruction>(*SameTypeIt)) ||
AreCompatible(*SameTypeIt, *It))) {
auto *I = dyn_cast<Instruction>(*SameTypeIt);
++SameTypeIt;
if (I && !R.isDeleted(I))
VL.push_back(cast<T>(I));
}
unsigned NumElts = VL.size();
if (NumElts > 1 && TryToVectorizeHelper(ArrayRef(VL),
/*MaxVFOnly=*/false))
Changed = true;
It = SameTypeIt;
}
}
Candidates.clear();
}
// Start over at the next instruction of a different type (or the end).
IncIt = SameTypeIt;
}
return Changed;
}
/// Compare two cmp instructions. If IsCompatibility is true, function returns
/// true if 2 cmps have same/swapped predicates and mos compatible corresponding
/// operands. If IsCompatibility is false, function implements strict weak
/// ordering relation between two cmp instructions, returning true if the first
/// instruction is "less" than the second, i.e. its predicate is less than the
/// predicate of the second or the operands IDs are less than the operands IDs
/// of the second cmp instruction.
template <bool IsCompatibility>
static bool compareCmp(const VectorizeOptions &Opts, Value *V, Value *V2,
TargetLibraryInfo &TLI, const DominatorTree &DT) {
assert(isValidElementType(V->getType(), Opts.slp_revec) &&
isValidElementType(V2->getType(), Opts.slp_revec) &&
"Expected valid element types only.");
if (V == V2)
return IsCompatibility;
auto *CI1 = cast<CmpInst>(V);
auto *CI2 = cast<CmpInst>(V2);
if (CI1->getOperand(0)->getType()->getTypeID() <
CI2->getOperand(0)->getType()->getTypeID())
return !IsCompatibility;
if (CI1->getOperand(0)->getType()->getTypeID() >
CI2->getOperand(0)->getType()->getTypeID())
return false;
if (CI1->getOperand(0)->getType()->getScalarSizeInBits() <
CI2->getOperand(0)->getType()->getScalarSizeInBits())
return !IsCompatibility;
if (CI1->getOperand(0)->getType()->getScalarSizeInBits() >
CI2->getOperand(0)->getType()->getScalarSizeInBits())
return false;
CmpInst::Predicate Pred1 = CI1->getPredicate();
CmpInst::Predicate Pred2 = CI2->getPredicate();
CmpInst::Predicate SwapPred1 = CmpInst::getSwappedPredicate(Pred1);
CmpInst::Predicate SwapPred2 = CmpInst::getSwappedPredicate(Pred2);
CmpInst::Predicate BasePred1 = std::min(Pred1, SwapPred1);
CmpInst::Predicate BasePred2 = std::min(Pred2, SwapPred2);
if (BasePred1 < BasePred2)
return !IsCompatibility;
if (BasePred1 > BasePred2)
return false;
// Compare operands.
bool CI1Preds = Pred1 == BasePred1;
bool CI2Preds = Pred2 == BasePred1;
for (int I = 0, E = CI1->getNumOperands(); I < E; ++I) {
auto *Op1 = CI1->getOperand(CI1Preds ? I : E - I - 1);
auto *Op2 = CI2->getOperand(CI2Preds ? I : E - I - 1);
if (Op1 == Op2)
continue;
if (Op1->getValueID() < Op2->getValueID())
return !IsCompatibility;
if (Op1->getValueID() > Op2->getValueID())
return false;
if (auto *I1 = dyn_cast<Instruction>(Op1))
if (auto *I2 = dyn_cast<Instruction>(Op2)) {
if (IsCompatibility) {
if (I1->getParent() != I2->getParent())
return false;
} else {
// Try to compare nodes with same parent.
DomTreeNodeBase<BasicBlock> *NodeI1 = DT.getNode(I1->getParent());
DomTreeNodeBase<BasicBlock> *NodeI2 = DT.getNode(I2->getParent());
if (!NodeI1)
return NodeI2 != nullptr;
if (!NodeI2)
return false;
assert((NodeI1 == NodeI2) ==
(NodeI1->getDFSNumIn() == NodeI2->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
}
InstructionsState S = getSameOpcode({I1, I2}, TLI);
if (S && (IsCompatibility || !S.isAltShuffle()))
continue;
if (IsCompatibility)
return false;
if (I1->getOpcode() != I2->getOpcode())
return I1->getOpcode() < I2->getOpcode();
}
}
return IsCompatibility;
}
template <typename ItT>
bool SLPVectorizerPass::vectorizeCmpInsts(
iterator_range<ItT> CmpInsts, BasicBlock *BB, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
bool Changed = false;
// Try to find reductions first.
for (CmpInst *I : make_filter_range(
CmpInsts, [&](CmpInst *I) { return !R.isDeleted(I); })) {
for (Instruction *RootOp : make_isa_range<Instruction>(I->operands())) {
Changed |=
vectorizeRootInstruction(nullptr, RootOp, BB, R, FMACandidates);
if (R.isDeleted(I))
break;
}
}
// Try to vectorize operands as vector bundles.
for (CmpInst *I :
make_filter_range(CmpInsts, [&](CmpInst *I) { return !R.isDeleted(I); }))
Changed |= tryToVectorize(I, R, FMACandidates);
// Try to vectorize list of compares.
// Sort by type, compare predicate, etc.
auto CompareSorter = [&](Value *V, Value *V2) {
if (V == V2)
return false;
return compareCmp<false>(*Opts, V, V2, *TLI, *DT);
};
auto AreCompatibleCompares = [&](ArrayRef<Value *> VL, Value *V1) {
if (VL.empty() || VL.back() == V1)
return true;
return compareCmp<true>(*Opts, V1, VL.back(), *TLI, *DT);
};
SmallVector<Value *> Vals;
for (Instruction *V : CmpInsts)
if (!R.isDeleted(V) &&
isValidElementType(
getValueType(V, Opts->slp_revec, /*LookThroughCmp=*/true),
Opts->slp_revec))
Vals.push_back(V);
if (Vals.size() <= 1)
return Changed;
Changed |= tryToVectorizeSequence<Value>(
*Opts, Vals, CompareSorter, AreCompatibleCompares,
[this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
// Exclude possible reductions from other blocks.
bool ArePossiblyReducedInOtherBlock = any_of(Candidates, [](Value *V) {
return any_of(V->users(), [V](User *U) {
auto *Select = dyn_cast<SelectInst>(U);
return Select &&
Select->getParent() != cast<Instruction>(V)->getParent();
});
});
if (ArePossiblyReducedInOtherBlock)
return false;
return tryToVectorizeList(Candidates, R, MaxVFOnly);
},
/*MaxVFOnly=*/true, R);
return Changed;
}
/// Returns true if \p I is an instruction whose result the SLP vectorizer
/// cannot turn into a vector instruction directly, but whose operand chains
/// may still be worth vectorizing as bundle seeds.
static bool isNonVectorizableInst(const VectorizeOptions &Opts,
const Instruction *I,
const TargetLibraryInfo *TLI) {
if (const auto *CB = dyn_cast<CallBase>(I)) {
if (CB->isInlineAsm())
return false;
if (const auto *II = dyn_cast<IntrinsicInst>(CB)) {
if (II->isAssumeLikeIntrinsic())
return false;
if (isa<AnyMemIntrinsic>(II))
return false;
}
if (const auto *CI = dyn_cast<CallInst>(CB)) {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
if (isTriviallyVectorizable(ID))
return false;
if (!VFDatabase::getMappings(*CI).empty())
return false;
if (all_of(make_isa_range<Instruction>(CI->args()),
[](const Instruction *Arg) {
return Arg->getType()->isPointerTy();
}))
return false;
if (any_of(CI->args(), [](const Value *Arg) {
return Arg->getType()->isPointerTy();
}))
return false;
}
// Skip vector-returning calls in non-revec mode - we cannot turn their
// results into wider vectors here.
return Opts.slp_revec || !CB->getType()->isVectorTy();
}
if (isa<AtomicRMWInst, AtomicCmpXchgInst>(I))
return true;
if (const auto *EV = dyn_cast<ExtractValueInst>(I)) {
const auto *Arg = EV->getAggregateOperand();
if (const auto *CI = dyn_cast<CallInst>(Arg)) {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
if (isTriviallyVectorizable(ID))
return true;
if (!VFDatabase::getMappings(*CI).empty())
return true;
return false;
}
}
if (const auto *RI = dyn_cast<ReturnInst>(I))
return RI->getNumOperands() > 0 &&
(Opts.slp_revec || !I->getOperand(0)->getType()->isVectorTy()) &&
isa<Instruction>(I->getOperand(0));
return false;
}
/// Visits the value operands of \p I that are candidates for operand-chain
/// vectorization.
template <typename Func>
static void forEachOperandChainCandidate(const VectorizeOptions &Opts,
Instruction *I, Func F,
bool ForReduction) {
if (auto *CB = dyn_cast<CallBase>(I)) {
if (ForReduction && !Opts.slp_non_vectorizables_as_reductions &&
CB->arg_size() > 1)
return;
for (auto [Idx, U] : enumerate(CB->args()))
F(U.get(), Idx);
return;
}
if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
F(AI->getValOperand(), 0);
return;
}
if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
F(AI->getCompareOperand(), 0);
F(AI->getNewValOperand(), 1);
return;
}
if (auto *EV = dyn_cast<ExtractValueInst>(I)) {
F(EV->getAggregateOperand(), 0);
return;
}
if (ForReduction && !Opts.slp_non_vectorizables_as_reductions)
return;
if (auto *SI = dyn_cast<StoreInst>(I)) {
F(SI->getValueOperand(), 0);
return;
}
if (auto *RI = dyn_cast<ReturnInst>(I)) {
if (RI->getNumOperands() > 0)
F(RI->getReturnValue(), 0);
return;
}
llvm_unreachable("Unexpected instruction kind for operand-chain seeding");
}
template <typename ItT>
bool SLPVectorizerPass::vectorizeNonVectorizableInsts(
iterator_range<ItT> InstRange, BasicBlock *BB, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
SmallVector<Instruction *> Insts(InstRange);
if (Insts.empty())
return false;
stable_sort(Insts, [](const Instruction *A, const Instruction *B) {
return A->getOpcode() < B->getOpcode();
});
bool Changed = false;
// Pass 1 - try to find horizontal reductions feeding the root operands.
SmallPtrSet<Value *, 8> RootSeen;
for (Instruction *I : make_filter_range(
Insts, [&](Instruction *I) { return !R.isDeleted(I); })) {
bool RootDeleted = false;
forEachOperandChainCandidate(
*Opts, I,
[&](Value *Op, unsigned /*Position*/) {
if (RootDeleted)
return;
auto *RootOp = dyn_cast<Instruction>(Op);
if (!RootOp || RootOp->getParent() != BB || R.isDeleted(RootOp) ||
isa<ShuffleVectorInst>(RootOp) ||
!isValidElementType(RootOp->getType(), Opts->slp_revec))
return;
if (!RootSeen.insert(RootOp).second)
return;
Changed |=
vectorizeRootInstruction(nullptr, RootOp, BB, R, FMACandidates);
if (R.isDeleted(I))
RootDeleted = true;
},
/*ForReduction=*/true);
}
// Pass 2 - collect the operand instructions across all roots and try to
// vectorize them as bundles.
if (Insts.size() < 2)
return Changed;
struct OperandGroupKey {
enum class Kind : unsigned {
NonCall = 0,
Intrinsic,
NamedFunction,
IndirectCall,
};
Kind RootKind;
unsigned KindID; // Intrinsic ID for Intrinsic, opcode for NonCall,
// callee value-kind for IndirectCall, 0 for
// NamedFunction.
unsigned SubOp; // AtomicRMW operation; 0 otherwise.
StringRef FuncName; // Non-empty only for NamedFunction.
unsigned Position; // Operand slot within the root.
bool operator==(const OperandGroupKey &O) const {
return RootKind == O.RootKind && KindID == O.KindID && SubOp == O.SubOp &&
FuncName == O.FuncName && Position == O.Position;
}
bool operator!=(const OperandGroupKey &O) const { return !(*this == O); }
bool less(const OperandGroupKey &O) const {
if (RootKind != O.RootKind)
return static_cast<unsigned>(RootKind) <
static_cast<unsigned>(O.RootKind);
if (KindID != O.KindID)
return KindID < O.KindID;
if (SubOp != O.SubOp)
return SubOp < O.SubOp;
if (int C = FuncName.compare(O.FuncName))
return C < 0;
return Position < O.Position;
}
};
SmallDenseMap<Value *, OperandGroupKey> OpKeys;
auto BuildKey = [](Instruction *I, unsigned Position) -> OperandGroupKey {
if (auto *CB = dyn_cast<CallBase>(I)) {
if (auto *II = dyn_cast<IntrinsicInst>(CB))
return {OperandGroupKey::Kind::Intrinsic,
II->getIntrinsicID(),
0,
{},
Position};
if (Function *F = CB->getCalledFunction())
return {OperandGroupKey::Kind::NamedFunction, 0, 0, F->getName(),
Position};
return {OperandGroupKey::Kind::IndirectCall,
CB->getCalledOperand()->getValueID(),
0,
{},
Position};
}
unsigned SubOp = 0;
if (auto *AI = dyn_cast<AtomicRMWInst>(I))
SubOp = static_cast<unsigned>(AI->getOperation());
return {
OperandGroupKey::Kind::NonCall, I->getOpcode(), SubOp, {}, Position};
};
auto OperandSorter = [&OpKeys](Value *V1, Value *V2) -> bool {
if (V1 == V2)
return false;
const OperandGroupKey &K1 = OpKeys.at(V1);
const OperandGroupKey &K2 = OpKeys.at(V2);
if (K1 != K2)
return K1.less(K2);
auto *I1 = cast<Instruction>(V1);
auto *I2 = cast<Instruction>(V2);
if (I1->getType()->getTypeID() != I2->getType()->getTypeID())
return I1->getType()->getTypeID() < I2->getType()->getTypeID();
if (I1->getType()->getScalarSizeInBits() !=
I2->getType()->getScalarSizeInBits())
return I1->getType()->getScalarSizeInBits() <
I2->getType()->getScalarSizeInBits();
if (I1->getOpcode() != I2->getOpcode())
return I1->getOpcode() < I2->getOpcode();
return I1->comesBefore(I2);
};
auto AreCompatibleOperands = [&OpKeys](ArrayRef<Value *> VL,
Value *V) -> bool {
if (VL.empty() || VL.back() == V)
return true;
const OperandGroupKey &KBack = OpKeys.at(VL.back());
const OperandGroupKey &K = OpKeys.at(V);
if (KBack != K)
return false;
auto *I1 = cast<Instruction>(VL.back());
auto *I2 = cast<Instruction>(V);
return I1->getType() == I2->getType() && I1->getOpcode() == I2->getOpcode();
};
SmallVector<Value *> Operands;
SmallPtrSet<Value *, 8> Seen;
for (Instruction *I : make_filter_range(
Insts, [&](Instruction *I) { return !R.isDeleted(I); })) {
forEachOperandChainCandidate(
*Opts, I,
[&](Value *Op, unsigned Position) {
auto *OpI = dyn_cast<Instruction>(Op);
if (!OpI || OpI->getParent() != BB || R.isDeleted(OpI) ||
isa<ShuffleVectorInst>(OpI) ||
(!isValidElementType(OpI->getType(), Opts->slp_revec) &&
!isa<IntrinsicInst>(OpI)))
return;
if (!Seen.insert(OpI).second)
return;
OpKeys.try_emplace(OpI, BuildKey(I, Position));
Operands.push_back(OpI);
},
/*ForReduction=*/false);
}
if (Operands.size() <= 1)
return Changed;
Changed |= tryToVectorizeSequence<Value>(
*Opts, Operands, OperandSorter, AreCompatibleOperands,
[this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
constexpr unsigned Limit = 32;
// Limit to StandaloneSeeds if !MaxVFOnly to avoid quadratic scan for
// large set of candidates.
return tryToVectorizeList(Candidates, R, MaxVFOnly,
/*StandaloneSeeds=*/!MaxVFOnly &&
Candidates.size() >= Limit);
},
/*MaxVFOnly=*/true, R);
return Changed;
}
bool SLPVectorizerPass::vectorizeInserts(
InstSetVector &Instructions, BasicBlock *BB, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
assert(all_of(Instructions, IsaPred<InsertElementInst, InsertValueInst>) &&
"This function only accepts Insert instructions");
bool OpsChanged = false;
SmallVector<WeakTrackingVH> PostponedInsts;
// pass1 - try to match and vectorize a buildvector sequence for MaxVF only.
for (auto *I : make_filter_range(reverse(Instructions), [&](auto *I) {
return !R.isDeleted(I) && !isa<CmpInst>(I);
})) {
// The packs of the aggregate are vectorized together, not each alone.
if (auto *IVI = dyn_cast<InsertValueInst>(I);
IVI && vectorizePackedAggregate(IVI, R, *TTI, *DL)) {
OpsChanged = true;
continue;
}
if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) {
OpsChanged |=
vectorizeInsertValueInst(LastInsertValue, BB, R, /*MaxVFOnly=*/true);
} else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) {
OpsChanged |=
vectorizeInsertElementInst(LastInsertElem, BB, R, /*MaxVFOnly=*/true);
}
// pass2 - try to vectorize reductions only
if (R.isDeleted(I))
continue;
OpsChanged |= vectorizeHorReduction(nullptr, I, BB, R, PostponedInsts);
if (R.isDeleted(I) || isa<CmpInst>(I))
continue;
// pass3 - try to match and vectorize a buildvector sequence.
if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) {
OpsChanged |=
vectorizeInsertValueInst(LastInsertValue, BB, R, /*MaxVFOnly=*/false);
} else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) {
OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R,
/*MaxVFOnly=*/false);
}
}
// Now try to vectorize postponed instructions.
OpsChanged |= tryToVectorize(PostponedInsts, R, FMACandidates);
Instructions.clear();
return OpsChanged;
}
bool SLPVectorizerPass::vectorizeChainsInBlock(
BasicBlock *BB, BoUpSLP &R,
SmallSetVector<Instruction *, 8> &FMACandidates) {
bool Changed = false;
SmallVector<Value *, 4> Incoming;
SmallPtrSet<Value *, 16> VisitedInstrs;
// Only the original compares are the seeds. With revec the vectorized
// alternate compares are the pairs of compares of the same operands, which
// otherwise are vectorized again and again.
SmallPtrSet<const CmpInst *, 16> OrigCmps;
for (const CmpInst &CI : make_isa_range<CmpInst>(*BB))
OrigCmps.insert(&CI);
// Maps phi nodes to the non-phi nodes found in the use tree for each phi
// node. Allows better to identify the chains that can be vectorized in the
// better way.
DenseMap<Value *, SmallVector<Value *, 4>> PHIToOpcodes;
auto PHICompare = [this, &PHIToOpcodes](Value *V1, Value *V2) {
assert(isValidElementType(V1->getType(), Opts->slp_revec) &&
isValidElementType(V2->getType(), Opts->slp_revec) &&
"Expected vectorizable types only.");
if (V1 == V2)
return false;
// It is fine to compare type IDs here, since we expect only vectorizable
// types, like ints, floats and pointers, we don't care about other type.
if (V1->getType()->getTypeID() < V2->getType()->getTypeID())
return true;
if (V1->getType()->getTypeID() > V2->getType()->getTypeID())
return false;
if (V1->getType()->getScalarSizeInBits() <
V2->getType()->getScalarSizeInBits())
return true;
if (V1->getType()->getScalarSizeInBits() >
V2->getType()->getScalarSizeInBits())
return false;
ArrayRef<Value *> Opcodes1 = PHIToOpcodes[V1];
ArrayRef<Value *> Opcodes2 = PHIToOpcodes[V2];
if (Opcodes1.size() < Opcodes2.size())
return true;
if (Opcodes1.size() > Opcodes2.size())
return false;
for (int I = 0, E = Opcodes1.size(); I < E; ++I) {
{
// Instructions come first.
auto *I1 = dyn_cast<Instruction>(Opcodes1[I]);
auto *I2 = dyn_cast<Instruction>(Opcodes2[I]);
if (I1 && I2) {
DomTreeNodeBase<BasicBlock> *NodeI1 = DT->getNode(I1->getParent());
DomTreeNodeBase<BasicBlock> *NodeI2 = DT->getNode(I2->getParent());
if (!NodeI1)
return NodeI2 != nullptr;
if (!NodeI2)
return false;
assert((NodeI1 == NodeI2) ==
(NodeI1->getDFSNumIn() == NodeI2->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
InstructionsState S = getSameOpcode({I1, I2}, *TLI);
if (S && !S.isAltShuffle() && I1->getOpcode() == I2->getOpcode()) {
const auto *E1 = dyn_cast<ExtractElementInst>(I1);
const auto *E2 = dyn_cast<ExtractElementInst>(I2);
if (!E1 || !E2)
continue;
// Sort on ExtractElementInsts primarily by vector operands. Prefer
// program order of the vector operands.
const auto *V1 = dyn_cast<Instruction>(E1->getVectorOperand());
const auto *V2 = dyn_cast<Instruction>(E2->getVectorOperand());
if (V1 != V2) {
if (V1 && !V2)
return true;
if (!V1 && V2)
return false;
DomTreeNodeBase<BasicBlock> *NodeI1 =
DT->getNode(V1->getParent());
DomTreeNodeBase<BasicBlock> *NodeI2 =
DT->getNode(V2->getParent());
if (!NodeI1)
return NodeI2 != nullptr;
if (!NodeI2)
return false;
assert((NodeI1 == NodeI2) ==
(NodeI1->getDFSNumIn() == NodeI2->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
return V1->comesBefore(V2);
}
// If we have the same vector operand, try to sort by constant
// index.
std::optional<unsigned> Id1 = getExtractIndex(E1);
std::optional<unsigned> Id2 = getExtractIndex(E2);
// Bring constants to the top
if (Id1 && !Id2)
return true;
if (!Id1 && Id2)
return false;
// First elements come first.
if (Id1 && Id2)
return *Id1 < *Id2;
continue;
}
if (I1->getOpcode() == I2->getOpcode())
continue;
return I1->getOpcode() < I2->getOpcode();
}
if (I1)
return true;
if (I2)
return false;
}
{
// Non-undef constants come next.
bool C1 = isa<Constant>(Opcodes1[I]) && !isa<UndefValue>(Opcodes1[I]);
bool C2 = isa<Constant>(Opcodes2[I]) && !isa<UndefValue>(Opcodes2[I]);
if (C1 && C2)
continue;
if (C1)
return true;
if (C2)
return false;
}
bool U1 = isa<UndefValue>(Opcodes1[I]);
bool U2 = isa<UndefValue>(Opcodes2[I]);
{
// Non-constant non-instructions come next.
if (!U1 && !U2) {
auto ValID1 = Opcodes1[I]->getValueID();
auto ValID2 = Opcodes2[I]->getValueID();
if (ValID1 == ValID2)
continue;
if (ValID1 < ValID2)
return true;
if (ValID1 > ValID2)
return false;
}
if (!U1)
return true;
if (!U2)
return false;
}
// Undefs come last.
assert(U1 && U2 && "The only thing left should be undef & undef.");
}
return false;
};
auto AreCompatiblePHIs = [&PHIToOpcodes, this, &R](ArrayRef<Value *> VL,
Value *V1) {
if (VL.empty() || V1 == VL.back())
return true;
Value *V2 = VL.back();
if (V1->getType() != V2->getType())
return false;
ArrayRef<Value *> Opcodes1 = PHIToOpcodes[V1];
ArrayRef<Value *> Opcodes2 = PHIToOpcodes[V2];
if (Opcodes1.size() != Opcodes2.size())
return false;
for (int I = 0, E = Opcodes1.size(); I < E; ++I) {
// Undefs are compatible with any other value.
if (isa<UndefValue>(Opcodes1[I]) || isa<UndefValue>(Opcodes2[I]))
continue;
if (auto *I1 = dyn_cast<Instruction>(Opcodes1[I]))
if (auto *I2 = dyn_cast<Instruction>(Opcodes2[I])) {
if (R.isDeleted(I1) || R.isDeleted(I2))
return false;
if (I1->getParent() != I2->getParent())
return false;
if (getSameOpcode({I1, I2}, *TLI))
continue;
return false;
}
if (isa<Constant>(Opcodes1[I]) && isa<Constant>(Opcodes2[I]))
continue;
if (Opcodes1[I]->getValueID() != Opcodes2[I]->getValueID())
return false;
}
return true;
};
bool HaveVectorizedPhiNodes = false;
do {
// Collect the incoming values from the PHIs.
Incoming.clear();
for (Instruction &I : *BB) {
auto *P = dyn_cast<PHINode>(&I);
if (!P || P->getNumIncomingValues() > MaxPHINumOperands)
break;
// No need to analyze deleted, vectorized and non-vectorizable
// instructions.
if (!VisitedInstrs.count(P) && !R.isDeleted(P) &&
isValidElementType(P->getType(), Opts->slp_revec))
Incoming.push_back(P);
}
if (Incoming.size() <= 1)
break;
// Find the corresponding non-phi nodes for better matching when trying to
// build the tree.
for (Value *V : Incoming) {
SmallVectorImpl<Value *> &Opcodes =
PHIToOpcodes.try_emplace(V).first->getSecond();
if (!Opcodes.empty())
continue;
SmallVector<Value *, 4> Nodes(1, V);
SmallPtrSet<Value *, 4> Visited;
while (!Nodes.empty()) {
auto *PHI = cast<PHINode>(Nodes.pop_back_val());
if (!Visited.insert(PHI).second)
continue;
for (Value *V : PHI->incoming_values()) {
if (auto *PHI1 = dyn_cast<PHINode>((V))) {
Nodes.push_back(PHI1);
continue;
}
Opcodes.emplace_back(V);
}
}
}
HaveVectorizedPhiNodes = tryToVectorizeSequence<Value>(
*Opts, Incoming, PHICompare, AreCompatiblePHIs,
[this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
return tryToVectorizeList(Candidates, R, MaxVFOnly);
},
/*MaxVFOnly=*/true, R);
Changed |= HaveVectorizedPhiNodes;
if (HaveVectorizedPhiNodes && any_of(PHIToOpcodes, [&](const auto &P) {
auto *PHI = dyn_cast<PHINode>(P.first);
return !PHI || R.isDeleted(PHI);
}))
PHIToOpcodes.clear();
VisitedInstrs.insert_range(Incoming);
} while (HaveVectorizedPhiNodes);
VisitedInstrs.clear();
InstSetVector PostProcessInserts;
SmallSetVector<CmpInst *, 8> PostProcessCmps;
// Non-vectorizable root instructions other than stores: calls
// (regular and intrinsic), invokes, callbrs, atomic RMW/cmpxchg, and
// returns.
SmallSetVector<Instruction *, 8> PostProcessInsts;
// Stores are processed after all other instructions/roots.
SmallSetVector<StoreInst *, 8> PostProcessStores;
SmallSetVector<Instruction *, 8> PoorThroughputSeeds;
PoorThroughputOpCache PoorThroughputCache;
auto VectorizeInsertsAndCmps = [&](bool AtTerminator) {
bool Changed = vectorizeInserts(PostProcessInserts, BB, R, FMACandidates);
if (AtTerminator) {
Changed |=
vectorizeCmpInsts(reverse(PostProcessCmps), BB, R, FMACandidates);
PostProcessCmps.clear();
if (!PostProcessInsts.empty())
Changed |= vectorizeNonVectorizableInsts(reverse(PostProcessInsts), BB,
R, FMACandidates);
PostProcessInsts.clear();
}
PostProcessInserts.clear();
return Changed;
};
// Returns true if `I` is in any of the post-process sets.
auto IsInPostProcessInstrs = [&](Instruction *I) {
if (auto *Cmp = dyn_cast<CmpInst>(I))
return PostProcessCmps.contains(Cmp);
if (PostProcessInsts.contains(I))
return true;
if (auto *SI = dyn_cast<StoreInst>(I))
return PostProcessStores.contains(SI);
return isa<InsertElementInst, InsertValueInst>(I) &&
PostProcessInserts.contains(I);
};
// Returns true if `I` is an instruction without users, like terminator, or
// function call with ignored return value, store. Ignore unused instructions
// (basing on instruction type, except for CallInst and InvokeInst).
auto HasNoUsers = [](Instruction *I) {
return I->use_empty() &&
(I->getType()->isVoidTy() || isa<CallInst, InvokeInst>(I));
};
// The number of the stores to each underlying object, which may start the
// reduction. Vectorized stores and stores of loads or non-instruction values
// never start it.
SmallDenseMap<Value *, unsigned> NumRdxRootStores;
for (const auto &[Obj, SIs] : Stores)
NumRdxRootStores[Obj] = count_if(SIs, [&](StoreInst *SI) {
return !R.isDeleted(SI) && isa<Instruction>(SI->getValueOperand()) &&
!isa<LoadInst>(SI->getValueOperand());
});
for (BasicBlock::iterator It = BB->begin(), E = BB->end(); It != E; ++It) {
// Skip instructions with scalable type. The num of elements is unknown at
// compile-time for scalable type.
if (isa<ScalableVectorType>(It->getType()))
continue;
// Skip instructions marked for the deletion.
if (R.isDeleted(&*It))
continue;
// We may go through BB multiple times so skip the one we have checked.
if (!VisitedInstrs.insert(&*It).second) {
if (HasNoUsers(&*It) &&
VectorizeInsertsAndCmps(/*AtTerminator=*/It->isTerminator())) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
Changed = true;
It = BB->begin();
E = BB->end();
}
continue;
}
// Try to vectorize reductions that use PHINodes.
if (PHINode *P = dyn_cast<PHINode>(It)) {
// Check that the PHI is a reduction PHI.
if (P->getNumIncomingValues() == 2) {
// Try to match and vectorize a horizontal reduction.
Instruction *Root = getReductionInstr(DT, P, BB, LI);
if (Root && vectorizeRootInstruction(P, Root, BB, R, FMACandidates)) {
Changed = true;
It = BB->begin();
E = BB->end();
continue;
}
}
// Try to vectorize the incoming values of the PHI, to catch reductions
// that feed into PHIs.
for (unsigned I : seq<unsigned>(P->getNumIncomingValues())) {
// Skip if the incoming block is the current BB for now. Also, bypass
// unreachable IR for efficiency and to avoid crashing.
// TODO: Collect the skipped incoming values and try to vectorize them
// after processing BB.
if (BB == P->getIncomingBlock(I) ||
!DT->isReachableFromEntry(P->getIncomingBlock(I)))
continue;
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *PI = dyn_cast<Instruction>(P->getIncomingValue(I));
PI && !IsInPostProcessInstrs(PI)) {
bool Res = vectorizeRootInstruction(
nullptr, PI, P->getIncomingBlock(I), R, FMACandidates);
Changed |= Res;
if (Res && R.isDeleted(P)) {
It = BB->begin();
E = BB->end();
break;
}
}
}
continue;
}
if (HasNoUsers(&*It)) {
bool OpsChanged = false;
auto *SI = dyn_cast<StoreInst>(It);
bool TryToVectorizeRoot = Opts->slp_vectorize_hor_store || !SI;
if (SI) {
// Try to vectorize chain in store, if at most one store to the address
// in the block may start the reduction.
// TODO: This is just a temporarily solution to save compile time. Need
// to investigate if we can safely turn on slp-vectorize-hor-store
// instead to allow lookup for reduction chains in all non-vectorized
// stores (need to check side effects and compile time).
TryToVectorizeRoot |= NumRdxRootStores.lookup(getUnderlyingObject(
SI->getPointerOperand())) <= 1 &&
SI->getValueOperand()->hasOneUse();
}
if (TryToVectorizeRoot) {
for (auto *V : It->operand_values()) {
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *VI = dyn_cast<Instruction>(V);
VI && !IsInPostProcessInstrs(VI))
// Try to match and vectorize a horizontal reduction.
OpsChanged |=
vectorizeRootInstruction(nullptr, VI, BB, R, FMACandidates);
}
}
// Start vectorization of post-process list of instructions from the
// top-tree instructions to try to vectorize as many instructions as
// possible.
OpsChanged |=
VectorizeInsertsAndCmps(/*AtTerminator=*/It->isTerminator());
if (OpsChanged) {
// We would like to start over since some instructions are deleted
// and the iterator may become invalid value.
Changed = true;
It = BB->begin();
E = BB->end();
continue;
}
}
// The values, feeding only the assumes, and the instructions, producing no
// code, are dropped before the codegen, the tree is rejected for them
// anyway.
if (isAssumeLikeIntrinsic(&*It) || isa<PseudoProbeInst>(It) ||
R.isEphemeralValue(&*It))
continue;
if (isa<InsertElementInst, InsertValueInst>(It))
PostProcessInserts.insert(&*It);
else if (auto *CI = dyn_cast<CmpInst>(It)) {
if (OrigCmps.contains(CI))
PostProcessCmps.insert(CI);
} else if (auto *SI = dyn_cast<StoreInst>(It);
SI &&
(Opts->slp_revec ||
!SI->getValueOperand()->getType()->isVectorTy()) &&
isa<Instruction>(SI->getValueOperand()))
PostProcessStores.insert(SI);
else if (isNonVectorizableInst(*Opts, &*It, TLI))
PostProcessInsts.insert(&*It);
else if (Opts->slp_vectorize_poor_throughput &&
isPoorThroughputOp(*Opts, &*It, *TTI, *TLI, PoorThroughputCache,
R.getCostKind(), R.getMaxVecRegSize()))
PoorThroughputSeeds.insert(&*It);
}
// Late post-process: run operand-chain vectorization for stores.
if (!PostProcessStores.empty() &&
(Opts->slp_non_vectorizables_as_reductions ||
PostProcessStores.size() >= 2)) {
bool TryVectorize = true;
if (!Opts->slp_postprocess_stores_operands && R.getCostThreshold() >= 0) {
// Use pessimistic cost estimation to avoid long compile time when there
// are many stores in the list.
Type *ScalarTy = getValueType(PostProcessStores.front(), Opts->slp_revec);
if (!::isValidElementType(ScalarTy, Opts->slp_revec)) {
TryVectorize = false;
} else {
auto *IF =
dyn_cast<Instruction>(PostProcessStores.front()->getValueOperand());
auto *IB =
dyn_cast<Instruction>(PostProcessStores.back()->getValueOperand());
if (!Opts->slp_non_vectorizables_as_reductions &&
PostProcessStores.size() == 2 &&
(!IF || !IB || IF->getOpcode() != IB->getOpcode())) {
TryVectorize = false;
} else {
ScalarTy = IntegerType::get(
ScalarTy->getContext(),
DL->getTypeSizeInBits(ScalarTy->getScalarType()));
if (auto *ValTy = dyn_cast<VectorType>(
PostProcessStores.front()->getValueOperand()->getType()))
ScalarTy = ::getWidenedType(ScalarTy, getNumElements(ValTy));
auto *VecTy = cast<VectorType>(
::getWidenedType(ScalarTy, PostProcessStores.size()));
InstructionCost ExtractsCost = getScalarizationOverhead(
*TTI, Opts->slp_revec, ScalarTy, VecTy,
APInt::getAllOnes(PostProcessStores.size()),
/*Insert=*/false, /*Extract=*/true, R.getCostKind(),
/*ForPoisonSrc=*/true, {}, TTI::VectorInstrContext::Store);
TryVectorize = ExtractsCost <= PostProcessStores.size() + 1;
}
}
}
if (TryVectorize) {
Changed |= vectorizeNonVectorizableInsts(reverse(PostProcessStores), BB,
R, FMACandidates);
}
}
Changed |= vectorizePoorThroughputSeeds(PoorThroughputSeeds.getArrayRef(), R);
return Changed;
}
bool SLPVectorizerPass::vectorizePoorThroughputSeeds(
ArrayRef<Instruction *> Candidates, BoUpSLP &R,
function_ref<bool(ArrayRef<Value *>)> Filter) {
if (Candidates.size() < 2)
return false;
SmallVector<Value *> Seeds;
SmallDenseMap<Value *, SeedGroupKey> SeedKeys;
for (Instruction *I : make_filter_range(Candidates, [&](Instruction *I) {
return !R.isDeleted(I) &&
isValidElementType(getValueType(I, Opts->slp_revec),
Opts->slp_revec) &&
!R.hasResolvedUser(I);
})) {
SeedKeys.try_emplace(I, getSeedGroupKey(I, *TLI));
Seeds.push_back(I);
}
return vectorizeSeeds(
Seeds,
[&](Value *V1, Value *V2) {
return SeedKeys.at(V1).less(SeedKeys.at(V2));
},
R, Filter);
}
// Seed the groups with scalars not stored locally (e.g. the store is sunk to
// the join block) before the store chains split them into a remainder too
// small to vectorize.
bool SLPVectorizerPass::vectorizeSplitStorePoorThroughputSeeds(BasicBlock *BB,
BoUpSLP &R) {
if (!Opts->slp_vectorize_poor_throughput || Stores.empty())
return false;
auto HasLocalStoreUser = [&](Value *V) {
return any_of(V->users(), [&](User *U) {
auto *SI = dyn_cast<StoreInst>(U);
return SI && SI->isSimple() && SI->getParent() == BB &&
SI->getValueOperand() == V;
});
};
auto IsCandidate = [&](Instruction &I, PoorThroughputOpCache &Cache) {
return !R.isDeleted(&I) && !isAssumeLikeIntrinsic(&I) &&
!isa<PseudoProbeInst>(I) && !R.isEphemeralValue(&I) &&
!isNonVectorizableInst(*Opts, &I, TLI) &&
isPoorThroughputOp(*Opts, &I, *TTI, *TLI, Cache, R.getCostKind(),
R.getMaxVecRegSize());
};
// The groups without the stored scalars are rejected, so the block is
// skipped if no stored scalar is a candidate. The stored scalars are checked
// in the block order, a scalar rejected here is rejected by the full scan.
PoorThroughputOpCache StoredCache;
if (none_of(*BB, [&](Instruction &I) {
return HasLocalStoreUser(&I) && IsCandidate(I, StoredCache);
}))
return false;
PoorThroughputOpCache PoorThroughputCache;
SmallVector<Instruction *> Candidates;
for (Instruction &I : *BB)
if (IsCandidate(I, PoorThroughputCache))
Candidates.push_back(&I);
return vectorizePoorThroughputSeeds(Candidates, R, [&](ArrayRef<Value *> VL) {
// The scalars not stored locally must widen the stored scalars, left by
// the store chains after the full vectors.
unsigned MaxVF = bit_floor(R.getMaxVecRegSize() /
DL->getTypeSizeInBits(VL.front()->getType()));
unsigned NumStored = count_if(VL, HasLocalStoreUser);
unsigned Rem = MaxVF ? NumStored % MaxVF : 0;
return Rem != 0 && bit_floor(Rem) < bit_floor(VL.size() - NumStored + Rem);
});
}
bool SLPVectorizerPass::vectorizeSeeds(
SmallVectorImpl<Value *> &Seeds,
function_ref<bool(Value *, Value *)> IsLessGroup, BoUpSLP &R,
function_ref<bool(ArrayRef<Value *>)> Filter) {
if (Seeds.size() < 2)
return false;
auto SeedSorter = [&](Value *V1, Value *V2) {
if (V1 == V2)
return false;
auto *I1 = cast<Instruction>(V1);
auto *I2 = cast<Instruction>(V2);
Type *T1 = I1->getType();
Type *T2 = I2->getType();
if (T1->getTypeID() != T2->getTypeID())
return T1->getTypeID() < T2->getTypeID();
if (T1->getScalarSizeInBits() != T2->getScalarSizeInBits())
return T1->getScalarSizeInBits() < T2->getScalarSizeInBits();
if (IsLessGroup(V1, V2))
return true;
if (IsLessGroup(V2, V1))
return false;
return I1->comesBefore(I2);
};
auto AreCompatibleSeeds = [&](ArrayRef<Value *> VL, Value *V) {
if (VL.empty() || VL.back() == V)
return true;
return cast<Instruction>(VL.back())->getType() ==
cast<Instruction>(V)->getType() &&
!IsLessGroup(VL.back(), V) && !IsLessGroup(V, VL.back());
};
return tryToVectorizeSequence<Value>(
*Opts, Seeds, SeedSorter, AreCompatibleSeeds,
[this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
// The same list may be retried for the different groups of the same
// type, the earlier rejection is still valid for the block.
if (R.isAnalyzedBundle(Candidates))
return false;
if (tryToVectorizeList(Candidates, R, MaxVFOnly,
/*StandaloneSeeds=*/true))
return true;
R.analyzedBundle(Candidates);
return false;
},
/*MaxVFOnly=*/false, R, Filter);
}
bool SLPVectorizerPass::vectorizeOnceUsedSeeds(BasicBlock *BB, BoUpSLP &R) {
// Loads are not seeds, their subkeys do not affect the grouping.
auto GenerateLoadsSubkey = [](size_t, LoadInst *LI) {
return hash_value(LI->getPointerOperand());
};
SmallVector<Value *> Seeds;
SmallDenseMap<std::pair<size_t, size_t>, unsigned> KeyToGroup;
SmallDenseMap<Value *, unsigned> SeedGroups;
PoorThroughputOpCache PoorThroughputCache;
for (Instruction &I : make_filter_range(*BB, [&](Instruction &I) {
return !R.isDeleted(&I) && I.hasOneUse() && !R.isEphemeralValue(&I) &&
!R.isVectorized(&I) && !R.isAnalyzedScalar(&I) &&
isValidElementType(getValueType(&I, Opts->slp_revec),
Opts->slp_revec) &&
isOnceUsedSeed(&I) && !isNonVectorizableInst(*Opts, &I, TLI) &&
!R.hasResolvedUser(&I);
})) {
// The poor-throughput ops are seeded on their own, with the different
// grouping.
if (Opts->slp_vectorize_poor_throughput &&
isPoorThroughputOp(*Opts, &I, *TTI, *TLI, PoorThroughputCache,
R.getCostKind(), R.getMaxVecRegSize()))
continue;
// The multiplication is contracted into the scalar FMA with its user, the
// vector node breaks the contraction.
if (I.getOpcode() == Instruction::FMul) {
auto *U = cast<Instruction>(I.user_back());
if (InstructionsState S = getSameOpcode(U, *TLI);
S && S.isAddSubLikeOp() &&
canConvertToFMA(U, S, *DT, *DL, *TTI, *TLI, R).isValid())
continue;
}
// The keys are hashes, so the groups are numbered by the first seed to
// keep the order deterministic.
std::pair<size_t, size_t> Key = generateKeySubkey(
&I, TLI, GenerateLoadsSubkey, /*AllowAlternate=*/false);
SeedGroups.try_emplace(
&I, KeyToGroup.try_emplace(Key, KeyToGroup.size()).first->second);
Seeds.push_back(&I);
}
return vectorizeSeeds(
Seeds,
[&](Value *V1, Value *V2) {
return SeedGroups.at(V1) < SeedGroups.at(V2);
},
R);
}
bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
auto Changed = false;
const Loop *L = LI->getLoopFor(BB);
auto IsAffineInLoop = [L](const SCEV *S) {
const auto *AddRec = dyn_cast<SCEVAddRecExpr>(S);
return AddRec && AddRec->getLoop() == L && AddRec->isAffine();
};
// If the getelementptr list has fewer than two elements, there's nothing
// to do.
for (auto &Entry : make_filter_range(
GEPs, [](auto &Entry) { return Entry.second.size() >= 2; })) {
LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n");
// Process the GEP list in chunks suitable for the target's supported
// vector size. If a vector register can't hold 1 element, we are done. We
// are trying to vectorize the index computations, so the maximum number of
// elements is based on the size of the index expression, rather than the
// size of the GEP itself (the target's pointer size).
auto *It = find_if(Entry.second, [&](GetElementPtrInst *GEP) {
return !R.isDeleted(GEP);
});
if (It == Entry.second.end())
continue;
unsigned MaxVecRegSize = R.getMaxVecRegSize();
unsigned EltSize = R.getVectorElementSize(*(*It)->idx_begin());
if (MaxVecRegSize < EltSize)
continue;
unsigned MaxElts = MaxVecRegSize / EltSize;
for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += MaxElts) {
auto Len = std::min<unsigned>(BE - BI, MaxElts);
ArrayRef<GetElementPtrInst *> GEPList(&Entry.second[BI], Len);
// Initialize a set a candidate getelementptrs. Note that we use a
// SetVector here to preserve program order. If the index computations
// are vectorizable and begin with loads, we want to minimize the chance
// of having to reorder them later.
SetVector<Value *> Candidates(llvm::from_range, GEPList);
// Some of the candidates may have already been vectorized after we
// initially collected them or their index is optimized to constant value.
// If so, they are marked as deleted, so remove them from the set of
// candidates.
Candidates.remove_if([&R](Value *I) {
return R.isDeleted(cast<Instruction>(I)) ||
isa<Constant>(cast<GetElementPtrInst>(I)->idx_begin()->get());
});
// Remove from the set of candidates all pairs of getelementptrs with
// constant differences. Such getelementptrs are likely not good
// candidates for vectorization in a bottom-up phase since one can be
// computed from the other. The same holds for affine recurrences of the
// enclosing loop that differ by a loop-invariant amount: loop strength
// reduction turns them into pointer increments. We also ensure all
// candidate getelementptr indices are unique.
for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
auto *GEPI = GEPList[I];
if (!Candidates.count(GEPI))
continue;
const SCEV *SCEVI = SE->getSCEV(GEPList[I]);
bool IsAffineI = IsAffineInLoop(SCEVI);
for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
auto *GEPJ = GEPList[J];
if (!Candidates.count(GEPJ))
continue;
const SCEV *SCEVJ = SE->getSCEV(GEPList[J]);
const SCEV *Diff = SE->getMinusSCEV(SCEVI, SCEVJ);
if (isa<SCEVConstant>(Diff) || (IsAffineI && IsAffineInLoop(SCEVJ) &&
SE->isLoopInvariant(Diff, L))) {
Candidates.remove(GEPI);
Candidates.remove(GEPJ);
} else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
Candidates.remove(GEPJ);
}
}
}
// We break out of the above computation as soon as we know there are
// fewer than two candidates remaining.
if (Candidates.size() < 2)
continue;
// Add the single, non-constant index of each candidate to the bundle. We
// ensured the indices met these constraints when we originally collected
// the getelementptrs.
SmallVector<Value *, 16> Bundle(Candidates.size());
auto BundleIndex = 0u;
for (auto *V : Candidates) {
auto *GEP = cast<GetElementPtrInst>(V);
auto *GEPIdx = GEP->idx_begin()->get();
assert(GEP->getNumIndices() == 1 && !isa<Constant>(GEPIdx));
Bundle[BundleIndex++] = GEPIdx;
}
// Try and vectorize the indices. We are currently only interested in
// gather-like cases of the form:
//
// ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
//
// where the loads of "a", the loads of "b", and the subtractions can be
// performed in parallel. It's likely that detecting this pattern in a
// bottom-up phase will be simpler and less costly than building a
// full-blown top-down phase beginning at the consecutive loads.
Changed |= tryToVectorizeList(Bundle, R);
}
}
return Changed;
}
bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
bool Changed = false;
// Sort by type, base pointers and values operand. Value operands must be
// compatible (have the same opcode, same parent), otherwise it is
// definitely not profitable to try to vectorize them.
auto &&StoreSorter = [this](StoreInst *V, StoreInst *V2) {
if (V->getValueOperand()->getType()->getTypeID() <
V2->getValueOperand()->getType()->getTypeID())
return true;
if (V->getValueOperand()->getType()->getTypeID() >
V2->getValueOperand()->getType()->getTypeID())
return false;
if (V->getPointerOperandType()->getTypeID() <
V2->getPointerOperandType()->getTypeID())
return true;
if (V->getPointerOperandType()->getTypeID() >
V2->getPointerOperandType()->getTypeID())
return false;
if (V->getValueOperand()->getType()->getScalarSizeInBits() <
V2->getValueOperand()->getType()->getScalarSizeInBits())
return true;
if (V->getValueOperand()->getType()->getScalarSizeInBits() >
V2->getValueOperand()->getType()->getScalarSizeInBits())
return false;
// UndefValues are compatible with all other values.
auto *I1 = dyn_cast<Instruction>(V->getValueOperand());
auto *I2 = dyn_cast<Instruction>(V2->getValueOperand());
if (I1 && I2) {
DomTreeNodeBase<llvm::BasicBlock> *NodeI1 = DT->getNode(I1->getParent());
DomTreeNodeBase<llvm::BasicBlock> *NodeI2 = DT->getNode(I2->getParent());
assert(NodeI1 && "Should only process reachable instructions");
assert(NodeI2 && "Should only process reachable instructions");
assert((NodeI1 == NodeI2) ==
(NodeI1->getDFSNumIn() == NodeI2->getDFSNumIn()) &&
"Different nodes should have different DFS numbers");
if (NodeI1 != NodeI2)
return NodeI1->getDFSNumIn() < NodeI2->getDFSNumIn();
return I1->getOpcode() < I2->getOpcode();
}
if (I1 && !I2)
return true;
if (!I1 && I2)
return false;
return V->getValueOperand()->getValueID() <
V2->getValueOperand()->getValueID();
};
bool SameParent = true;
auto AreCompatibleStores = [&](ArrayRef<StoreInst *> VL, StoreInst *V1) {
if (VL.empty()) {
SameParent = true;
return true;
}
StoreInst *V2 = VL.back();
if (V1 == V2)
return true;
if (V1->getValueOperand()->getType() != V2->getValueOperand()->getType())
return false;
if (V1->getPointerOperandType() != V2->getPointerOperandType())
return false;
// Undefs are compatible with any other value.
if (isa<UndefValue>(V1->getValueOperand()) ||
isa<UndefValue>(V2->getValueOperand()))
return true;
if (isa<Constant>(V1->getValueOperand()) &&
isa<Constant>(V2->getValueOperand()))
return true;
// Check if the operands of the stores can be vectorized. They can be
// vectorized, if they have compatible operands or have operands, which can
// be vectorized as copyables.
auto *I1 = dyn_cast<Instruction>(V1->getValueOperand());
auto *I2 = dyn_cast<Instruction>(V2->getValueOperand());
if (I1 || I2) {
// Accept only tail-following non-compatible values for now.
// TODO: investigate if it is possible to vectorize incompatible values,
// if the copyables are first in the list.
if (I1 && !I2)
return false;
SameParent &= I1 && I2 && I1->getParent() == I2->getParent();
SmallVector<Value *> NewVL(VL.size() + 1);
for (auto [SI, V] : zip(VL, NewVL))
V = SI->getValueOperand();
NewVL.back() = V1->getValueOperand();
InstructionsCompatibilityAnalysis Analysis(*DT, *DL, *TTI, *TLI);
InstructionsState S = Analysis.buildInstructionsState(
NewVL, R, /*WithProfitabilityCheck=*/true,
/*SkipSameCodeCheck=*/!SameParent);
// A copyable GEP node with non-constant indices is vectorizable only as
// the address vector of a masked gather, not as a stored value.
if (S && S.getOpcode() == Instruction::GetElementPtr &&
S.areInstructionsWithCopyableElements() &&
any_of(NewVL, [&](Value *V) {
auto *GEP = dyn_cast<GetElementPtrInst>(V);
return GEP && !S.isCopyableElement(V) &&
!isa<ConstantInt>(GEP->getOperand(1));
}))
return false;
if (S)
return true;
if (!SameParent)
return false;
}
return V1->getValueOperand()->getValueID() ==
V2->getValueOperand()->getValueID();
};
// Returns true if any two stores in the group are at consecutive (stride-1)
// addresses. Masked stores are disabled for the entire base-object group when
// this is true - consecutive vectorization is always preferred.
auto HasConsecutiveStoresOrSameAddress = [&](ArrayRef<StoreInst *> SIs) {
if (SIs.size() < 2)
return false;
Type *Ty = SIs.front()->getValueOperand()->getType();
Value *Base = SIs.front()->getPointerOperand();
SmallVector<int64_t> Offsets;
Offsets.reserve(SIs.size());
for (StoreInst *SI : SIs) {
if (std::optional<int64_t> Off =
getPointersDiff(Ty, Base, Ty, SI->getPointerOperand(), *DL, *SE))
Offsets.push_back(*Off);
}
sort(Offsets);
return any_of(seq<size_t>(1, Offsets.size()), [&](size_t I) {
return Offsets[I] - Offsets[I - 1] == 1 || Offsets[I] == Offsets[I - 1];
});
};
// Attempt to sort and vectorize each of the store-groups.
DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>> Attempted;
// The longer groups go first: the shorter ones sharing the scalars with them
// then reuse their vectors instead of splitting the longer chains.
SmallVector<StoreListMap::value_type *> Groups(make_pointer_range(Stores));
stable_sort(Groups, [](const auto *A, const auto *B) {
return A->second.size() > B->second.size();
});
auto TryToVectorizeGroup = [&](StoreListMap::value_type &Pair) {
LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length "
<< Pair.second.size() << ".\n");
if (!isValidElementType(Pair.second.front()->getValueOperand()->getType(),
Opts->slp_revec))
return false;
// Masked stores are only attempted when no consecutive pair exists in the
// full base-object group.
const bool AllowMaskedStores =
Opts->slp_enable_masked_stores &&
!HasConsecutiveStoresOrSameAddress(Pair.second);
// Reverse stores to do bottom-to-top analysis. This is important if the
// values are stores to the same addresses several times, in this case need
// to follow the stores order (reversed to meet the memory dependecies).
SmallVector<StoreInst *> ReversedStores(Pair.second.rbegin(),
Pair.second.rend());
return tryToVectorizeSequence<StoreInst>(
*Opts, ReversedStores, StoreSorter, AreCompatibleStores,
[&](ArrayRef<StoreInst *> Candidates, bool) {
return vectorizeStores(Candidates, R, Attempted, AllowMaskedStores);
},
/*MaxVFOnly=*/false, R);
};
// The groups, rejected before the vectorized one, were analyzed for the
// original scalars and may become profitable now, so they are retried once.
SmallVector<StoreListMap::value_type *> Rejected;
unsigned NumToRetry = 0;
for (auto &Pair :
make_filter_range(make_pointee_range(Groups),
[](auto &Pair) { return Pair.second.size() >= 2; })) {
if (TryToVectorizeGroup(Pair)) {
Changed = true;
NumToRetry = Rejected.size();
} else {
Rejected.push_back(&Pair);
}
}
Attempted.clear();
for (auto *Pair : ArrayRef(Rejected).take_front(NumToRetry))
Changed |= TryToVectorizeGroup(*Pair);
return Changed;
}