[LV] Prevent interleaving if computeMaxVF returned None.
As discussed in D57382, interleaving should be avoided if computeMaxVF
returns None, same as we currently do for vectorization.
Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6477
Reviewers: Ayal, dcaballe, hsaito, mkuper, rengolin
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D57837
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@353461 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 7e9fee8..7264eb9 100644
--- a/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -171,6 +171,9 @@
unsigned Width;
// Cost of the loop with that width
unsigned Cost;
+
+ // Width 1 means no vectorization, cost 0 means uncomputed cost.
+ static VectorizationFactor Disabled() { return {1, 0}; }
};
/// Planner drives the vectorization process after having passed
@@ -221,8 +224,9 @@
LoopVectorizationCostModel &CM)
: OrigLoop(L), LI(LI), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM) {}
- /// Plan how to best vectorize, return the best VF and its cost.
- VectorizationFactor plan(bool OptForSize, unsigned UserVF);
+ /// Plan how to best vectorize, return the best VF and its cost, or None if
+ /// vectorization and interleaving should be avoided up front.
+ Optional<VectorizationFactor> plan(bool OptForSize, unsigned UserVF);
/// Use the VPlan-native path to plan how to best vectorize, return the best
/// VF and its cost.
diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp
index cafb3b4..9328795 100644
--- a/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -843,7 +843,7 @@
AC(AC), ORE(ORE), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
/// \return An upper bound for the vectorization factor, or None if
- /// vectorization should be avoided up front.
+ /// vectorization and interleaving should be avoided up front.
Optional<unsigned> computeMaxVF(bool OptForSize);
/// \return The most profitable vectorization factor and the cost of that VF.
@@ -6079,9 +6079,6 @@
VectorizationFactor
LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
unsigned UserVF) {
- // Width 1 means no vectorization, cost 0 means uncomputed cost.
- const VectorizationFactor NoVectorization = {1U, 0U};
-
// Outer loop handling: They may require CFG and instruction level
// transformations before even evaluating whether vectorization is profitable.
// Since we cannot modify the incoming IR, we need to build VPlan upfront in
@@ -6101,7 +6098,7 @@
// For VPlan build stress testing, we bail out after VPlan construction.
if (VPlanBuildStressTest)
- return NoVectorization;
+ return VectorizationFactor::Disabled();
return {UserVF, 0};
}
@@ -6109,17 +6106,15 @@
LLVM_DEBUG(
dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
"VPlan-native path.\n");
- return NoVectorization;
+ return VectorizationFactor::Disabled();
}
-VectorizationFactor
-LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
+Optional<VectorizationFactor> LoopVectorizationPlanner::plan(bool OptForSize,
+ unsigned UserVF) {
assert(OrigLoop->empty() && "Inner loop expected.");
- // Width 1 means no vectorization, cost 0 means uncomputed cost.
- const VectorizationFactor NoVectorization = {1U, 0U};
Optional<unsigned> MaybeMaxVF = CM.computeMaxVF(OptForSize);
- if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize.
- return NoVectorization;
+ if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
+ return None;
// Invalidate interleave groups if all blocks of loop will be predicated.
if (CM.blockNeedsPredication(OrigLoop->getHeader()) &&
@@ -6139,7 +6134,7 @@
CM.selectUserVectorizationFactor(UserVF);
buildVPlansWithVPRecipes(UserVF, UserVF);
LLVM_DEBUG(printPlans(dbgs()));
- return {UserVF, 0};
+ return {{UserVF, 0}};
}
unsigned MaxVF = MaybeMaxVF.getValue();
@@ -6158,7 +6153,7 @@
buildVPlansWithVPRecipes(1, MaxVF);
LLVM_DEBUG(printPlans(dbgs()));
if (MaxVF == 1)
- return NoVectorization;
+ return VectorizationFactor::Disabled();
// Select the optimal vectorization factor.
return CM.selectVectorizationFactor(MaxVF);
@@ -7323,14 +7318,18 @@
unsigned UserVF = Hints.getWidth();
// Plan how to best vectorize, return the best VF and its cost.
- VectorizationFactor VF = LVP.plan(OptForSize, UserVF);
+ Optional<VectorizationFactor> MaybeVF = LVP.plan(OptForSize, UserVF);
- // Select the interleave count.
- unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
-
- // Get user interleave count.
+ VectorizationFactor VF = VectorizationFactor::Disabled();
+ unsigned IC = 1;
unsigned UserIC = Hints.getInterleave();
+ if (MaybeVF) {
+ VF = *MaybeVF;
+ // Select the interleave count.
+ IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
+ }
+
// Identify the diagnostic messages that should be produced.
std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
bool VectorizeLoop = true, InterleaveLoop = true;
@@ -7349,7 +7348,17 @@
VectorizeLoop = false;
}
- if (IC == 1 && UserIC <= 1) {
+ if (!MaybeVF && UserIC > 1) {
+ // Tell the user interleaving was avoided up-front, despite being explicitly
+ // requested.
+ LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
+ "interleaving should be avoided up front\n");
+ IntDiagMsg = std::make_pair(
+ "InterleavingAvoided",
+ "Ignoring UserIC, because interleaving was avoided up front");
+ InterleaveLoop = false;
+ UserIC = 1;
+ } else if (IC == 1 && UserIC <= 1) {
// Tell the user interleaving is not beneficial.
LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
IntDiagMsg = std::make_pair(
diff --git a/test/Transforms/LoopVectorize/no-interleave-up-front.ll b/test/Transforms/LoopVectorize/no-interleave-up-front.ll
new file mode 100644
index 0000000..a0c5345
--- /dev/null
+++ b/test/Transforms/LoopVectorize/no-interleave-up-front.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test case based on reproducer for
+; http://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6477
+
+define void @test1(i32 %n) #0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: br i1 false, label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK: .lr.ph.preheader:
+; CHECK-NEXT: br label [[DOTLR_PH:%.*]]
+; CHECK: .lr.ph:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]], !llvm.loop !0
+; CHECK: ._crit_edge.loopexit:
+; CHECK-NEXT: br label [[DOT_CRIT_EDGE]]
+; CHECK: ._crit_edge:
+; CHECK-NEXT: ret void
+;
+ br i1 false, label %.lr.ph, label %._crit_edge
+
+.lr.ph: ; preds = %.lr.ph, %0
+ %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+ %indvars.iv.next = add i64 %indvars.iv, 1
+ br i1 true, label %._crit_edge, label %.lr.ph, !llvm.loop !0
+
+._crit_edge: ; preds = %.lr.ph, %0
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
