[AArch64][GlobalISel] Improve codegen for G_VECREDUCE_{SMIN,SMAX,UMIN,UMAX} for odd-sized vectors (#82740)
i8 vectors do not have their sizes changed as I noticed regressions in
some tests when that was done.
This patch also adds support for most G_VECREDUCE_* operations to
moreElementsVector in LegalizerHelper.cpp.
The code for getting the "neutral" element is taken almost exactly as it
is in SelectionDAG, with the exception that support for
G_VECREDUCE_{FMAXIMUM,FMINIMUM} was not added.
The code for SelectionDAG is located at
SelectionDAG::getNeutralELement().
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 30f12bf..8079f85 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5216,6 +5216,43 @@
return Legalized;
}
+MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce(
+ unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) {
+ assert(Ty.isScalar() && "Expected scalar type to make neutral element for");
+
+ switch (Opcode) {
+ default:
+ llvm_unreachable(
+ "getNeutralElementForVecReduce called with invalid opcode!");
+ case TargetOpcode::G_VECREDUCE_ADD:
+ case TargetOpcode::G_VECREDUCE_OR:
+ case TargetOpcode::G_VECREDUCE_XOR:
+ case TargetOpcode::G_VECREDUCE_UMAX:
+ return MIRBuilder.buildConstant(Ty, 0);
+ case TargetOpcode::G_VECREDUCE_MUL:
+ return MIRBuilder.buildConstant(Ty, 1);
+ case TargetOpcode::G_VECREDUCE_AND:
+ case TargetOpcode::G_VECREDUCE_UMIN:
+ return MIRBuilder.buildConstant(
+ Ty, APInt::getAllOnes(Ty.getScalarSizeInBits()));
+ case TargetOpcode::G_VECREDUCE_SMAX:
+ return MIRBuilder.buildConstant(
+ Ty, APInt::getSignedMinValue(Ty.getSizeInBits()));
+ case TargetOpcode::G_VECREDUCE_SMIN:
+ return MIRBuilder.buildConstant(
+ Ty, APInt::getSignedMaxValue(Ty.getSizeInBits()));
+ case TargetOpcode::G_VECREDUCE_FADD:
+ return MIRBuilder.buildFConstant(Ty, -0.0);
+ case TargetOpcode::G_VECREDUCE_FMUL:
+ return MIRBuilder.buildFConstant(Ty, 1.0);
+ case TargetOpcode::G_VECREDUCE_FMINIMUM:
+ case TargetOpcode::G_VECREDUCE_FMAXIMUM:
+ assert(false && "getNeutralElementForVecReduce unimplemented for "
+ "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
+ }
+ llvm_unreachable("switch expected to return!");
+}
+
LegalizerHelper::LegalizeResult
LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
LLT MoreTy) {
@@ -5420,6 +5457,37 @@
Observer.changedInstr(MI);
return Legalized;
}
+ case TargetOpcode::G_VECREDUCE_FADD:
+ case TargetOpcode::G_VECREDUCE_FMUL:
+ case TargetOpcode::G_VECREDUCE_ADD:
+ case TargetOpcode::G_VECREDUCE_MUL:
+ case TargetOpcode::G_VECREDUCE_AND:
+ case TargetOpcode::G_VECREDUCE_OR:
+ case TargetOpcode::G_VECREDUCE_XOR:
+ case TargetOpcode::G_VECREDUCE_SMAX:
+ case TargetOpcode::G_VECREDUCE_SMIN:
+ case TargetOpcode::G_VECREDUCE_UMAX:
+ case TargetOpcode::G_VECREDUCE_UMIN: {
+ LLT OrigTy = MRI.getType(MI.getOperand(1).getReg());
+ MachineOperand &MO = MI.getOperand(1);
+ auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO);
+ auto NeutralElement = getNeutralElementForVecReduce(
+ MI.getOpcode(), MIRBuilder, MoreTy.getElementType());
+
+ LLT IdxTy(TLI.getVectorIdxTy(MIRBuilder.getDataLayout()));
+ for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements();
+ i != e; i++) {
+ auto Idx = MIRBuilder.buildConstant(IdxTy, i);
+ NewVec = MIRBuilder.buildInsertVectorElement(MoreTy, NewVec,
+ NeutralElement, Idx);
+ }
+
+ Observer.changingInstr(MI);
+ MO.setReg(NewVec.getReg(0));
+ Observer.changedInstr(MI);
+ return Legalized;
+ }
+
default:
return UnableToLegalize;
}