blob: 12f1d005249d60a77b02a22975749a827919f435 [file] [log] [blame]
//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements integer type expansion and promotion for LegalizeTypes.
// Promotion is the act of changing a computation in an illegal type into a
// computation in a larger type. For example, implementing i8 arithmetic in an
// i32 register (often needed on powerpc).
// Expansion is the act of changing a computation in an illegal type into a
// computation in two identical registers of a smaller type. For example,
// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
// targets).
//
//===----------------------------------------------------------------------===//
#include "LegalizeTypes.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
using namespace llvm;
#define DEBUG_TYPE "legalize-types"
//===----------------------------------------------------------------------===//
// Integer Result Promotion
//===----------------------------------------------------------------------===//
/// PromoteIntegerResult - This method is called when a result of a node is
/// found to be in need of promotion to a larger type. At this point, the node
/// may also have invalid operands or may have other results that need
/// expansion, we just know that (at least) one result needs promotion.
void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
SDValue Res = SDValue();
// See if the target wants to custom expand this node.
if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
return;
}
switch (N->getOpcode()) {
default:
#ifndef NDEBUG
dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
N->dump(&DAG); dbgs() << "\n";
#endif
report_fatal_error("Do not know how to promote this operator!");
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
case ISD::VP_BITREVERSE:
case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
case ISD::VP_BSWAP:
case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
case ISD::VP_CTLZ_ZERO_UNDEF:
case ISD::VP_CTLZ:
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
case ISD::PARITY:
case ISD::VP_CTPOP:
case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
case ISD::VP_CTTZ_ZERO_UNDEF:
case ISD::VP_CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
case ISD::VP_CTTZ_ELTS:
Res = PromoteIntRes_VP_CttzElements(N);
break;
case ISD::EXTRACT_VECTOR_ELT:
Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
break;
case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
break;
case ISD::SELECT:
case ISD::VSELECT:
case ISD::VP_SELECT:
case ISD::VP_MERGE:
Res = PromoteIntRes_Select(N);
break;
case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS:
case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
case ISD::SMIN:
case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
case ISD::UMIN:
case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
case ISD::SHL:
case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
case ISD::SIGN_EXTEND_INREG:
Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
case ISD::SRA:
case ISD::VP_SRA: Res = PromoteIntRes_SRA(N); break;
case ISD::SRL:
case ISD::VP_SRL: Res = PromoteIntRes_SRL(N); break;
case ISD::VP_TRUNCATE:
case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
case ISD::EXTRACT_SUBVECTOR:
Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
case ISD::INSERT_SUBVECTOR:
Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
case ISD::VECTOR_REVERSE:
Res = PromoteIntRes_VECTOR_REVERSE(N); break;
case ISD::VECTOR_SHUFFLE:
Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
case ISD::VECTOR_SPLICE:
Res = PromoteIntRes_VECTOR_SPLICE(N); break;
case ISD::VECTOR_INTERLEAVE:
case ISD::VECTOR_DEINTERLEAVE:
Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
return;
case ISD::INSERT_VECTOR_ELT:
Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
case ISD::BUILD_VECTOR:
Res = PromoteIntRes_BUILD_VECTOR(N);
break;
case ISD::SPLAT_VECTOR:
case ISD::SCALAR_TO_VECTOR:
Res = PromoteIntRes_ScalarOp(N);
break;
case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
case ISD::CONCAT_VECTORS:
Res = PromoteIntRes_CONCAT_VECTORS(N); break;
case ISD::ANY_EXTEND_VECTOR_INREG:
case ISD::SIGN_EXTEND_VECTOR_INREG:
case ISD::ZERO_EXTEND_VECTOR_INREG:
Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
case ISD::SIGN_EXTEND:
case ISD::VP_SIGN_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::VP_ZERO_EXTEND:
case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
case ISD::VP_FP_TO_SINT:
case ISD::VP_FP_TO_UINT:
case ISD::STRICT_FP_TO_SINT:
case ISD::STRICT_FP_TO_UINT:
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
case ISD::FP_TO_BF16:
case ISD::FP_TO_FP16:
Res = PromoteIntRes_FP_TO_FP16_BF16(N);
break;
case ISD::STRICT_FP_TO_BF16:
case ISD::STRICT_FP_TO_FP16:
Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
break;
case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
case ISD::AND:
case ISD::OR:
case ISD::XOR:
case ISD::ADD:
case ISD::SUB:
case ISD::MUL:
case ISD::VP_AND:
case ISD::VP_OR:
case ISD::VP_XOR:
case ISD::VP_ADD:
case ISD::VP_SUB:
case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
case ISD::VP_SMIN:
case ISD::VP_SMAX:
case ISD::SDIV:
case ISD::SREM:
case ISD::VP_SDIV:
case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
case ISD::VP_UMIN:
case ISD::VP_UMAX:
case ISD::UDIV:
case ISD::UREM:
case ISD::VP_UDIV:
case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
case ISD::SADDO:
case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
case ISD::UADDO:
case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
case ISD::SMULO:
case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
case ISD::ADDE:
case ISD::SUBE:
case ISD::UADDO_CARRY:
case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
case ISD::SADDO_CARRY:
case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
case ISD::SADDSAT:
case ISD::UADDSAT:
case ISD::SSUBSAT:
case ISD::USUBSAT:
case ISD::SSHLSAT:
case ISD::USHLSAT:
Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N);
break;
case ISD::VP_SADDSAT:
case ISD::VP_UADDSAT:
case ISD::VP_SSUBSAT:
case ISD::VP_USUBSAT:
Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N);
break;
case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
case ISD::SDIVFIX:
case ISD::SDIVFIXSAT:
case ISD::UDIVFIX:
case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
case ISD::ATOMIC_LOAD:
Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
case ISD::ATOMIC_LOAD_MIN:
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
case ISD::ATOMIC_SWAP:
Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
break;
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN:
Res = PromoteIntRes_VECREDUCE(N);
break;
case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
Res = PromoteIntRes_VP_REDUCE(N);
break;
case ISD::FREEZE:
Res = PromoteIntRes_FREEZE(N);
break;
case ISD::ROTL:
case ISD::ROTR:
Res = PromoteIntRes_Rotate(N);
break;
case ISD::FSHL:
case ISD::FSHR:
Res = PromoteIntRes_FunnelShift(N);
break;
case ISD::VP_FSHL:
case ISD::VP_FSHR:
Res = PromoteIntRes_VPFunnelShift(N);
break;
case ISD::IS_FPCLASS:
Res = PromoteIntRes_IS_FPCLASS(N);
break;
case ISD::FFREXP:
Res = PromoteIntRes_FFREXP(N);
break;
case ISD::LRINT:
case ISD::LLRINT:
Res = PromoteIntRes_XRINT(N);
break;
}
// If the result is null then the sub-method took care of registering it.
if (Res.getNode())
SetPromotedInteger(SDValue(N, ResNo), Res);
}
SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
unsigned ResNo) {
SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
return GetPromotedInteger(Op);
}
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
// Sign-extend the new bits, and continue the assertion.
SDValue Op = SExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::AssertSext, SDLoc(N),
Op.getValueType(), Op, N->getOperand(1));
}
SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
// Zero the new bits, and continue the assertion.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::AssertZext, SDLoc(N),
Op.getValueType(), Op, N->getOperand(1));
}
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(), ResVT,
N->getChain(), N->getBasePtr(),
N->getMemOperand());
if (N->getOpcode() == ISD::ATOMIC_LOAD) {
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
if (ETy == ISD::NON_EXTLOAD) {
switch (TLI.getExtendForAtomicOps()) {
case ISD::SIGN_EXTEND:
ETy = ISD::SEXTLOAD;
break;
case ISD::ZERO_EXTEND:
ETy = ISD::ZEXTLOAD;
break;
case ISD::ANY_EXTEND:
ETy = ISD::EXTLOAD;
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
}
cast<AtomicSDNode>(Res)->setExtensionType(ETy);
}
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(),
N->getChain(), N->getBasePtr(),
Op2, N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
unsigned ResNo) {
if (ResNo == 1) {
assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
// Only use the result of getSetCCResultType if it is legal,
// otherwise just use the promoted result type (NVT).
if (!TLI.isTypeLegal(SVT))
SVT = NVT;
SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
SDValue Res = DAG.getAtomicCmpSwap(
ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
N->getMemOperand());
ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
}
// Op2 is used for the comparison and thus must be extended according to the
// target's atomic operations. Op3 is merely stored and so can be left alone.
SDValue Op2 = N->getOperand(2);
SDValue Op3 = GetPromotedInteger(N->getOperand(3));
switch (TLI.getExtendForAtomicCmpSwapArg()) {
case ISD::SIGN_EXTEND:
Op2 = SExtPromotedInteger(Op2);
break;
case ISD::ZERO_EXTEND:
Op2 = ZExtPromotedInteger(Op2);
break;
case ISD::ANY_EXTEND:
Op2 = GetPromotedInteger(Op2);
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
SDVTList VTs =
DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
SDValue Res = DAG.getAtomicCmpSwap(
N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
N->getBasePtr(), Op2, Op3, N->getMemOperand());
// Update the use to N with the newly created Res.
for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
ReplaceValueWith(SDValue(N, i), Res.getValue(i));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDLoc dl(N);
switch (getTypeAction(InVT)) {
case TargetLowering::TypeLegal:
break;
case TargetLowering::TypePromoteInteger:
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
// The input promotes to the same size. Convert the promoted value.
return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
break;
case TargetLowering::TypeSoftenFloat:
// Promote the integer operand by hand.
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
case TargetLowering::TypeSoftPromoteHalf:
// Promote the integer operand by hand.
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
case TargetLowering::TypePromoteFloat: {
// Convert the promoted float by hand.
if (!NOutVT.isVector())
return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
break;
}
case TargetLowering::TypeExpandInteger:
case TargetLowering::TypeExpandFloat:
break;
case TargetLowering::TypeScalarizeVector:
// Convert the element to an integer and promote it by hand.
if (!NOutVT.isVector())
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
BitConvertToInteger(GetScalarizedVector(InOp)));
break;
case TargetLowering::TypeScalarizeScalableVector:
report_fatal_error("Scalarization of scalable vectors is not supported.");
case TargetLowering::TypeSplitVector: {
if (!NOutVT.isVector()) {
// For example, i32 = BITCAST v2i16 on alpha. Convert the split
// pieces of the input into integers and reassemble in the final type.
SDValue Lo, Hi;
GetSplitVector(N->getOperand(0), Lo, Hi);
Lo = BitConvertToInteger(Lo);
Hi = BitConvertToInteger(Hi);
if (DAG.getDataLayout().isBigEndian())
std::swap(Lo, Hi);
InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
EVT::getIntegerVT(*DAG.getContext(),
NOutVT.getSizeInBits()),
JoinIntegers(Lo, Hi));
return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
}
break;
}
case TargetLowering::TypeWidenVector:
// The input is widened to the same size. Convert to the widened value.
// Make sure that the outgoing value is not a vector, because this would
// make us bitcast between two vectors which are legalized in different ways.
if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
SDValue Res =
DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
// For big endian targets we need to shift the casted value or the
// interesting bits will end up at the wrong place.
if (DAG.getDataLayout().isBigEndian()) {
unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
}
return Res;
}
// If the output type is also a vector and widening it to the same size
// as the widened input type would be a legal type, we can widen the bitcast
// and handle the promotion after.
if (NOutVT.isVector()) {
TypeSize WidenInSize = NInVT.getSizeInBits();
TypeSize OutSize = OutVT.getSizeInBits();
if (WidenInSize.hasKnownScalarFactor(OutSize)) {
unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
EVT WideOutVT =
EVT::getVectorVT(*DAG.getContext(), OutVT.getVectorElementType(),
OutVT.getVectorElementCount() * Scale);
if (isTypeLegal(WideOutVT)) {
InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
DAG.getVectorIdxConstant(0, dl));
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
}
}
}
}
return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
CreateStackStoreLoad(InOp, OutVT));
}
SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
SDValue V = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::FREEZE, SDLoc(N),
V.getValueType(), V);
}
SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);
// If the larger BSWAP isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. We only do this for scalars since we have a shuffle
// based lowering for vectors in LegalizeVectorOps.
if (!OVT.isVector() &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
if (SDValue Res = TLI.expandBSWAP(N, DAG))
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
}
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
if (N->getOpcode() == ISD::BSWAP)
return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
ShAmt);
SDValue Mask = N->getOperand(1);
SDValue EVL = N->getOperand(2);
return DAG.getNode(ISD::VP_SRL, dl, NVT,
DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);
// If the larger BITREVERSE isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. We only do this for scalars since we have a shuffle
// based lowering for vectors in LegalizeVectorOps.
if (!OVT.isVector() && OVT.isSimple() &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
}
unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
if (N->getOpcode() == ISD::BITREVERSE)
return DAG.getNode(ISD::SRL, dl, NVT,
DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
SDValue Mask = N->getOperand(1);
SDValue EVL = N->getOperand(2);
return DAG.getNode(ISD::VP_SRL, dl, NVT,
DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL),
ShAmt, Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
// The pair element type may be legal, or may not promote to the same type as
// the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0)), JoinIntegers(N->getOperand(0),
N->getOperand(1)));
}
SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
EVT VT = N->getValueType(0);
// FIXME there is no actual debug info here
SDLoc dl(N);
// Zero extend things like i1, sign extend everything else. It shouldn't
// matter in theory which one we pick, but this tends to give better code?
unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
SDValue Result = DAG.getNode(Opc, dl,
TLI.getTypeToTransformTo(*DAG.getContext(), VT),
SDValue(N, 0));
assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
return Result;
}
SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
SDLoc dl(N);
// If the larger CTLZ isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type.
if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) {
if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
return Result;
}
}
// Subtract off the extra leading bits in the bigger type.
SDValue ExtractLeadingBits = DAG.getConstant(
NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
if (!N->isVPOpcode()) {
// Zero extend to the promoted type and do the count there.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::SUB, dl, NVT,
DAG.getNode(N->getOpcode(), dl, NVT, Op),
ExtractLeadingBits);
}
SDValue Mask = N->getOperand(1);
SDValue EVL = N->getOperand(2);
// Zero extend to the promoted type and do the count there.
SDValue Op = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
return DAG.getNode(ISD::VP_SUB, dl, NVT,
DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL),
ExtractLeadingBits, Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
// If the larger CTPOP isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type.
// TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
// TargetLowering.
if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) {
if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
return Result;
}
}
// Zero extend to the promoted type and do the count or parity there.
if (!N->isVPOpcode()) {
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
}
SDValue Mask = N->getOperand(1);
SDValue EVL = N->getOperand(2);
SDValue Op = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op, Mask,
EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);
// If the larger CTTZ isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. Don't expand if we can use CTPOP or CTLZ expansion on the
// larger type.
if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) &&
!TLI.isOperationLegal(ISD::CTPOP, NVT) &&
!TLI.isOperationLegal(ISD::CTLZ, NVT)) {
if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
return Result;
}
}
unsigned NewOpc = N->getOpcode();
if (NewOpc == ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
// The count is the same in the promoted type except if the original
// value was zero. This can be handled by setting the bit just off
// the top of the original type.
auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
OVT.getScalarSizeInBits());
if (NewOpc == ISD::CTTZ) {
Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
NewOpc = ISD::CTTZ_ZERO_UNDEF;
} else {
Op =
DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
N->getOperand(1), N->getOperand(2));
NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
}
}
if (!N->isVPOpcode())
return DAG.getNode(NewOpc, dl, NVT, Op);
return DAG.getNode(NewOpc, dl, NVT, Op, N->getOperand(1), N->getOperand(2));
}
SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(SDNode *N) {
SDLoc DL(N);
EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
return DAG.getNode(N->getOpcode(), DL, NewVT, N->ops());
}
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
SDLoc dl(N);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);
// If the input also needs to be promoted, do that first so we can get a
// get a good idea for the output type.
if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
== TargetLowering::TypePromoteInteger) {
SDValue In = GetPromotedInteger(Op0);
// If the new type is larger than NVT, use it. We probably won't need to
// promote it again.
EVT SVT = In.getValueType().getScalarType();
if (SVT.bitsGE(NVT)) {
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
}
}
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
}
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
unsigned NewOpc = N->getOpcode();
SDLoc dl(N);
// If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
// not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
// and SINT conversions are Custom, there is no way to tell which is
// preferable. We choose SINT because that's the right thing on PPC.)
if (N->getOpcode() == ISD::FP_TO_UINT &&
!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
NewOpc = ISD::FP_TO_SINT;
if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
!TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
NewOpc = ISD::STRICT_FP_TO_SINT;
if (N->getOpcode() == ISD::VP_FP_TO_UINT &&
!TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) &&
TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT))
NewOpc = ISD::VP_FP_TO_SINT;
SDValue Res;
if (N->isStrictFPOpcode()) {
Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
{N->getOperand(0), N->getOperand(1)});
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
} else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
N->getOperand(2)});
} else {
Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
}
// Assert that the converted value fits in the original type. If it doesn't
// (eg: because the value being converted is too big), then the result of the
// original operation was undefined anyway, so the assert is still correct.
//
// NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
// before legalization: fp-to-uint16, 65534. -> 0xfffe
// after legalization: fp-to-sint32, 65534. -> 0x0000fffe
return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
N->getOpcode() == ISD::VP_FP_TO_UINT)
? ISD::AssertZext
: ISD::AssertSext,
dl, NVT, Res,
DAG.getValueType(N->getValueType(0).getScalarType()));
}
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
// Promote the result type, while keeping the original width in Op1.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
N->getOperand(1));
}
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
}
SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
N->getOperand(0), N->getOperand(1));
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
}
SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
SDValue Res =
DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
if (getTypeAction(N->getOperand(0).getValueType())
== TargetLowering::TypePromoteInteger) {
SDValue Res = GetPromotedInteger(N->getOperand(0));
assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
// If the result and operand types are the same after promotion, simplify
// to an in-register extension. Unless this is a VP_*_EXTEND.
if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
// The high bits are not guaranteed to be anything. Insert an extend.
if (N->getOpcode() == ISD::SIGN_EXTEND)
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
DAG.getValueType(N->getOperand(0).getValueType()));
if (N->getOpcode() == ISD::ZERO_EXTEND)
return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
return Res;
}
}
// Otherwise, just extend the original operand all the way to the larger type.
if (N->getNumOperands() != 1) {
assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
assert(N->isVPOpcode() && "Expected VP opcode");
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
N->getOperand(1), N->getOperand(2));
}
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
}
SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
ISD::LoadExtType ExtType =
ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
SDLoc dl(N);
SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
N->getMemoryVT(), N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
ISD::LoadExtType ExtType = N->getExtensionType();
if (ExtType == ISD::NON_EXTLOAD)
ExtType = ISD::EXTLOAD;
SDLoc dl(N);
SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
N->getOffset(), N->getMask(), ExtPassThru,
N->getMemoryVT(), N->getMemOperand(),
N->getAddressingMode(), ExtType,
N->isExpandingLoad());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
assert(NVT == ExtPassThru.getValueType() &&
"Gather result type and the passThru argument type should be the same");
ISD::LoadExtType ExtType = N->getExtensionType();
if (ExtType == ISD::NON_EXTLOAD)
ExtType = ISD::EXTLOAD;
SDLoc dl(N);
SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
N->getIndex(), N->getScale() };
SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
N->getMemoryVT(), dl, Ops,
N->getMemOperand(), N->getIndexType(),
ExtType);
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
}
/// Promote the overflow flag of an overflowing arithmetic node.
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
// Change the return type of the boolean result while obeying
// getSetCCResultType.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
EVT VT = N->getValueType(0);
EVT SVT = getSetCCResultType(VT);
SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
unsigned NumOps = N->getNumOperands();
assert(NumOps <= 3 && "Too many operands");
if (NumOps == 3)
Ops[2] = PromoteTargetBoolean(N->getOperand(2), VT);
SDLoc dl(N);
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
ArrayRef(Ops, NumOps));
// Modified the sum result - switch anything that used the old sum to use
// the new one.
ReplaceValueWith(SDValue(N, 0), Res);
// Convert to the expected type.
return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
}
template <class MatchContextClass>
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
// If the promoted type is legal, we can convert this to:
// 1. ANY_EXTEND iN to iM
// 2. SHL by M-N
// 3. [US][ADD|SUB|SHL]SAT
// 4. L/ASHR by M-N
// Else it is more efficient to convert this to a min and a max
// operation in the higher precision arithmetic.
SDLoc dl(N);
SDValue Op1 = N->getOperand(0);
SDValue Op2 = N->getOperand(1);
MatchContextClass matcher(DAG, TLI, N);
unsigned OldBits = Op1.getScalarValueSizeInBits();
unsigned Opcode = matcher.getRootBaseOpcode();
bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
// FIXME: We need vp-aware PromotedInteger functions.
SDValue Op1Promoted, Op2Promoted;
if (IsShift) {
Op1Promoted = GetPromotedInteger(Op1);
Op2Promoted = ZExtPromotedInteger(Op2);
} else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
Op1Promoted = ZExtPromotedInteger(Op1);
Op2Promoted = ZExtPromotedInteger(Op2);
} else {
Op1Promoted = SExtPromotedInteger(Op1);
Op2Promoted = SExtPromotedInteger(Op2);
}
EVT PromotedType = Op1Promoted.getValueType();
unsigned NewBits = PromotedType.getScalarSizeInBits();
if (Opcode == ISD::UADDSAT) {
APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
SDValue Add =
matcher.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
return matcher.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
}
// USUBSAT can always be promoted as long as we have zero-extended the args.
if (Opcode == ISD::USUBSAT)
return matcher.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
Op2Promoted);
// Shift cannot use a min/max expansion, we can't detect overflow if all of
// the bits have been shifted out.
if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
unsigned ShiftOp;
switch (Opcode) {
case ISD::SADDSAT:
case ISD::SSUBSAT:
case ISD::SSHLSAT:
ShiftOp = ISD::SRA;
break;
case ISD::USHLSAT:
ShiftOp = ISD::SRL;
break;
default:
llvm_unreachable("Expected opcode to be signed or unsigned saturation "
"addition, subtraction or left shift");
}
unsigned SHLAmount = NewBits - OldBits;
SDValue ShiftAmount =
DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
Op1Promoted =
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
if (!IsShift)
Op2Promoted =
matcher.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
SDValue Result =
matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
}
unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
SDValue Result =
matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
return Result;
}
SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
// Can just promote the operands then continue with operation.
SDLoc dl(N);
SDValue Op1Promoted, Op2Promoted;
bool Signed =
N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
bool Saturating =
N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
if (Signed) {
Op1Promoted = SExtPromotedInteger(N->getOperand(0));
Op2Promoted = SExtPromotedInteger(N->getOperand(1));
} else {
Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
}
EVT OldType = N->getOperand(0).getValueType();
EVT PromotedType = Op1Promoted.getValueType();
unsigned DiffSize =
PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
if (Saturating) {
// Promoting the operand and result values changes the saturation width,
// which is extends the values that we clamp to on saturation. This could be
// resolved by shifting one of the operands the same amount, which would
// also shift the result we compare against, then shifting back.
Op1Promoted =
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
Op2Promoted, N->getOperand(2));
unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
return DAG.getNode(ShiftOp, dl, PromotedType, Result,
DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
}
return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
N->getOperand(2));
}
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
unsigned SatW, bool Signed,
const TargetLowering &TLI,
SelectionDAG &DAG) {
EVT VT = V.getValueType();
unsigned VTW = VT.getScalarSizeInBits();
if (!Signed) {
// Saturate to the unsigned maximum by getting the minimum of V and the
// maximum.
return DAG.getNode(ISD::UMIN, dl, VT, V,
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
dl, VT));
}
// Saturate to the signed maximum (the low SatW - 1 bits) by taking the
// signed minimum of it and V.
V = DAG.getNode(ISD::SMIN, dl, VT, V,
DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
dl, VT));
// Saturate to the signed minimum (the high SatW + 1 bits) by taking the
// signed maximum of it and V.
V = DAG.getNode(ISD::SMAX, dl, VT, V,
DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
dl, VT));
return V;
}
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
unsigned Scale, const TargetLowering &TLI,
SelectionDAG &DAG, unsigned SatW = 0) {
EVT VT = LHS.getValueType();
unsigned VTSize = VT.getScalarSizeInBits();
bool Signed = N->getOpcode() == ISD::SDIVFIX ||
N->getOpcode() == ISD::SDIVFIXSAT;
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
N->getOpcode() == ISD::UDIVFIXSAT;
SDLoc dl(N);
// Widen the types by a factor of two. This is guaranteed to expand, since it
// will always have enough high bits in the LHS to shift into.
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
if (VT.isVector())
WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
VT.getVectorElementCount());
LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
DAG);
assert(Res && "Expanding DIVFIX with wide type failed?");
if (Saturating) {
// If the caller has told us to saturate at something less, use that width
// instead of the type before doubling. However, it cannot be more than
// what we just widened!
assert(SatW <= VTSize &&
"Tried to saturate to more than the original type?");
Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
TLI, DAG);
}
return DAG.getZExtOrTrunc(Res, dl, VT);
}
SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
SDLoc dl(N);
SDValue Op1Promoted, Op2Promoted;
bool Signed = N->getOpcode() == ISD::SDIVFIX ||
N->getOpcode() == ISD::SDIVFIXSAT;
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
N->getOpcode() == ISD::UDIVFIXSAT;
if (Signed) {
Op1Promoted = SExtPromotedInteger(N->getOperand(0));
Op2Promoted = SExtPromotedInteger(N->getOperand(1));
} else {
Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
}
EVT PromotedType = Op1Promoted.getValueType();
unsigned Scale = N->getConstantOperandVal(2);
// If the type is already legal and the operation is legal in that type, we
// should not early expand.
if (TLI.isTypeLegal(PromotedType)) {
TargetLowering::LegalizeAction Action =
TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
unsigned Diff = PromotedType.getScalarSizeInBits() -
N->getValueType(0).getScalarSizeInBits();
if (Saturating)
Op1Promoted =
DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
DAG.getShiftAmountConstant(Diff, PromotedType, dl));
SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
Op2Promoted, N->getOperand(2));
if (Saturating)
Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
DAG.getShiftAmountConstant(Diff, PromotedType, dl));
return Res;
}
}
// See if we can perform the division in this type without expanding.
if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
Op2Promoted, Scale, DAG)) {
if (Saturating)
Res = SaturateWidenedDIVFIX(Res, dl,
N->getValueType(0).getScalarSizeInBits(),
Signed, TLI, DAG);
return Res;
}
// If we cannot, expand it to twice the type width. If we are saturating, give
// it the original width as a saturating width so we don't need to emit
// two saturations.
return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
N->getValueType(0).getScalarSizeInBits());
}
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
if (ResNo == 1)
return PromoteIntRes_Overflow(N);
// The operation overflowed iff the result in the larger type is not the
// sign extension of its truncation to the original type.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));
EVT OVT = N->getOperand(0).getValueType();
EVT NVT = LHS.getValueType();
SDLoc dl(N);
// Do the arithmetic in the larger type.
unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
// Calculate the overflow flag: sign extend the arithmetic result from
// the original type.
SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
DAG.getValueType(OVT));
// Overflowed if and only if this is not equal to Res.
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Ofl);
return Res;
}
SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
SDValue Mask = N->getOperand(0);
SDValue LHS = GetPromotedInteger(N->getOperand(1));
SDValue RHS = GetPromotedInteger(N->getOperand(2));
unsigned Opcode = N->getOpcode();
if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
N->getOperand(3));
return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
SDValue LHS = GetPromotedInteger(N->getOperand(2));
SDValue RHS = GetPromotedInteger(N->getOperand(3));
return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
LHS.getValueType(), N->getOperand(0),
N->getOperand(1), LHS, RHS, N->getOperand(4));
}
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
EVT InVT = N->getOperand(OpNo).getValueType();
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
EVT SVT = getSetCCResultType(InVT);
// If we got back a type that needs to be promoted, this likely means the
// the input type also needs to be promoted. So get the promoted type for
// the input and try the query again.
if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
SVT = getSetCCResultType(InVT);
} else {
// Input type isn't promoted, just use the default promoted type.
SVT = NVT;
}
}
SDLoc dl(N);
assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
"Vector compare must return a vector result!");
// Get the SETCC result using the canonical SETCC type.
SDValue SetCC;
if (N->isStrictFPOpcode()) {
SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
N->getOperand(2), N->getOperand(3)};
SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
} else
SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
N->getOperand(1), N->getOperand(2), N->getFlags());
// Convert to the expected type.
return DAG.getSExtOrTrunc(SetCC, dl, NVT);
}
SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
SDLoc DL(N);
SDValue Arg = N->getOperand(0);
SDValue Test = N->getOperand(1);
EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
}
SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
EVT VT = N->getValueType(0);
SDLoc dl(N);
SDValue Res =
DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
ReplaceValueWith(SDValue(N, 0), Res);
return Res.getValue(1);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
SDValue LHS = GetPromotedInteger(N->getOperand(0));
SDValue RHS = N->getOperand(1);
if (N->getOpcode() != ISD::VP_SHL) {
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = ZExtPromotedInteger(RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
}
SDValue Mask = N->getOperand(2);
SDValue EVL = N->getOperand(3);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
Op.getValueType(), Op, N->getOperand(1));
}
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
// The input may have strange things in the top bits of the registers, but
// these operations don't care. They may have weird bits going out, but
// that too is okay if they are integer operations.
SDValue LHS = GetPromotedInteger(N->getOperand(0));
SDValue RHS = GetPromotedInteger(N->getOperand(1));
if (N->getNumOperands() == 2)
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
assert(N->isVPOpcode() && "Expected VP opcode");
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
N->getOperand(2), N->getOperand(3));
}
SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
if (N->getNumOperands() == 2) {
// Sign extend the input.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
}
assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
assert(N->isVPOpcode() && "Expected VP opcode");
SDValue Mask = N->getOperand(2);
SDValue EVL = N->getOperand(3);
// Sign extend the input.
SDValue LHS = VPSExtPromotedInteger(N->getOperand(0), Mask, EVL);
SDValue RHS = VPSExtPromotedInteger(N->getOperand(1), Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
if (N->getNumOperands() == 2) {
// Zero extend the input.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
}
assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
assert(N->isVPOpcode() && "Expected VP opcode");
// Zero extend the input.
SDValue Mask = N->getOperand(2);
SDValue EVL = N->getOperand(3);
SDValue LHS = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
SDValue RHS = VPZExtPromotedInteger(N->getOperand(1), Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
// It doesn't matter if we sign extend or zero extend in the inputs. So do
// whatever is best for the target and the promoted operands.
SExtOrZExtPromotedOperands(LHS, RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N),
LHS.getValueType(), LHS, RHS);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
SDValue RHS = N->getOperand(1);
if (N->getOpcode() != ISD::VP_SRA) {
// The input value must be properly sign extended.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = ZExtPromotedInteger(RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
}
SDValue Mask = N->getOperand(2);
SDValue EVL = N->getOperand(3);
// The input value must be properly sign extended.
SDValue LHS = VPSExtPromotedInteger(N->getOperand(0), Mask, EVL);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
SDValue RHS = N->getOperand(1);
if (N->getOpcode() != ISD::VP_SRL) {
// The input value must be properly zero extended.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = ZExtPromotedInteger(RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
}
SDValue Mask = N->getOperand(2);
SDValue EVL = N->getOperand(3);
// The input value must be properly zero extended.
SDValue LHS = VPZExtPromotedInteger(N->getOperand(0), Mask, EVL);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
// Lower the rotate to shifts and ORs which can be promoted.
SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
ReplaceValueWith(SDValue(N, 0), Res);
return SDValue();
}
SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
SDValue Hi = GetPromotedInteger(N->getOperand(0));
SDValue Lo = GetPromotedInteger(N->getOperand(1));
SDValue Amt = N->getOperand(2);
if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
Amt = ZExtPromotedInteger(Amt);
EVT AmtVT = Amt.getValueType();
SDLoc DL(N);
EVT OldVT = N->getOperand(0).getValueType();
EVT VT = Lo.getValueType();
unsigned Opcode = N->getOpcode();
bool IsFSHR = Opcode == ISD::FSHR;
unsigned OldBits = OldVT.getScalarSizeInBits();
unsigned NewBits = VT.getScalarSizeInBits();
// Amount has to be interpreted modulo the old bit width.
Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
DAG.getConstant(OldBits, DL, AmtVT));
// If the promoted type is twice the size (or more), then we use the
// traditional funnel 'double' shift codegen. This isn't necessary if the
// shift amount is constant.
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
!TLI.isOperationLegalOrCustom(Opcode, VT)) {
SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
if (!IsFSHR)
Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
return Res;
}
// Shift Lo up to occupy the upper bits of the promoted type.
SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
// Increase Amount to shift the result into the lower bits of the promoted
// type.
if (IsFSHR)
Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
}
// A vp version of PromoteIntRes_FunnelShift.
SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
SDValue Hi = GetPromotedInteger(N->getOperand(0));
SDValue Lo = GetPromotedInteger(N->getOperand(1));
SDValue Amt = N->getOperand(2);
SDValue Mask = N->getOperand(3);
SDValue EVL = N->getOperand(4);
if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
EVT AmtVT = Amt.getValueType();
SDLoc DL(N);
EVT OldVT = N->getOperand(0).getValueType();
EVT VT = Lo.getValueType();
unsigned Opcode = N->getOpcode();
bool IsFSHR = Opcode == ISD::VP_FSHR;
unsigned OldBits = OldVT.getScalarSizeInBits();
unsigned NewBits = VT.getScalarSizeInBits();
// Amount has to be interpreted modulo the old bit width.
Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt,
DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL);
// If the promoted type is twice the size (or more), then we use the
// traditional funnel 'double' shift codegen. This isn't necessary if the
// shift amount is constant.
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
!TLI.isOperationLegalOrCustom(Opcode, VT)) {
SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL);
Lo = DAG.getVPZeroExtendInReg(Lo, Mask, EVL, DL, OldVT);
SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL);
Res = DAG.getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL, DL, VT, Res, Amt,
Mask, EVL);
if (!IsFSHR)
Res = DAG.getNode(ISD::VP_SRL, DL, VT, Res, HiShift, Mask, EVL);
return Res;
}
// Shift Lo up to occupy the upper bits of the promoted type.
SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL);
// Increase Amount to shift the result into the lower bits of the promoted
// type.
if (IsFSHR)
Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL);
}
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res;
SDValue InOp = N->getOperand(0);
SDLoc dl(N);
switch (getTypeAction(InOp.getValueType())) {
default: llvm_unreachable("Unknown type action!");
case TargetLowering::TypeLegal:
case TargetLowering::TypeExpandInteger:
Res = InOp;
break;
case TargetLowering::TypePromoteInteger:
Res = GetPromotedInteger(InOp);
break;
case TargetLowering::TypeSplitVector: {
EVT InVT = InOp.getValueType();
assert(InVT.isVector() && "Cannot split scalar types");
ElementCount NumElts = InVT.getVectorElementCount();
assert(NumElts == NVT.getVectorElementCount() &&
"Dst and Src must have the same number of elements");
assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
"Promoted vector type must be a power of two");
SDValue EOp1, EOp2;
GetSplitVector(InOp, EOp1, EOp2);
EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
NumElts.divideCoefficientBy(2));
if (N->getOpcode() == ISD::TRUNCATE) {
EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
} else {
assert(N->getOpcode() == ISD::VP_TRUNCATE &&
"Expected VP_TRUNCATE opcode");
SDValue MaskLo, MaskHi, EVLLo, EVLHi;
std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
std::tie(EVLLo, EVLHi) =
DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
}
return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
}
// TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
// targets.
case TargetLowering::TypeWidenVector: {
SDValue WideInOp = GetWidenedVector(InOp);
// Truncate widened InOp.
unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
N->getValueType(0).getScalarType(), NumElem);
SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
// Zero extend so that the elements are of same type as those of NVT
EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
NumElem);
SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
// Extract the low NVT subvector.
SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
}
}
// Truncate to NVT instead of VT
if (N->getOpcode() == ISD::VP_TRUNCATE)
return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
N->getOperand(2));
return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
}
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
if (ResNo == 1)
return PromoteIntRes_Overflow(N);
// The operation overflowed iff the result in the larger type is not the
// zero extension of its truncation to the original type.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
EVT OVT = N->getOperand(0).getValueType();
EVT NVT = LHS.getValueType();
SDLoc dl(N);
// Do the arithmetic in the larger type.
unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
// Calculate the overflow flag: zero extend the arithmetic result from
// the original type.
SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
// Overflowed if and only if this is not equal to Res.
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Ofl);
return Res;
}
// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
// the third operand of ADDE/SUBE nodes is carry flag, which differs from
// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
unsigned ResNo) {
if (ResNo == 1)
return PromoteIntRes_Overflow(N);
// We need to sign-extend the operands so the carry value computed by the
// wide operation will be equivalent to the carry value computed by the
// narrow operation.
// An UADDO_CARRY can generate carry only if any of the operands has its
// most significant bit set. Sign extension propagates the most significant
// bit into the higher bits which means the extra bit that the narrow
// addition would need (i.e. the carry) will be propagated through the higher
// bits of the wide addition.
// A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
// be preserved by sign extension.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));
EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
// Do the arithmetic in the wide type.
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
LHS, RHS, N->getOperand(2));
// Update the users of the original carry/borrow value.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return SDValue(Res.getNode(), 0);
}
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
unsigned ResNo) {
assert(ResNo == 1 && "Don't know how to promote other results yet.");
return PromoteIntRes_Overflow(N);
}
SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
// If a larger ABS or SMAX isn't supported by the target, try to expand now.
// If we expand later we'll end up sign extending more than just the sra input
// in sra+xor+sub expansion.
if (!OVT.isVector() &&
!TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) &&
!TLI.isOperationLegal(ISD::SMAX, NVT)) {
if (SDValue Res = TLI.expandABS(N, DAG))
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
}
SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
}
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
// Promote the overflow bit trivially.
if (ResNo == 1)
return PromoteIntRes_Overflow(N);
SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
SDLoc DL(N);
EVT SmallVT = LHS.getValueType();
// To determine if the result overflowed in a larger type, we extend the
// input to the larger type, do the multiply (checking if it overflows),
// then also check the high bits of the result to see if overflow happened
// there.
if (N->getOpcode() == ISD::SMULO) {
LHS = SExtPromotedInteger(LHS);
RHS = SExtPromotedInteger(RHS);
} else {
LHS = ZExtPromotedInteger(LHS);
RHS = ZExtPromotedInteger(RHS);
}
SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
// Overflow occurred if it occurred in the larger type, or if the high part
// of the result does not zero/sign-extend the low part. Check this second
// possibility first.
SDValue Overflow;
if (N->getOpcode() == ISD::UMULO) {
// Unsigned overflow occurred if the high part is non-zero.
unsigned Shift = SmallVT.getScalarSizeInBits();
SDValue Hi =
DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
DAG.getConstant(0, DL, Hi.getValueType()),
ISD::SETNE);
} else {
// Signed overflow occurred if the high part does not sign extend the low.
SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
Mul, DAG.getValueType(SmallVT));
Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
}
// The only other way for overflow to occur is if the multiplication in the
// larger type itself overflowed.
Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
SDValue(Mul.getNode(), 1));
// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Overflow);
return Mul;
}
SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0)));
}
SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
const APInt &MulImm = N->getConstantOperandAPInt(0);
return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
}
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
SDValue Chain = N->getOperand(0); // Get the chain.
SDValue Ptr = N->getOperand(1); // Get the pointer.
EVT VT = N->getValueType(0);
SDLoc dl(N);
MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
// The argument is passed as NumRegs registers of type RegVT.
SmallVector<SDValue, 8> Parts(NumRegs);
for (unsigned i = 0; i < NumRegs; ++i) {
Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
N->getConstantOperandVal(3));
Chain = Parts[i].getValue(1);
}
// Handle endianness of the load.
if (DAG.getDataLayout().isBigEndian())
std::reverse(Parts.begin(), Parts.end());
// Assemble the parts in the promoted type.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
for (unsigned i = 1; i < NumRegs; ++i) {
SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
// Shift it to the right position and "or" it in.
Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
DAG.getConstant(i * RegVT.getSizeInBits(), dl,
TLI.getPointerTy(DAG.getDataLayout())));
Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
}
// Modified the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Chain);
return Res;
}
//===----------------------------------------------------------------------===//
// Integer Operand Promotion
//===----------------------------------------------------------------------===//
/// PromoteIntegerOperand - This method is called when the specified operand of
/// the specified node is found to need promotion. At this point, all of the
/// result types of the node are known to be legal, but other operands of the
/// node may need promotion or expansion as well as the specified one.
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
SDValue Res = SDValue();
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
return false;
}
switch (N->getOpcode()) {
default:
#ifndef NDEBUG
dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
N->dump(&DAG); dbgs() << "\n";
#endif
report_fatal_error("Do not know how to promote this operator's operand!");
case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
case ISD::ATOMIC_STORE:
Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
break;
case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
case ISD::INSERT_VECTOR_ELT:
Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
break;
case ISD::SPLAT_VECTOR:
case ISD::SCALAR_TO_VECTOR:
Res = PromoteIntOp_ScalarOp(N);
break;
case ISD::VSELECT:
case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
case ISD::VP_SETCC:
case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
case ISD::VP_SINT_TO_FP:
case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
OpNo); break;
case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
OpNo); break;
case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
OpNo); break;
case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
OpNo); break;
case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
OpNo); break;
case ISD::VP_TRUNCATE:
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
case ISD::BF16_TO_FP:
case ISD::FP16_TO_FP:
case ISD::VP_UINT_TO_FP:
case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
case ISD::STRICT_FP16_TO_FP:
case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
case ISD::ROTL:
case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
case ISD::FSHL:
case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
case ISD::FRAMEADDR:
case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
case ISD::UMULFIXSAT:
case ISD::SDIVFIX:
case ISD::SDIVFIXSAT:
case ISD::UDIVFIX:
case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
case ISD::FPOWI:
case ISD::STRICT_FPOWI:
case ISD::FLDEXP:
case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
Res = PromoteIntOp_VP_REDUCE(N, OpNo);
break;
case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
case ISD::STACKMAP:
Res = PromoteIntOp_STACKMAP(N, OpNo);
break;
case ISD::PATCHPOINT:
Res = PromoteIntOp_PATCHPOINT(N, OpNo);
break;
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
Res = PromoteIntOp_VP_STRIDED(N, OpNo);
break;
case ISD::EXPERIMENTAL_VP_SPLICE:
Res = PromoteIntOp_VP_SPLICE(N, OpNo);
break;
}
// If the result is null, the sub-method took care of registering results etc.
if (!Res.getNode()) return false;
// If the result is N, the sub-method updated N in place. Tell the legalizer
// core about this.
if (Res.getNode() == N)
return true;
const bool IsStrictFp = N->isStrictFPOpcode();
assert(Res.getValueType() == N->getValueType(0) &&
N->getNumValues() == (IsStrictFp ? 2 : 1) &&
"Invalid operand expansion");
LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
Res.dump());
ReplaceValueWith(SDValue(N, 0), Res);
if (IsStrictFp)
ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
return false;
}
// These operands can be either sign extended or zero extended as long as we
// treat them the same. If an extension is free, choose that. Otherwise, follow
// target preference.
void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
SDValue OpL = GetPromotedInteger(LHS);
SDValue OpR = GetPromotedInteger(RHS);
if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
// The target would prefer to promote the comparison operand with sign
// extension. Honor that unless the promoted values are already zero
// extended.
unsigned OpLEffectiveBits =
DAG.computeKnownBits(OpL).countMaxActiveBits();
unsigned OpREffectiveBits =
DAG.computeKnownBits(OpR).countMaxActiveBits();
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
LHS = OpL;
RHS = OpR;
return;
}
// The promoted values aren't zero extended, use a sext_inreg.
LHS = SExtPromotedInteger(LHS);
RHS = SExtPromotedInteger(RHS);
return;
}
// Prefer to promote the comparison operand with zero extension.
// If the width of OpL/OpR excluding the duplicated sign bits is no greater
// than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
// that we might not be able to remove.
unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
LHS = OpL;
RHS = OpR;
return;
}
// Otherwise, use zext_inreg.
LHS = ZExtPromotedInteger(LHS);
RHS = ZExtPromotedInteger(RHS);
}
/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
/// shared among BR_CC, SELECT_CC, and SETCC handlers.
void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
ISD::CondCode CCCode) {
// We have to insert explicit sign or zero extends. Note that we could
// insert sign extends for ALL conditions. For those operations where either
// zero or sign extension would be valid, we ask the target which extension
// it would prefer.
// Signed comparisons always require sign extension.
if (ISD::isSignedIntSetCC(CCCode)) {
LHS = SExtPromotedInteger(LHS);
RHS = SExtPromotedInteger(RHS);
return;
}
assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) &&
"Unknown integer comparison!");
SExtOrZExtPromotedOperands(LHS, RHS);
}
SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
}
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
SDValue Op1 = GetPromotedInteger(N->getOperand(1));
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
}
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
// This should only occur in unusual situations like bitcasting to an
// x86_fp80, so just turn it into a store+load
return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
}
SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 2 && "Don't know how to promote this operand!");
SDValue LHS = N->getOperand(2);
SDValue RHS = N->getOperand(3);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
// The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
// legal types.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
N->getOperand(1), LHS, RHS, N->getOperand(4)),
0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
assert(OpNo == 1 && "only know how to promote condition");
// Promote all the way up to the canonical SetCC type.
SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
// The chain (Op#0) and basic block destination (Op#2) are always legal types.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
N->getOperand(2)), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
// Since the result type is legal, the operands must promote to it.
EVT OVT = N->getOperand(0).getValueType();
SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
SDValue Hi = GetPromotedInteger(N->getOperand(1));
assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
SDLoc dl(N);
Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
DAG.getConstant(OVT.getSizeInBits(), dl,
TLI.getPointerTy(DAG.getDataLayout())));
return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
}
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
// The vector type is legal but the element type is not. This implies
// that the vector is a power-of-two in length and that the element
// type does not have a strange size (eg: it is not i1).
EVT VecVT = N->getValueType(0);
unsigned NumElts = VecVT.getVectorNumElements();
assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
"Legal vector of one illegal element?");
// Promote the inserted value. The type does not need to match the
// vector element type. Check that any extra bits introduced will be
// truncated away.
assert(N->getOperand(0).getValueSizeInBits() >=
N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!");
SmallVector<SDValue, 16> NewOps;
for (unsigned i = 0; i < NumElts; ++i)
NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
unsigned OpNo) {
if (OpNo == 1) {
// Promote the inserted value. This is valid because the type does not
// have to match the vector element type.
// Check that any extra bits introduced will be truncated away.
assert(N->getOperand(1).getValueSizeInBits() >=
N->getValueType(0).getScalarSizeInBits() &&
"Type of inserted value narrower than vector element type!");
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
GetPromotedInteger(N->getOperand(1)),
N->getOperand(2)),
0);
}
assert(OpNo == 2 && "Different operand and result vector types?");
// Promote the index.
SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
TLI.getVectorIdxTy(DAG.getDataLayout()));
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
N->getOperand(1), Idx), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
// Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
// so just promote the operand in place.
return SDValue(DAG.UpdateNodeOperands(N,
GetPromotedInteger(N->getOperand(0))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Only know how to promote the condition!");
SDValue Cond = N->getOperand(0);
EVT OpTy = N->getOperand(1).getValueType();
if (N->getOpcode() == ISD::VSELECT)
if (SDValue Res = WidenVSELECTMask(N))
return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
Res, N->getOperand(1), N->getOperand(2));
// Promote all the way up to the canonical SetCC type.
EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
Cond = PromoteTargetBoolean(Cond, OpVT);
return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
N->getOperand(2)), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!");
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
// The CC (#4) and the possible return values (#2 and #3) have legal types.
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
N->getOperand(3), N->getOperand(4)), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!");
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
// The CC (#2) is always legal.
if (N->getOpcode() == ISD::SETCC)
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
N->getOperand(3), N->getOperand(4)),
0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
ZExtPromotedInteger(N->getOperand(1))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
ZExtPromotedInteger(N->getOperand(2))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
SDLoc dl(N);
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
Op, DAG.getValueType(N->getOperand(0).getValueType()));
}
SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
SDLoc dl(N);
EVT VT = N->getValueType(0);
SDValue Op = GetPromotedInteger(N->getOperand(0));
// FIXME: There is no VP_ANY_EXTEND yet.
Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
N->getOperand(2));
unsigned Diff =
VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits();
SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl);
// FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1),
N->getOperand(2));
return DAG.getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt, N->getOperand(1),
N->getOperand(2));
}
SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
if (N->getOpcode() == ISD::VP_SINT_TO_FP)
return SDValue(DAG.UpdateNodeOperands(N,
SExtPromotedInteger(N->getOperand(0)),
N->getOperand(1), N->getOperand(2)),
0);
return SDValue(DAG.UpdateNodeOperands(N,
SExtPromotedInteger(N->getOperand(0))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
SExtPromotedInteger(N->getOperand(1))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
SDLoc dl(N);
SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
// Truncate the value and store the result.
return DAG.getTruncStore(Ch, dl, Val, Ptr,
N->getMemoryVT(), N->getMemOperand());
}
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
unsigned OpNo) {
SDValue DataOp = N->getValue();
SDValue Mask = N->getMask();
if (OpNo == 4) {
// The Mask. Update in place.
EVT DataVT = DataOp.getValueType();
Mask = PromoteTargetBoolean(Mask, DataVT);
SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
NewOps[4] = Mask;
return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}
assert(OpNo == 1 && "Unexpected operand for promotion");
DataOp = GetPromotedInteger(DataOp);
return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
N->getOffset(), Mask, N->getMemoryVT(),
N->getMemOperand(), N->getAddressingMode(),
/*IsTruncating*/ true, N->isCompressingStore());
}
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
unsigned OpNo) {
assert(OpNo == 3 && "Only know how to promote the mask!");
EVT DataVT = N->getValueType(0);
SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
NewOps[OpNo] = Mask;
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
if (Res == N)
return SDValue(Res, 0);
// Update triggered CSE, do our own replacement since caller can't.
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
return SDValue();
}
SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
unsigned OpNo) {
SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
if (OpNo == 2) {
// The Mask
EVT DataVT = N->getValueType(0);
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
} else if (OpNo == 4) {
// The Index
if (N->isIndexSigned())
// Need to sign extend the index since the bits will likely be used.
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
else
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
} else
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
if (Res == N)
return SDValue(Res, 0);
// Update triggered CSE, do our own replacement since caller can't.
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
return SDValue();
}
SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
unsigned OpNo) {
bool TruncateStore = N->isTruncatingStore();
SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
if (OpNo == 2) {
// The Mask
EVT DataVT = N->getValue().getValueType();
NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
} else if (OpNo == 4) {
// The Index
if (N->isIndexSigned())
// Need to sign extend the index since the bits will likely be used.
NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
else
NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
} else {
NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
TruncateStore = true;
}
return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
SDLoc(N), NewOps, N->getMemOperand(),
N->getIndexType(), TruncateStore);
}
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
if (N->getOpcode() == ISD::VP_TRUNCATE)
return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
N->getOperand(1), N->getOperand(2));
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
}
SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
if (N->getOpcode() == ISD::VP_UINT_TO_FP)
return SDValue(DAG.UpdateNodeOperands(N,
ZExtPromotedInteger(N->getOperand(0)),
N->getOperand(1), N->getOperand(2)),
0);
return SDValue(DAG.UpdateNodeOperands(N,
ZExtPromotedInteger(N->getOperand(0))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
ZExtPromotedInteger(N->getOperand(1))), 0);
}
SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {