//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This implements the TargetLowering class.
//
//===----------------------------------------------------------------------===//

#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/Support/DivisionByConstantInfo.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include <cctype>
#include <deque>
using namespace llvm;
using namespace llvm::SDPatternMatch;

/// NOTE: The TargetMachine owns TLOF.
TargetLowering::TargetLowering(const TargetMachine &tm,
                               const TargetSubtargetInfo &STI)
    : TargetLoweringBase(tm, STI) {}

// Define the virtual destructor out-of-line for build efficiency.
TargetLowering::~TargetLowering() = default;

const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
  return nullptr;
}

bool TargetLowering::isPositionIndependent() const {
  return getTargetMachine().isPositionIndependent();
}

/// Check whether a given call node is in tail position within its function. If
/// so, it sets Chain to the input chain of the tail call.
bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
                                          SDValue &Chain) const {
  const Function &F = DAG.getMachineFunction().getFunction();

  // First, check if tail calls have been disabled in this function.
  if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
    return false;

  // Conservatively require the attributes of the call to match those of
  // the return. Ignore following attributes because they don't affect the
  // call sequence.
  AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
                           Attribute::DereferenceableOrNull, Attribute::NoAlias,
                           Attribute::NonNull, Attribute::NoUndef,
                           Attribute::Range, Attribute::NoFPClass})
    CallerAttrs.removeAttribute(Attr);

  if (CallerAttrs.hasAttributes())
    return false;

  // It's not safe to eliminate the sign / zero extension of the return value.
  if (CallerAttrs.contains(Attribute::ZExt) ||
      CallerAttrs.contains(Attribute::SExt))
    return false;

  // Check if the only use is a function return node.
  return isUsedByReturnOnly(Node, Chain);
}

bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
    const uint32_t *CallerPreservedMask,
    const SmallVectorImpl<CCValAssign> &ArgLocs,
    const SmallVectorImpl<SDValue> &OutVals) const {
  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
    const CCValAssign &ArgLoc = ArgLocs[I];
    if (!ArgLoc.isRegLoc())
      continue;
    MCRegister Reg = ArgLoc.getLocReg();
    // Only look at callee saved registers.
    if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
      continue;
    // Check that we pass the value used for the caller.
    // (We look for a CopyFromReg reading a virtual register that is used
    //  for the function live-in value of register Reg)
    SDValue Value = OutVals[I];
    if (Value->getOpcode() == ISD::AssertZext)
      Value = Value.getOperand(0);
    if (Value->getOpcode() != ISD::CopyFromReg)
      return false;
    Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
    if (MRI.getLiveInPhysReg(ArgReg) != Reg)
      return false;
  }
  return true;
}

/// Set CallLoweringInfo attribute flags based on a call instruction
/// and called function attributes.
void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
                                                     unsigned ArgIdx) {
  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
  IsNoExt = Call->paramHasAttr(ArgIdx, Attribute::NoExt);
  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
  IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
  IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
  Alignment = Call->getParamStackAlign(ArgIdx);
  IndirectType = nullptr;
  assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 &&
         "multiple ABI attributes?");
  if (IsByVal) {
    IndirectType = Call->getParamByValType(ArgIdx);
    if (!Alignment)
      Alignment = Call->getParamAlign(ArgIdx);
  }
  if (IsPreallocated)
    IndirectType = Call->getParamPreallocatedType(ArgIdx);
  if (IsInAlloca)
    IndirectType = Call->getParamInAllocaType(ArgIdx);
  if (IsSRet)
    IndirectType = Call->getParamStructRetType(ArgIdx);
}

/// Generate a libcall taking the given operands as arguments and returning a
/// result of type RetVT.
std::pair<SDValue, SDValue>
TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
                            EVT RetVT, ArrayRef<SDValue> Ops,
                            MakeLibCallOptions CallOptions, const SDLoc &dl,
                            SDValue InChain) const {
  if (LibcallImpl == RTLIB::Unsupported)
    reportFatalInternalError("unsupported library call operation");

  if (!InChain)
    InChain = DAG.getEntryNode();

  TargetLowering::ArgListTy Args;
  Args.reserve(Ops.size());

  ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
  for (unsigned i = 0; i < Ops.size(); ++i) {
    SDValue NewOp = Ops[i];
    Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
                   ? OpsTypeOverrides[i]
                   : NewOp.getValueType().getTypeForEVT(*DAG.getContext());
    TargetLowering::ArgListEntry Entry(NewOp, Ty);
    if (CallOptions.IsSoften)
      Entry.OrigTy =
          CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(*DAG.getContext());

    Entry.IsSExt =
        shouldSignExtendTypeInLibCall(Entry.Ty, CallOptions.IsSigned);
    Entry.IsZExt = !Entry.IsSExt;

    if (CallOptions.IsSoften &&
        !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
      Entry.IsSExt = Entry.IsZExt = false;
    }
    Args.push_back(Entry);
  }

  SDValue Callee =
      DAG.getExternalSymbol(LibcallImpl, getPointerTy(DAG.getDataLayout()));

  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
  Type *OrigRetTy = RetTy;
  TargetLowering::CallLoweringInfo CLI(DAG);
  bool signExtend = shouldSignExtendTypeInLibCall(RetTy, CallOptions.IsSigned);
  bool zeroExtend = !signExtend;

  if (CallOptions.IsSoften) {
    OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(*DAG.getContext());
    if (!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften))
      signExtend = zeroExtend = false;
  }

  CLI.setDebugLoc(dl)
      .setChain(InChain)
      .setLibCallee(getLibcallImplCallingConv(LibcallImpl), RetTy, OrigRetTy,
                    Callee, std::move(Args))
      .setNoReturn(CallOptions.DoesNotReturn)
      .setDiscardResult(!CallOptions.IsReturnValueUsed)
      .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
      .setSExtResult(signExtend)
      .setZExtResult(zeroExtend);
  return LowerCallTo(CLI);
}

bool TargetLowering::findOptimalMemOpLowering(
    LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
    const MemOp &Op, unsigned DstAS, unsigned SrcAS,
    const AttributeList &FuncAttributes, EVT *LargestVT) const {
  EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);

  if (VT == MVT::Other) {
    // Use the largest integer type whose alignment constraints are satisfied.
    VT = MVT::LAST_INTEGER_VALUETYPE;
    if (Op.isFixedDstAlign()) {
      bool LoadsFromSrc = Op.isMemcpy() && !Op.isMemcpyStrSrc();
      while (VT != MVT::i8) {
        unsigned VTSize = VT.getSizeInBits() / 8;
        bool DstOk =
            Op.getDstAlign() >= VTSize ||
            allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign());
        bool SrcOk =
            !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
            allowsMisalignedMemoryAccesses(VT, SrcAS, Op.getSrcAlign());
        if (DstOk && SrcOk)
          break;
        VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
      }
    }
    assert(VT.isInteger());

    // Find the largest legal integer type.
    MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
    while (!isTypeLegal(LVT))
      LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
    assert(LVT.isInteger());

    // If the type we've chosen is larger than the largest legal integer type
    // then use the largest legal type.
    if (VT.bitsGT(LVT))
      VT = LVT;
  }

  unsigned NumMemOps = 0;
  uint64_t Size = Op.size();
  while (Size) {
    unsigned VTSize = VT.getSizeInBits() / 8;
    while (VTSize > Size) {
      // For now, only use non-vector load / store's for the left-over pieces.
      EVT NewVT = VT;
      unsigned NewVTSize;

      bool Found = false;
      if (VT.isVector() || VT.isFloatingPoint()) {
        NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
        if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
            isSafeMemOpType(NewVT.getSimpleVT()))
          Found = true;
        else if (NewVT == MVT::i64 &&
                 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
                 isSafeMemOpType(MVT::f64)) {
          // i64 is usually not legal on 32-bit targets, but f64 may be.
          NewVT = MVT::f64;
          Found = true;
        }
      }

      if (!Found) {
        do {
          NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
          if (NewVT == MVT::i8)
            break;
        } while (!isSafeMemOpType(NewVT.getSimpleVT()));
      }
      NewVTSize = NewVT.getSizeInBits() / 8;

      // If the new VT cannot cover all of the remaining bits, then consider
      // issuing a (or a pair of) unaligned and overlapping load / store.
      unsigned Fast;
      if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
          allowsMisalignedMemoryAccesses(
              VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
              MachineMemOperand::MONone, &Fast) &&
          Fast)
        VTSize = Size;
      else {
        VT = NewVT;
        VTSize = NewVTSize;
      }
    }

    if (++NumMemOps > Limit)
      return false;

    MemOps.push_back(VT);
    Size -= VTSize;
  }

  return true;
}

/// Soften the operands of a comparison. This code is shared among BR_CC,
/// SELECT_CC, and SETCC handlers.
void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
                                         SDValue &NewLHS, SDValue &NewRHS,
                                         ISD::CondCode &CCCode,
                                         const SDLoc &dl, const SDValue OldLHS,
                                         const SDValue OldRHS) const {
  SDValue Chain;
  return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
                             OldRHS, Chain);
}

void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
                                         SDValue &NewLHS, SDValue &NewRHS,
                                         ISD::CondCode &CCCode,
                                         const SDLoc &dl, const SDValue OldLHS,
                                         const SDValue OldRHS,
                                         SDValue &Chain,
                                         bool IsSignaling) const {
  // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
  // not supporting it. We can update this code when libgcc provides such
  // functions.

  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
         && "Unsupported setcc type!");

  // Expand into one or more soft-fp libcall(s).
  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
  bool ShouldInvertCC = false;
  switch (CCCode) {
  case ISD::SETEQ:
  case ISD::SETOEQ:
    LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
          (VT == MVT::f64) ? RTLIB::OEQ_F64 :
          (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
    break;
  case ISD::SETNE:
  case ISD::SETUNE:
    LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
          (VT == MVT::f64) ? RTLIB::UNE_F64 :
          (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
    break;
  case ISD::SETGE:
  case ISD::SETOGE:
    LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
          (VT == MVT::f64) ? RTLIB::OGE_F64 :
          (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
    break;
  case ISD::SETLT:
  case ISD::SETOLT:
    LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
          (VT == MVT::f64) ? RTLIB::OLT_F64 :
          (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
    break;
  case ISD::SETLE:
  case ISD::SETOLE:
    LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
          (VT == MVT::f64) ? RTLIB::OLE_F64 :
          (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
    break;
  case ISD::SETGT:
  case ISD::SETOGT:
    LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
          (VT == MVT::f64) ? RTLIB::OGT_F64 :
          (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
    break;
  case ISD::SETO:
    ShouldInvertCC = true;
    [[fallthrough]];
  case ISD::SETUO:
    LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
          (VT == MVT::f64) ? RTLIB::UO_F64 :
          (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
    break;
  case ISD::SETONE:
    // SETONE = O && UNE
    ShouldInvertCC = true;
    [[fallthrough]];
  case ISD::SETUEQ:
    LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
          (VT == MVT::f64) ? RTLIB::UO_F64 :
          (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
    LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
          (VT == MVT::f64) ? RTLIB::OEQ_F64 :
          (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
    break;
  default:
    // Invert CC for unordered comparisons
    ShouldInvertCC = true;
    switch (CCCode) {
    case ISD::SETULT:
      LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
            (VT == MVT::f64) ? RTLIB::OGE_F64 :
            (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
      break;
    case ISD::SETULE:
      LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
            (VT == MVT::f64) ? RTLIB::OGT_F64 :
            (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
      break;
    case ISD::SETUGT:
      LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
            (VT == MVT::f64) ? RTLIB::OLE_F64 :
            (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
      break;
    case ISD::SETUGE:
      LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
            (VT == MVT::f64) ? RTLIB::OLT_F64 :
            (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
      break;
    default: llvm_unreachable("Do not know how to soften this setcc!");
    }
  }

  // Use the target specific return value for comparison lib calls.
  EVT RetVT = getCmpLibcallReturnType();
  SDValue Ops[2] = {NewLHS, NewRHS};
  TargetLowering::MakeLibCallOptions CallOptions;
  EVT OpsVT[2] = { OldLHS.getValueType(),
                   OldRHS.getValueType() };
  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
  auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
  NewLHS = Call.first;
  NewRHS = DAG.getConstant(0, dl, RetVT);

  RTLIB::LibcallImpl LC1Impl = getLibcallImpl(LC1);
  if (LC1Impl == RTLIB::Unsupported) {
    reportFatalUsageError(
        "no libcall available to soften floating-point compare");
  }

  CCCode = getSoftFloatCmpLibcallPredicate(LC1Impl);
  if (ShouldInvertCC) {
    assert(RetVT.isInteger());
    CCCode = getSetCCInverse(CCCode, RetVT);
  }

  if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
    // Update Chain.
    Chain = Call.second;
  } else {
    RTLIB::LibcallImpl LC2Impl = getLibcallImpl(LC2);
    if (LC2Impl == RTLIB::Unsupported) {
      reportFatalUsageError(
          "no libcall available to soften floating-point compare");
    }

    assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
           "unordered call should be simple boolean");

    EVT SetCCVT =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
    if (getBooleanContents(RetVT) == ZeroOrOneBooleanContent) {
      NewLHS = DAG.getNode(ISD::AssertZext, dl, RetVT, Call.first,
                           DAG.getValueType(MVT::i1));
    }

    SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
    auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
    CCCode = getSoftFloatCmpLibcallPredicate(LC2Impl);
    if (ShouldInvertCC)
      CCCode = getSetCCInverse(CCCode, RetVT);
    NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
    if (Chain)
      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
                          Call2.second);
    NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
                         Tmp.getValueType(), Tmp, NewLHS);
    NewRHS = SDValue();
  }
}

/// Return the entry encoding for a jump table in the current function. The
/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
unsigned TargetLowering::getJumpTableEncoding() const {
  // In non-pic modes, just use the address of a block.
  if (!isPositionIndependent())
    return MachineJumpTableInfo::EK_BlockAddress;

  // Otherwise, use a label difference.
  return MachineJumpTableInfo::EK_LabelDifference32;
}

SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
                                                 SelectionDAG &DAG) const {
  return Table;
}

/// This returns the relocation base for the given PIC jumptable, the same as
/// getPICJumpTableRelocBase, but as an MCExpr.
const MCExpr *
TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                             unsigned JTI,MCContext &Ctx) const{
  // The normal PIC reloc base is the label at the start of the jump table.
  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
}

SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
                                               SDValue Addr, int JTI,
                                               SelectionDAG &DAG) const {
  SDValue Chain = Value;
  // Jump table debug info is only needed if CodeView is enabled.
  if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
    Chain = DAG.getJumpTableDebugInfo(JTI, Chain, dl);
  }
  return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr);
}

bool
TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
  const TargetMachine &TM = getTargetMachine();
  const GlobalValue *GV = GA->getGlobal();

  // If the address is not even local to this DSO we will have to load it from
  // a got and then add the offset.
  if (!TM.shouldAssumeDSOLocal(GV))
    return false;

  // If the code is position independent we will have to add a base register.
  if (isPositionIndependent())
    return false;

  // Otherwise we can do it.
  return true;
}

//===----------------------------------------------------------------------===//
//  Optimization Methods
//===----------------------------------------------------------------------===//

/// If the specified instruction has a constant integer operand and there are
/// bits set in that constant that are not demanded, then clear those bits and
/// return true.
bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
                                            const APInt &DemandedBits,
                                            const APInt &DemandedElts,
                                            TargetLoweringOpt &TLO) const {
  SDLoc DL(Op);
  unsigned Opcode = Op.getOpcode();

  // Early-out if we've ended up calling an undemanded node, leave this to
  // constant folding.
  if (DemandedBits.isZero() || DemandedElts.isZero())
    return false;

  // Do target-specific constant optimization.
  if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
    return TLO.New.getNode();

  // FIXME: ISD::SELECT, ISD::SELECT_CC
  switch (Opcode) {
  default:
    break;
  case ISD::XOR:
  case ISD::AND:
  case ISD::OR: {
    auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
    if (!Op1C || Op1C->isOpaque())
      return false;

    // If this is a 'not' op, don't touch it because that's a canonical form.
    const APInt &C = Op1C->getAPIntValue();
    if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
      return false;

    if (!C.isSubsetOf(DemandedBits)) {
      EVT VT = Op.getValueType();
      SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
      SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC,
                                      Op->getFlags());
      return TLO.CombineTo(Op, NewOp);
    }

    break;
  }
  }

  return false;
}

bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
                                            const APInt &DemandedBits,
                                            TargetLoweringOpt &TLO) const {
  EVT VT = Op.getValueType();
  APInt DemandedElts = VT.isVector()
                           ? APInt::getAllOnes(VT.getVectorNumElements())
                           : APInt(1, 1);
  return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
}

/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
/// but it could be generalized for targets with other types of implicit
/// widening casts.
bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
                                      const APInt &DemandedBits,
                                      TargetLoweringOpt &TLO) const {
  assert(Op.getNumOperands() == 2 &&
         "ShrinkDemandedOp only supports binary operators!");
  assert(Op.getNode()->getNumValues() == 1 &&
         "ShrinkDemandedOp only supports nodes with one result!");

  EVT VT = Op.getValueType();
  SelectionDAG &DAG = TLO.DAG;
  SDLoc dl(Op);

  // Early return, as this function cannot handle vector types.
  if (VT.isVector())
    return false;

  assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
         Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
         "ShrinkDemandedOp only supports operands that have the same size!");

  // Don't do this if the node has another user, which may require the
  // full value.
  if (!Op.getNode()->hasOneUse())
    return false;

  // Search for the smallest integer type with free casts to and from
  // Op's type. For expedience, just check power-of-2 integer types.
  unsigned DemandedSize = DemandedBits.getActiveBits();
  for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
       SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
    EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
    if (isTruncateFree(Op, SmallVT) && isZExtFree(SmallVT, VT)) {
      // We found a type with free casts.

      // If the operation has the 'disjoint' flag, then the
      // operands on the new node are also disjoint.
      SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
                                                     : SDNodeFlags::None);
      unsigned Opcode = Op.getOpcode();
      if (Opcode == ISD::PTRADD) {
        // It isn't a ptradd anymore if it doesn't operate on the entire
        // pointer.
        Opcode = ISD::ADD;
      }
      SDValue X = DAG.getNode(
          Opcode, dl, SmallVT,
          DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
          DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)), Flags);
      assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
      SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X);
      return TLO.CombineTo(Op, Z);
    }
  }
  return false;
}

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                          DAGCombinerInfo &DCI) const {
  SelectionDAG &DAG = DCI.DAG;
  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                        !DCI.isBeforeLegalizeOps());
  KnownBits Known;

  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
  if (Simplified) {
    DCI.AddToWorklist(Op.getNode());
    DCI.CommitTargetLoweringOpt(TLO);
  }
  return Simplified;
}

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                          const APInt &DemandedElts,
                                          DAGCombinerInfo &DCI) const {
  SelectionDAG &DAG = DCI.DAG;
  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                        !DCI.isBeforeLegalizeOps());
  KnownBits Known;

  bool Simplified =
      SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
  if (Simplified) {
    DCI.AddToWorklist(Op.getNode());
    DCI.CommitTargetLoweringOpt(TLO);
  }
  return Simplified;
}

bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                          KnownBits &Known,
                                          TargetLoweringOpt &TLO,
                                          unsigned Depth,
                                          bool AssumeSingleUse) const {
  EVT VT = Op.getValueType();

  // Since the number of lanes in a scalable vector is unknown at compile time,
  // we track one bit which is implicitly broadcast to all lanes.  This means
  // that all lanes in a scalable vector are considered demanded.
  APInt DemandedElts = VT.isFixedLengthVector()
                           ? APInt::getAllOnes(VT.getVectorNumElements())
                           : APInt(1, 1);
  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
                              AssumeSingleUse);
}

// TODO: Under what circumstances can we create nodes? Constant folding?
SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
    SelectionDAG &DAG, unsigned Depth) const {
  EVT VT = Op.getValueType();

  // Limit search depth.
  if (Depth >= SelectionDAG::MaxRecursionDepth)
    return SDValue();

  // Ignore UNDEFs.
  if (Op.isUndef())
    return SDValue();

  // Not demanding any bits/elts from Op.
  if (DemandedBits == 0 || DemandedElts == 0)
    return DAG.getUNDEF(VT);

  bool IsLE = DAG.getDataLayout().isLittleEndian();
  unsigned NumElts = DemandedElts.getBitWidth();
  unsigned BitWidth = DemandedBits.getBitWidth();
  KnownBits LHSKnown, RHSKnown;
  switch (Op.getOpcode()) {
  case ISD::BITCAST: {
    if (VT.isScalableVector())
      return SDValue();

    SDValue Src = peekThroughBitcasts(Op.getOperand(0));
    EVT SrcVT = Src.getValueType();
    EVT DstVT = Op.getValueType();
    if (SrcVT == DstVT)
      return Src;

    unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
    unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
    if (NumSrcEltBits == NumDstEltBits)
      if (SDValue V = SimplifyMultipleUseDemandedBits(
              Src, DemandedBits, DemandedElts, DAG, Depth + 1))
        return DAG.getBitcast(DstVT, V);

    if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
      unsigned Scale = NumDstEltBits / NumSrcEltBits;
      unsigned NumSrcElts = SrcVT.getVectorNumElements();
      APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
      for (unsigned i = 0; i != Scale; ++i) {
        unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
        unsigned BitOffset = EltOffset * NumSrcEltBits;
        DemandedSrcBits |= DemandedBits.extractBits(NumSrcEltBits, BitOffset);
      }
      // Recursive calls below may turn not demanded elements into poison, so we
      // need to demand all smaller source elements that maps to a demanded
      // destination element.
      APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);

      if (SDValue V = SimplifyMultipleUseDemandedBits(
              Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
        return DAG.getBitcast(DstVT, V);
    }

    // TODO - bigendian once we have test coverage.
    if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
      unsigned Scale = NumSrcEltBits / NumDstEltBits;
      unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
      APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
      APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
      for (unsigned i = 0; i != NumElts; ++i)
        if (DemandedElts[i]) {
          unsigned Offset = (i % Scale) * NumDstEltBits;
          DemandedSrcBits.insertBits(DemandedBits, Offset);
          DemandedSrcElts.setBit(i / Scale);
        }

      if (SDValue V = SimplifyMultipleUseDemandedBits(
              Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
        return DAG.getBitcast(DstVT, V);
    }

    break;
  }
  case ISD::AND: {
    LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
    RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

    // If all of the demanded bits are known 1 on one side, return the other.
    // These bits cannot contribute to the result of the 'and' in this
    // context.
    if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
      return Op.getOperand(0);
    if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
      return Op.getOperand(1);
    break;
  }
  case ISD::OR: {
    LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
    RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

    // If all of the demanded bits are known zero on one side, return the
    // other.  These bits cannot contribute to the result of the 'or' in this
    // context.
    if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
      return Op.getOperand(0);
    if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
      return Op.getOperand(1);
    break;
  }
  case ISD::XOR: {
    LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
    RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

    // If all of the demanded bits are known zero on one side, return the
    // other.
    if (DemandedBits.isSubsetOf(RHSKnown.Zero))
      return Op.getOperand(0);
    if (DemandedBits.isSubsetOf(LHSKnown.Zero))
      return Op.getOperand(1);
    break;
  }
  case ISD::ADD:
  case ISD::MUL:
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX: {
    if (DAG.isIdentityElement(Op.getOpcode(), Op->getFlags(), Op.getOperand(1),
                              DemandedElts, 1, Depth + 1))
      return Op.getOperand(0);

    if (DAG.isIdentityElement(Op.getOpcode(), Op->getFlags(), Op.getOperand(0),
                              DemandedElts, 0, Depth + 1))
      return Op.getOperand(1);
    break;
  }
  case ISD::SHL: {
    // If we are only demanding sign bits then we can use the shift source
    // directly.
    if (std::optional<unsigned> MaxSA =
            DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
      SDValue Op0 = Op.getOperand(0);
      unsigned ShAmt = *MaxSA;
      unsigned NumSignBits =
          DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
      unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
      if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
        return Op0;
    }
    break;
  }
  case ISD::SRL: {
    // If we are only demanding sign bits then we can use the shift source
    // directly.
    if (std::optional<unsigned> MaxSA =
            DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
      SDValue Op0 = Op.getOperand(0);
      unsigned ShAmt = *MaxSA;
      // Must already be signbits in DemandedBits bounds, and can't demand any
      // shifted in zeroes.
      if (DemandedBits.countl_zero() >= ShAmt) {
        unsigned NumSignBits =
            DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
        if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
          return Op0;
      }
    }
    break;
  }
  case ISD::SETCC: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
    // If (1) we only need the sign-bit, (2) the setcc operands are the same
    // width as the setcc result, and (3) the result of a setcc conforms to 0 or
    // -1, we may be able to bypass the setcc.
    if (DemandedBits.isSignMask() &&
        Op0.getScalarValueSizeInBits() == BitWidth &&
        getBooleanContents(Op0.getValueType()) ==
            BooleanContent::ZeroOrNegativeOneBooleanContent) {
      // If we're testing X < 0, then this compare isn't needed - just use X!
      // FIXME: We're limiting to integer types here, but this should also work
      // if we don't care about FP signed-zero. The use of SETLT with FP means
      // that we don't care about NaNs.
      if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
          (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
        return Op0;
    }
    break;
  }
  case ISD::SIGN_EXTEND_INREG: {
    // If none of the extended bits are demanded, eliminate the sextinreg.
    SDValue Op0 = Op.getOperand(0);
    EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    unsigned ExBits = ExVT.getScalarSizeInBits();
    if (DemandedBits.getActiveBits() <= ExBits &&
        shouldRemoveRedundantExtend(Op))
      return Op0;
    // If the input is already sign extended, just drop the extension.
    unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
    if (NumSignBits >= (BitWidth - ExBits + 1))
      return Op0;
    break;
  }
  case ISD::ANY_EXTEND_VECTOR_INREG:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
  case ISD::ZERO_EXTEND_VECTOR_INREG: {
    if (VT.isScalableVector())
      return SDValue();

    // If we only want the lowest element and none of extended bits, then we can
    // return the bitcasted source vector.
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    EVT DstVT = Op.getValueType();
    if (IsLE && DemandedElts == 1 &&
        DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
        DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
      return DAG.getBitcast(DstVT, Src);
    }
    break;
  }
  case ISD::INSERT_VECTOR_ELT: {
    if (VT.isScalableVector())
      return SDValue();

    // If we don't demand the inserted element, return the base vector.
    SDValue Vec = Op.getOperand(0);
    auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
    EVT VecVT = Vec.getValueType();
    if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
        !DemandedElts[CIdx->getZExtValue()])
      return Vec;
    break;
  }
  case ISD::INSERT_SUBVECTOR: {
    if (VT.isScalableVector())
      return SDValue();

    SDValue Vec = Op.getOperand(0);
    SDValue Sub = Op.getOperand(1);
    uint64_t Idx = Op.getConstantOperandVal(2);
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
    APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
    // If we don't demand the inserted subvector, return the base vector.
    if (DemandedSubElts == 0)
      return Vec;
    break;
  }
  case ISD::VECTOR_SHUFFLE: {
    assert(!VT.isScalableVector());
    ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

    // If all the demanded elts are from one operand and are inline,
    // then we can use the operand directly.
    bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
    for (unsigned i = 0; i != NumElts; ++i) {
      int M = ShuffleMask[i];
      if (M < 0 || !DemandedElts[i])
        continue;
      AllUndef = false;
      IdentityLHS &= (M == (int)i);
      IdentityRHS &= ((M - NumElts) == i);
    }

    if (AllUndef)
      return DAG.getUNDEF(Op.getValueType());
    if (IdentityLHS)
      return Op.getOperand(0);
    if (IdentityRHS)
      return Op.getOperand(1);
    break;
  }
  default:
    // TODO: Probably okay to remove after audit; here to reduce change size
    // in initial enablement patch for scalable vectors
    if (VT.isScalableVector())
      return SDValue();

    if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
      if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
              Op, DemandedBits, DemandedElts, DAG, Depth))
        return V;
    break;
  }
  return SDValue();
}

SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
    SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
    unsigned Depth) const {
  EVT VT = Op.getValueType();
  // Since the number of lanes in a scalable vector is unknown at compile time,
  // we track one bit which is implicitly broadcast to all lanes.  This means
  // that all lanes in a scalable vector are considered demanded.
  APInt DemandedElts = VT.isFixedLengthVector()
                           ? APInt::getAllOnes(VT.getVectorNumElements())
                           : APInt(1, 1);
  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
                                         Depth);
}

SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
    SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
    unsigned Depth) const {
  APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
                                         Depth);
}

// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
//      or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
static SDValue combineShiftToAVG(SDValue Op,
                                 TargetLowering::TargetLoweringOpt &TLO,
                                 const TargetLowering &TLI,
                                 const APInt &DemandedBits,
                                 const APInt &DemandedElts, unsigned Depth) {
  assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
         "SRL or SRA node is required here!");
  // Is the right shift using an immediate value of 1?
  ConstantSDNode *N1C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
  if (!N1C || !N1C->isOne())
    return SDValue();

  // We are looking for an avgfloor
  // add(ext, ext)
  // or one of these as a avgceil
  // add(add(ext, ext), 1)
  // add(add(ext, 1), ext)
  // add(ext, add(ext, 1))
  SDValue Add = Op.getOperand(0);
  if (Add.getOpcode() != ISD::ADD)
    return SDValue();

  SDValue ExtOpA = Add.getOperand(0);
  SDValue ExtOpB = Add.getOperand(1);
  SDValue Add2;
  auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
    ConstantSDNode *ConstOp;
    if ((ConstOp = isConstOrConstSplat(Op2, DemandedElts)) &&
        ConstOp->isOne()) {
      ExtOpA = Op1;
      ExtOpB = Op3;
      Add2 = A;
      return true;
    }
    if ((ConstOp = isConstOrConstSplat(Op3, DemandedElts)) &&
        ConstOp->isOne()) {
      ExtOpA = Op1;
      ExtOpB = Op2;
      Add2 = A;
      return true;
    }
    return false;
  };
  bool IsCeil =
      (ExtOpA.getOpcode() == ISD::ADD &&
       MatchOperands(ExtOpA.getOperand(0), ExtOpA.getOperand(1), ExtOpB, ExtOpA)) ||
      (ExtOpB.getOpcode() == ISD::ADD &&
       MatchOperands(ExtOpB.getOperand(0), ExtOpB.getOperand(1), ExtOpA, ExtOpB));

  // If the shift is signed (sra):
  //  - Needs >= 2 sign bit for both operands.
  //  - Needs >= 2 zero bits.
  // If the shift is unsigned (srl):
  //  - Needs >= 1 zero bit for both operands.
  //  - Needs 1 demanded bit zero and >= 2 sign bits.
  SelectionDAG &DAG = TLO.DAG;
  unsigned ShiftOpc = Op.getOpcode();
  bool IsSigned = false;
  unsigned KnownBits;
  unsigned NumSignedA = DAG.ComputeNumSignBits(ExtOpA, DemandedElts, Depth);
  unsigned NumSignedB = DAG.ComputeNumSignBits(ExtOpB, DemandedElts, Depth);
  unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
  unsigned NumZeroA =
      DAG.computeKnownBits(ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
  unsigned NumZeroB =
      DAG.computeKnownBits(ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
  unsigned NumZero = std::min(NumZeroA, NumZeroB);

  switch (ShiftOpc) {
  default:
    llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
  case ISD::SRA: {
    if (NumZero >= 2 && NumSigned < NumZero) {
      IsSigned = false;
      KnownBits = NumZero;
      break;
    }
    if (NumSigned >= 1) {
      IsSigned = true;
      KnownBits = NumSigned;
      break;
    }
    return SDValue();
  }
  case ISD::SRL: {
    if (NumZero >= 1 && NumSigned < NumZero) {
      IsSigned = false;
      KnownBits = NumZero;
      break;
    }
    if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
      IsSigned = true;
      KnownBits = NumSigned;
      break;
    }
    return SDValue();
  }
  }

  unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
                           : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);

  // Find the smallest power-2 type that is legal for this vector size and
  // operation, given the original type size and the number of known sign/zero
  // bits.
  EVT VT = Op.getValueType();
  unsigned MinWidth =
      std::max<unsigned>(VT.getScalarSizeInBits() - KnownBits, 8);
  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), llvm::bit_ceil(MinWidth));
  if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
    return SDValue();
  if (VT.isVector())
    NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount());
  if (TLO.LegalTypes() && !TLI.isOperationLegal(AVGOpc, NVT)) {
    // If we could not transform, and (both) adds are nuw/nsw, we can use the
    // larger type size to do the transform.
    if (TLO.LegalOperations() && !TLI.isOperationLegal(AVGOpc, VT))
      return SDValue();
    if (DAG.willNotOverflowAdd(IsSigned, Add.getOperand(0),
                               Add.getOperand(1)) &&
        (!Add2 || DAG.willNotOverflowAdd(IsSigned, Add2.getOperand(0),
                                         Add2.getOperand(1))))
      NVT = VT;
    else
      return SDValue();
  }

  // Don't create a AVGFLOOR node with a scalar constant unless its legal as
  // this is likely to stop other folds (reassociation, value tracking etc.)
  if (!IsCeil && !TLI.isOperationLegal(AVGOpc, NVT) &&
      (isa<ConstantSDNode>(ExtOpA) || isa<ConstantSDNode>(ExtOpB)))
    return SDValue();

  SDLoc DL(Op);
  SDValue ResultAVG =
      DAG.getNode(AVGOpc, DL, NVT, DAG.getExtOrTrunc(IsSigned, ExtOpA, DL, NVT),
                  DAG.getExtOrTrunc(IsSigned, ExtOpB, DL, NVT));
  return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT);
}

/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
/// result of Op are ever used downstream. If we can use this information to
/// simplify Op, create a new simplified DAG node and return true, returning the
/// original and new nodes in Old and New. Otherwise, analyze the expression and
/// return a mask of Known bits for the expression (used to simplify the
/// caller).  The Known bits may only be accurate for those bits in the
/// OriginalDemandedBits and OriginalDemandedElts.
bool TargetLowering::SimplifyDemandedBits(
    SDValue Op, const APInt &OriginalDemandedBits,
    const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
    unsigned Depth, bool AssumeSingleUse) const {
  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
  assert(Op.getScalarValueSizeInBits() == BitWidth &&
         "Mask size mismatches value type size!");

  // Don't know anything.
  Known = KnownBits(BitWidth);

  EVT VT = Op.getValueType();
  bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
  unsigned NumElts = OriginalDemandedElts.getBitWidth();
  assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
         "Unexpected vector size");

  APInt DemandedBits = OriginalDemandedBits;
  APInt DemandedElts = OriginalDemandedElts;
  SDLoc dl(Op);

  // Undef operand.
  if (Op.isUndef())
    return false;

  // We can't simplify target constants.
  if (Op.getOpcode() == ISD::TargetConstant)
    return false;

  if (Op.getOpcode() == ISD::Constant) {
    // We know all of the bits for a constant!
    Known = KnownBits::makeConstant(Op->getAsAPIntVal());
    return false;
  }

  if (Op.getOpcode() == ISD::ConstantFP) {
    // We know all of the bits for a floating point constant!
    Known = KnownBits::makeConstant(
        cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
    return false;
  }

  // Other users may use these bits.
  bool HasMultiUse = false;
  if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
    if (Depth >= SelectionDAG::MaxRecursionDepth) {
      // Limit search depth.
      return false;
    }
    // Allow multiple uses, just set the DemandedBits/Elts to all bits.
    DemandedBits = APInt::getAllOnes(BitWidth);
    DemandedElts = APInt::getAllOnes(NumElts);
    HasMultiUse = true;
  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
    // Not demanding any bits/elts from Op.
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
  } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
    // Limit search depth.
    return false;
  }

  KnownBits Known2;
  switch (Op.getOpcode()) {
  case ISD::SCALAR_TO_VECTOR: {
    if (VT.isScalableVector())
      return false;
    if (!DemandedElts[0])
      return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

    KnownBits SrcKnown;
    SDValue Src = Op.getOperand(0);
    unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
    APInt SrcDemandedBits = DemandedBits.zext(SrcBitWidth);
    if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
      return true;

    // Upper elements are undef, so only get the knownbits if we just demand
    // the bottom element.
    if (DemandedElts == 1)
      Known = SrcKnown.anyextOrTrunc(BitWidth);
    break;
  }
  case ISD::BUILD_VECTOR:
    // Collect the known bits that are shared by every demanded element.
    // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false; // Don't fall through, will infinitely loop.
  case ISD::SPLAT_VECTOR: {
    SDValue Scl = Op.getOperand(0);
    APInt DemandedSclBits = DemandedBits.zextOrTrunc(Scl.getValueSizeInBits());
    KnownBits KnownScl;
    if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
      return true;

    // Implicitly truncate the bits to match the official semantics of
    // SPLAT_VECTOR.
    Known = KnownScl.trunc(BitWidth);
    break;
  }
  case ISD::FREEZE: {
    SDValue N0 = Op.getOperand(0);
    if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
            N0, DemandedElts, UndefPoisonKind::UndefOrPoison, Depth + 1))
      return TLO.CombineTo(Op, N0);
    break;
  }
  case ISD::LOAD: {
    auto *LD = cast<LoadSDNode>(Op);
    if (getTargetConstantFromLoad(LD)) {
      Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
      return false; // Don't fall through, will infinitely loop.
    }
    if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
      // If this is a ZEXTLoad and we are looking at the loaded value.
      EVT MemVT = LD->getMemoryVT();
      unsigned MemBits = MemVT.getScalarSizeInBits();
      Known.Zero.setBitsFrom(MemBits);
      return false; // Don't fall through, will infinitely loop.
    }
    break;
  }
  case ISD::INSERT_VECTOR_ELT: {
    if (VT.isScalableVector())
      return false;
    SDValue Vec = Op.getOperand(0);
    SDValue Scl = Op.getOperand(1);
    auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
    EVT VecVT = Vec.getValueType();

    // If index isn't constant, assume we need all vector elements AND the
    // inserted element.
    APInt DemandedVecElts(DemandedElts);
    if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
      unsigned Idx = CIdx->getZExtValue();
      DemandedVecElts.clearBit(Idx);

      // Inserted element is not required.
      if (!DemandedElts[Idx])
        return TLO.CombineTo(Op, Vec);
    }

    KnownBits KnownScl;
    unsigned NumSclBits = Scl.getScalarValueSizeInBits();
    APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
    if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
      return true;

    Known = KnownScl.anyextOrTrunc(BitWidth);

    KnownBits KnownVec;
    if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
                             Depth + 1))
      return true;

    if (!!DemandedVecElts)
      Known = Known.intersectWith(KnownVec);

    return false;
  }
  case ISD::INSERT_SUBVECTOR: {
    if (VT.isScalableVector())
      return false;
    // Demand any elements from the subvector and the remainder from the src its
    // inserted into.
    SDValue Src = Op.getOperand(0);
    SDValue Sub = Op.getOperand(1);
    uint64_t Idx = Op.getConstantOperandVal(2);
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
    APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
    APInt DemandedSrcElts = DemandedElts;
    DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);

    KnownBits KnownSub, KnownSrc;
    if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
                             Depth + 1))
      return true;
    if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
                             Depth + 1))
      return true;

    Known.setAllConflict();
    if (!!DemandedSubElts)
      Known = Known.intersectWith(KnownSub);
    if (!!DemandedSrcElts)
      Known = Known.intersectWith(KnownSrc);

    // Attempt to avoid multi-use src if we don't need anything from it.
    if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
        !DemandedSrcElts.isAllOnes()) {
      SDValue NewSub = SimplifyMultipleUseDemandedBits(
          Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
      SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
      if (NewSub || NewSrc) {
        NewSub = NewSub ? NewSub : Sub;
        NewSrc = NewSrc ? NewSrc : Src;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
                                        Op.getOperand(2));
        return TLO.CombineTo(Op, NewOp);
      }
    }
    break;
  }
  case ISD::EXTRACT_SUBVECTOR: {
    if (VT.isScalableVector())
      return false;
    // Offset the demanded elts by the subvector index.
    SDValue Src = Op.getOperand(0);
    if (Src.getValueType().isScalableVector())
      break;
    uint64_t Idx = Op.getConstantOperandVal(1);
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);

    if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
                             Depth + 1))
      return true;

    // Attempt to avoid multi-use src if we don't need anything from it.
    if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
      SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
          Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
      if (DemandedSrc) {
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
                                        Op.getOperand(1));
        return TLO.CombineTo(Op, NewOp);
      }
    }
    break;
  }
  case ISD::CONCAT_VECTORS: {
    if (VT.isScalableVector())
      return false;
    Known.setAllConflict();
    EVT SubVT = Op.getOperand(0).getValueType();
    unsigned NumSubVecs = Op.getNumOperands();
    unsigned NumSubElts = SubVT.getVectorNumElements();
    for (unsigned i = 0; i != NumSubVecs; ++i) {
      APInt DemandedSubElts =
          DemandedElts.extractBits(NumSubElts, i * NumSubElts);
      if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
                               Known2, TLO, Depth + 1))
        return true;
      // Known bits are shared by every demanded subvector element.
      if (!!DemandedSubElts)
        Known = Known.intersectWith(Known2);
    }
    break;
  }
  case ISD::VECTOR_SHUFFLE: {
    assert(!VT.isScalableVector());
    ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

    // Collect demanded elements from shuffle operands..
    APInt DemandedLHS, DemandedRHS;
    if (!getShuffleDemandedElts(NumElts, ShuffleMask, DemandedElts, DemandedLHS,
                                DemandedRHS))
      break;

    if (!!DemandedLHS || !!DemandedRHS) {
      SDValue Op0 = Op.getOperand(0);
      SDValue Op1 = Op.getOperand(1);

      Known.setAllConflict();
      if (!!DemandedLHS) {
        if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
                                 Depth + 1))
          return true;
        Known = Known.intersectWith(Known2);
      }
      if (!!DemandedRHS) {
        if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
                                 Depth + 1))
          return true;
        Known = Known.intersectWith(Known2);
      }

      // Attempt to avoid multi-use ops if we don't need anything from them.
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
      SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
          Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
      if (DemandedOp0 || DemandedOp1) {
        Op0 = DemandedOp0 ? DemandedOp0 : Op0;
        Op1 = DemandedOp1 ? DemandedOp1 : Op1;
        SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
        return TLO.CombineTo(Op, NewOp);
      }
    }
    break;
  }
  case ISD::AND: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    // If the RHS is a constant, check to see if the LHS would be zero without
    // using the bits from the RHS.  Below, we use knowledge about the RHS to
    // simplify the LHS, here we're using information from the LHS to simplify
    // the RHS.
    if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1, DemandedElts)) {
      // Do not increment Depth here; that can cause an infinite loop.
      KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
      // If the LHS already has zeros where RHSC does, this 'and' is dead.
      if ((LHSKnown.Zero & DemandedBits) ==
          (~RHSC->getAPIntValue() & DemandedBits))
        return TLO.CombineTo(Op, Op0);

      // If any of the set bits in the RHS are known zero on the LHS, shrink
      // the constant.
      if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
                                 DemandedElts, TLO))
        return true;

      // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
      // constant, but if this 'and' is only clearing bits that were just set by
      // the xor, then this 'and' can be eliminated by shrinking the mask of
      // the xor. For example, for a 32-bit X:
      // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
      if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
          LHSKnown.One == ~RHSC->getAPIntValue()) {
        SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
        return TLO.CombineTo(Op, Xor);
      }
    }

    // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
    SDValue X, Y;
    if (sd_match(Op,
                 m_And(m_Value(Y),
                       m_OneUse(m_AnyOf(m_Add(m_Value(X), m_Deferred(Y)),
                                        m_Sub(m_Value(X), m_Deferred(Y)))))) &&
        TLO.DAG.isKnownToBeAPowerOfTwo(Y, DemandedElts, /*OrZero=*/true)) {
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::AND, dl, VT, TLO.DAG.getNOT(dl, X, VT), Y));
    }

    // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
    // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
    if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
        (Op0.getOperand(0).isUndef() ||
         ISD::isBuildVectorOfConstantSDNodes(Op0.getOperand(0).getNode())) &&
        Op0->hasOneUse()) {
      unsigned NumSubElts =
          Op0.getOperand(1).getValueType().getVectorNumElements();
      unsigned SubIdx = Op0.getConstantOperandVal(2);
      APInt DemandedSub =
          APInt::getBitsSet(NumElts, SubIdx, SubIdx + NumSubElts);
      KnownBits KnownSubMask =
          TLO.DAG.computeKnownBits(Op1, DemandedSub & DemandedElts, Depth + 1);
      if (DemandedBits.isSubsetOf(KnownSubMask.One)) {
        SDValue NewAnd =
            TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1);
        SDValue NewInsert =
            TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd,
                            Op0.getOperand(1), Op0.getOperand(2));
        return TLO.CombineTo(Op, NewInsert);
      }
    }

    if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
                             Known2, TLO, Depth + 1))
      return true;

    // If all of the demanded bits are known one on one side, return the other.
    // These bits cannot contribute to the result of the 'and'.
    if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
      return TLO.CombineTo(Op, Op0);
    if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
      return TLO.CombineTo(Op, Op1);
    // If all of the demanded bits in the inputs are known zeros, return zero.
    if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
    // If the RHS is a constant, see if we can simplify it.
    if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
                               TLO))
      return true;
    // If the operation can be done in a smaller type, do so.
    if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
      return true;

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
          Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      if (DemandedOp0 || DemandedOp1) {
        Op0 = DemandedOp0 ? DemandedOp0 : Op0;
        Op1 = DemandedOp1 ? DemandedOp1 : Op1;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    Known &= Known2;
    break;
  }
  case ISD::OR: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                             Depth + 1)) {
      Op->dropFlags(SDNodeFlags::Disjoint);
      return true;
    }

    if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
                             Known2, TLO, Depth + 1)) {
      Op->dropFlags(SDNodeFlags::Disjoint);
      return true;
    }

    // If all of the demanded bits are known zero on one side, return the other.
    // These bits cannot contribute to the result of the 'or'.
    if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
      return TLO.CombineTo(Op, Op0);
    if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
      return TLO.CombineTo(Op, Op1);
    // If the RHS is a constant, see if we can simplify it.
    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
      return true;
    // If the operation can be done in a smaller type, do so.
    if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
      return true;

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
          Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      if (DemandedOp0 || DemandedOp1) {
        Op0 = DemandedOp0 ? DemandedOp0 : Op0;
        Op1 = DemandedOp1 ? DemandedOp1 : Op1;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
    // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
    if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND &&
        Op0->hasOneUse() && Op1->hasOneUse()) {
      // Attempt to match all commutations - m_c_Or would've been useful!
      for (int I = 0; I != 2; ++I) {
        SDValue X = Op.getOperand(I).getOperand(0);
        SDValue C1 = Op.getOperand(I).getOperand(1);
        SDValue Alt = Op.getOperand(1 - I).getOperand(0);
        SDValue C2 = Op.getOperand(1 - I).getOperand(1);
        if (Alt.getOpcode() == ISD::OR) {
          for (int J = 0; J != 2; ++J) {
            if (X == Alt.getOperand(J)) {
              SDValue Y = Alt.getOperand(1 - J);
              if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT,
                                                               {C1, C2})) {
                SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12);
                SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2);
                return TLO.CombineTo(
                    Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY));
              }
            }
          }
        }
      }
    }

    Known |= Known2;
    break;
  }
  case ISD::XOR: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;

    // If all of the demanded bits are known zero on one side, return the other.
    // These bits cannot contribute to the result of the 'xor'.
    if (DemandedBits.isSubsetOf(Known.Zero))
      return TLO.CombineTo(Op, Op0);
    if (DemandedBits.isSubsetOf(Known2.Zero))
      return TLO.CombineTo(Op, Op1);
    // If the operation can be done in a smaller type, do so.
    if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
      return true;

    // If all of the unknown bits are known to be zero on one side or the other
    // turn this into an *inclusive* or.
    //    e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
    if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

    ConstantSDNode *C = isConstOrConstSplat(Op1, DemandedElts);
    if (C) {
      // If one side is a constant, and all of the set bits in the constant are
      // also known set on the other side, turn this into an AND, as we know
      // the bits will be cleared.
      //    e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
      // NB: it is okay if more bits are known than are requested
      if (C->getAPIntValue() == Known2.One) {
        SDValue ANDC =
            TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
        return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
      }

      // If the RHS is a constant, see if we can change it. Don't alter a -1
      // constant because that's a 'not' op, and that is better for combining
      // and codegen.
      if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
        // We're flipping all demanded bits. Flip the undemanded bits too.
        SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
        return TLO.CombineTo(Op, New);
      }

      unsigned Op0Opcode = Op0.getOpcode();
      if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
        if (ConstantSDNode *ShiftC =
                isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
          // Don't crash on an oversized shift. We can not guarantee that a
          // bogus shift has been simplified to undef.
          if (ShiftC->getAPIntValue().ult(BitWidth)) {
            uint64_t ShiftAmt = ShiftC->getZExtValue();
            APInt Ones = APInt::getAllOnes(BitWidth);
            Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt)
                                         : Ones.lshr(ShiftAmt);
            if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
                isDesirableToCommuteXorWithShift(Op.getNode())) {
              // If the xor constant is a demanded mask, do a 'not' before the
              // shift:
              // xor (X << ShiftC), XorC --> (not X) << ShiftC
              // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
              SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT);
              return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not,
                                                       Op0.getOperand(1)));
            }
          }
        }
      }
    }

    // If we can't turn this into a 'not', try to shrink the constant.
    if (!C || !C->isAllOnes())
      if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
        return true;

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
          Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
      if (DemandedOp0 || DemandedOp1) {
        Op0 = DemandedOp0 ? DemandedOp0 : Op0;
        Op1 = DemandedOp1 ? DemandedOp1 : Op1;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    Known ^= Known2;
    break;
  }
  case ISD::SELECT:
    if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
                             Known, TLO, Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
                             Known2, TLO, Depth + 1))
      return true;

    // If the operands are constants, see if we can simplify them.
    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
      return true;

    // Only known if known in both the LHS and RHS.
    Known = Known.intersectWith(Known2);
    break;
  case ISD::VSELECT:
    if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
                             Known, TLO, Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts,
                             Known2, TLO, Depth + 1))
      return true;

    // Only known if known in both the LHS and RHS.
    Known = Known.intersectWith(Known2);
    break;
  case ISD::SELECT_CC:
    if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, DemandedElts,
                             Known, TLO, Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts,
                             Known2, TLO, Depth + 1))
      return true;

    // If the operands are constants, see if we can simplify them.
    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
      return true;

    // Only known if known in both the LHS and RHS.
    Known = Known.intersectWith(Known2);
    break;
  case ISD::SETCC: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
    // If we're testing X < 0, X >= 0, X <= -1 or X > -1
    // (X is of integer type) then we only need the sign mask of the previous
    // result
    if (Op1.getValueType().isInteger() &&
        (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(Op1)) ||
         ((CC == ISD::SETLE || CC == ISD::SETGT) &&
          isAllOnesOrAllOnesSplat(Op1)))) {
      KnownBits KnownOp0;
      if (SimplifyDemandedBits(
              Op0, APInt::getSignMask(Op0.getScalarValueSizeInBits()),
              DemandedElts, KnownOp0, TLO, Depth + 1))
        return true;
      // If (1) we only need the sign-bit, (2) the setcc operands are the same
      // width as the setcc result, and (3) the result of a setcc conforms to 0
      // or -1, we may be able to bypass the setcc.
      if (DemandedBits.isSignMask() &&
          Op0.getScalarValueSizeInBits() == BitWidth &&
          getBooleanContents(Op0.getValueType()) ==
              BooleanContent::ZeroOrNegativeOneBooleanContent) {
        // If we remove a >= 0 or > -1 (for integers), we need to introduce a
        // NOT Operation
        if (CC == ISD::SETGE || CC == ISD::SETGT) {
          SDLoc DL(Op);
          EVT VT = Op0.getValueType();
          SDValue NotOp0 = TLO.DAG.getNOT(DL, Op0, VT);
          return TLO.CombineTo(Op, NotOp0);
        }
        return TLO.CombineTo(Op, Op0);
      }
    }
    if (getBooleanContents(Op0.getValueType()) ==
            TargetLowering::ZeroOrOneBooleanContent &&
        BitWidth > 1)
      Known.Zero.setBitsFrom(1);
    break;
  }
  case ISD::SHL: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    EVT ShiftVT = Op1.getValueType();

    if (std::optional<unsigned> KnownSA =
            TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
      unsigned ShAmt = *KnownSA;
      if (ShAmt == 0)
        return TLO.CombineTo(Op, Op0);

      // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
      // single shift.  We can do this if the bottom bits (which are shifted
      // out) are never demanded.
      // TODO - support non-uniform vector amounts.
      if (Op0.getOpcode() == ISD::SRL) {
        if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
          if (std::optional<unsigned> InnerSA =
                  TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
            unsigned C1 = *InnerSA;
            unsigned Opc = ISD::SHL;
            int Diff = ShAmt - C1;
            if (Diff < 0) {
              Diff = -Diff;
              Opc = ISD::SRL;
            }
            SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
          }
        }
      }

      // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
      // are not demanded. This will likely allow the anyext to be folded away.
      // TODO - support non-uniform vector amounts.
      if (Op0.getOpcode() == ISD::ANY_EXTEND) {
        SDValue InnerOp = Op0.getOperand(0);
        EVT InnerVT = InnerOp.getValueType();
        unsigned InnerBits = InnerVT.getScalarSizeInBits();
        if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
            isTypeDesirableForOp(ISD::SHL, InnerVT)) {
          SDValue NarrowShl = TLO.DAG.getNode(
              ISD::SHL, dl, InnerVT, InnerOp,
              TLO.DAG.getShiftAmountConstant(ShAmt, InnerVT, dl));
          return TLO.CombineTo(
              Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
        }

        // Repeat the SHL optimization above in cases where an extension
        // intervenes: (shl (anyext (shr x, c1)), c2) to
        // (shl (anyext x), c2-c1).  This requires that the bottom c1 bits
        // aren't demanded (as above) and that the shifted upper c1 bits of
        // x aren't demanded.
        // TODO - support non-uniform vector amounts.
        if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
            InnerOp.hasOneUse()) {
          if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
                  InnerOp, DemandedElts, Depth + 2)) {
            unsigned InnerShAmt = *SA2;
            if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
                DemandedBits.getActiveBits() <=
                    (InnerBits - InnerShAmt + ShAmt) &&
                DemandedBits.countr_zero() >= ShAmt) {
              SDValue NewSA =
                  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
              SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
                                               InnerOp.getOperand(0));
              return TLO.CombineTo(
                  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
            }
          }
        }
      }

      APInt InDemandedMask = DemandedBits.lshr(ShAmt);
      if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                               Depth + 1)) {
        // Disable the nsw and nuw flags. We can no longer guarantee that we
        // won't wrap after simplification.
        Op->dropFlags(SDNodeFlags::NoWrap);
        return true;
      }
      Known <<= ShAmt;
      // low bits known zero.
      Known.Zero.setLowBits(ShAmt);

      // Attempt to avoid multi-use ops if we don't need anything from them.
      if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
        SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
            Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
        if (DemandedOp0) {
          SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1);
          return TLO.CombineTo(Op, NewOp);
        }
      }

      // TODO: Can we merge this fold with the one below?
      // Try shrinking the operation as long as the shift amount will still be
      // in range.
      if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
          Op.getNode()->hasOneUse()) {
        // Search for the smallest integer type with free casts to and from
        // Op's type. For expedience, just check power-of-2 integer types.
        unsigned DemandedSize = DemandedBits.getActiveBits();
        for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
             SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
          EVT SmallVT = EVT::getIntegerVT(*TLO.DAG.getContext(), SmallVTBits);
          if (isNarrowingProfitable(Op.getNode(), VT, SmallVT) &&
              isTypeDesirableForOp(ISD::SHL, SmallVT) &&
              isTruncateFree(VT, SmallVT) && isZExtFree(SmallVT, VT) &&
              (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, SmallVT))) {
            assert(DemandedSize <= SmallVTBits &&
                   "Narrowed below demanded bits?");
            // We found a type with free casts.
            SDValue NarrowShl = TLO.DAG.getNode(
                ISD::SHL, dl, SmallVT,
                TLO.DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
                TLO.DAG.getShiftAmountConstant(ShAmt, SmallVT, dl));
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
          }
        }
      }

      // Narrow shift to lower half - similar to ShrinkDemandedOp.
      // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
      // Only do this if we demand the upper half so the knownbits are correct.
      unsigned HalfWidth = BitWidth / 2;
      if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
          DemandedBits.countLeadingOnes() >= HalfWidth) {
        EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), HalfWidth);
        if (isNarrowingProfitable(Op.getNode(), VT, HalfVT) &&
            isTypeDesirableForOp(ISD::SHL, HalfVT) &&
            isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
            (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, HalfVT))) {
          // If we're demanding the upper bits at all, we must ensure
          // that the upper bits of the shift result are known to be zero,
          // which is equivalent to the narrow shift being NUW.
          if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
            bool IsNSW = Known.countMinSignBits() > HalfWidth;
            SDNodeFlags Flags;
            Flags.setNoSignedWrap(IsNSW);
            Flags.setNoUnsignedWrap(IsNUW);
            SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
            SDValue NewShiftAmt =
                TLO.DAG.getShiftAmountConstant(ShAmt, HalfVT, dl);
            SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp,
                                               NewShiftAmt, Flags);
            SDValue NewExt =
                TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift);
            return TLO.CombineTo(Op, NewExt);
          }
        }
      }
    } else {
      // This is a variable shift, so we can't shift the demand mask by a known
      // amount. But if we are not demanding high bits, then we are not
      // demanding those bits from the pre-shifted operand either.
      if (unsigned CTLZ = DemandedBits.countl_zero()) {
        APInt DemandedFromOp(APInt::getLowBitsSet(BitWidth, BitWidth - CTLZ));
        if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
                                 Depth + 1)) {
          // Disable the nsw and nuw flags. We can no longer guarantee that we
          // won't wrap after simplification.
          Op->dropFlags(SDNodeFlags::NoWrap);
          return true;
        }
        Known.resetAll();
      }
    }

    // If we are only demanding sign bits then we can use the shift source
    // directly.
    if (std::optional<unsigned> MaxSA =
            TLO.DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
      unsigned ShAmt = *MaxSA;
      unsigned NumSignBits =
          TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
      unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
      if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
        return TLO.CombineTo(Op, Op0);
    }
    break;
  }
  case ISD::SRL: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    EVT ShiftVT = Op1.getValueType();

    if (std::optional<unsigned> KnownSA =
            TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
      unsigned ShAmt = *KnownSA;
      if (ShAmt == 0)
        return TLO.CombineTo(Op, Op0);

      // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
      // single shift.  We can do this if the top bits (which are shifted out)
      // are never demanded.
      // TODO - support non-uniform vector amounts.
      if (Op0.getOpcode() == ISD::SHL) {
        if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
          if (std::optional<unsigned> InnerSA =
                  TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
            unsigned C1 = *InnerSA;
            unsigned Opc = ISD::SRL;
            int Diff = ShAmt - C1;
            if (Diff < 0) {
              Diff = -Diff;
              Opc = ISD::SHL;
            }
            SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
          }
        }
      }

      // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
      // single sra. We can do this if the top bits are never demanded.
      if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
        if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
          if (std::optional<unsigned> InnerSA =
                  TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
            unsigned C1 = *InnerSA;
            // Clamp the combined shift amount if it exceeds the bit width.
            unsigned Combined = std::min(C1 + ShAmt, BitWidth - 1);
            SDValue NewSA = TLO.DAG.getConstant(Combined, dl, ShiftVT);
            return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRA, dl, VT,
                                                     Op0.getOperand(0), NewSA));
          }
        }
      }

      APInt InDemandedMask = (DemandedBits << ShAmt);

      // If the shift is exact, then it does demand the low bits (and knows that
      // they are zero).
      if (Op->getFlags().hasExact())
        InDemandedMask.setLowBits(ShAmt);

      // Narrow shift to lower half - similar to ShrinkDemandedOp.
      // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
      if ((BitWidth % 2) == 0 && !VT.isVector()) {
        APInt HiBits = APInt::getHighBitsSet(BitWidth, BitWidth / 2);
        EVT HalfVT = EVT::getIntegerVT(*TLO.DAG.getContext(), BitWidth / 2);
        if (isNarrowingProfitable(Op.getNode(), VT, HalfVT) &&
            isTypeDesirableForOp(ISD::SRL, HalfVT) &&
            isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) &&
            (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) &&
            ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
             TLO.DAG.MaskedValueIsZero(Op0, HiBits))) {
          SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0);
          SDValue NewShiftAmt =
              TLO.DAG.getShiftAmountConstant(ShAmt, HalfVT, dl);
          SDValue NewShift =
              TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt);
          return TLO.CombineTo(
              Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift));
        }
      }

      // Compute the new bits that are at the top now.
      if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                               Depth + 1))
        return true;
      Known >>= ShAmt;
      // High bits known zero.
      Known.Zero.setHighBits(ShAmt);

      // Attempt to avoid multi-use ops if we don't need anything from them.
      if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
        SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
            Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
        if (DemandedOp0) {
          SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1);
          return TLO.CombineTo(Op, NewOp);
        }
      }
    } else {
      // Use generic knownbits computation as it has support for non-uniform
      // shift amounts.
      Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    }

    // If we are only demanding sign bits then we can use the shift source
    // directly.
    if (std::optional<unsigned> MaxSA =
            TLO.DAG.getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1)) {
      unsigned ShAmt = *MaxSA;
      // Must already be signbits in DemandedBits bounds, and can't demand any
      // shifted in zeroes.
      if (DemandedBits.countl_zero() >= ShAmt) {
        unsigned NumSignBits =
            TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
        if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
          return TLO.CombineTo(Op, Op0);
      }
    }

    // Try to match AVG patterns (after shift simplification).
    if (SDValue AVG = combineShiftToAVG(Op, TLO, *this, DemandedBits,
                                        DemandedElts, Depth + 1))
      return TLO.CombineTo(Op, AVG);

    break;
  }
  case ISD::SRA: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    EVT ShiftVT = Op1.getValueType();

    // If we only want bits that already match the signbit then we don't need
    // to shift.
    unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
    if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
        NumHiDemandedBits)
      return TLO.CombineTo(Op, Op0);

    // If this is an arithmetic shift right and only the low-bit is set, we can
    // always convert this into a logical shr, even if the shift amount is
    // variable.  The low bit of the shift cannot be an input sign bit unless
    // the shift amount is >= the size of the datatype, which is undefined.
    if (DemandedBits.isOne())
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

    if (std::optional<unsigned> KnownSA =
            TLO.DAG.getValidShiftAmount(Op, DemandedElts, Depth + 1)) {
      unsigned ShAmt = *KnownSA;
      if (ShAmt == 0)
        return TLO.CombineTo(Op, Op0);

      // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
      // supports sext_inreg.
      if (Op0.getOpcode() == ISD::SHL) {
        if (std::optional<unsigned> InnerSA =
                TLO.DAG.getValidShiftAmount(Op0, DemandedElts, Depth + 2)) {
          unsigned LowBits = BitWidth - ShAmt;
          EVT ExtVT = VT.changeElementType(
              *TLO.DAG.getContext(),
              EVT::getIntegerVT(*TLO.DAG.getContext(), LowBits));

          if (*InnerSA == ShAmt) {
            if (!TLO.LegalOperations() ||
                getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) == Legal)
              return TLO.CombineTo(
                  Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT,
                                      Op0.getOperand(0),
                                      TLO.DAG.getValueType(ExtVT)));

            // Even if we can't convert to sext_inreg, we might be able to
            // remove this shift pair if the input is already sign extended.
            unsigned NumSignBits =
                TLO.DAG.ComputeNumSignBits(Op0.getOperand(0), DemandedElts);
            if (NumSignBits > ShAmt)
              return TLO.CombineTo(Op, Op0.getOperand(0));
          }
        }
      }

      APInt InDemandedMask = (DemandedBits << ShAmt);

      // If the shift is exact, then it does demand the low bits (and knows that
      // they are zero).
      if (Op->getFlags().hasExact())
        InDemandedMask.setLowBits(ShAmt);

      // If any of the demanded bits are produced by the sign extension, we also
      // demand the input sign bit.
      if (DemandedBits.countl_zero() < ShAmt)
        InDemandedMask.setSignBit();

      if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                               Depth + 1))
        return true;
      Known >>= ShAmt;

      // If the input sign bit is known to be zero, or if none of the top bits
      // are demanded, turn this into an unsigned shift right.
      if (Known.Zero[BitWidth - ShAmt - 1] ||
          DemandedBits.countl_zero() >= ShAmt) {
        SDNodeFlags Flags;
        Flags.setExact(Op->getFlags().hasExact());
        return TLO.CombineTo(
            Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
      }

      int Log2 = DemandedBits.exactLogBase2();
      if (Log2 >= 0) {
        // The bit must come from the sign.
        SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
        return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
      }

      if (Known.One[BitWidth - ShAmt - 1])
        // New bits are known one.
        Known.One.setHighBits(ShAmt);

      // Attempt to avoid multi-use ops if we don't need anything from them.
      if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
        SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
            Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
        if (DemandedOp0) {
          SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
          return TLO.CombineTo(Op, NewOp);
        }
      }
    }

    // Try to match AVG patterns (after shift simplification).
    if (SDValue AVG = combineShiftToAVG(Op, TLO, *this, DemandedBits,
                                        DemandedElts, Depth + 1))
      return TLO.CombineTo(Op, AVG);

    break;
  }
  case ISD::FSHL:
  case ISD::FSHR: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    SDValue Op2 = Op.getOperand(2);
    bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

    if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
      unsigned Amt = SA->getAPIntValue().urem(BitWidth);

      // For fshl, 0-shift returns the 1st arg.
      // For fshr, 0-shift returns the 2nd arg.
      if (Amt == 0) {
        if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
                                 Known, TLO, Depth + 1))
          return true;
        break;
      }

      // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
      // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
      APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
      APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
      if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
                               Depth + 1))
        return true;
      if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
                               Depth + 1))
        return true;

      Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
      Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
      Known = Known.unionWith(Known2);

      // Attempt to avoid multi-use ops if we don't need anything from them.
      if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
          !DemandedElts.isAllOnes()) {
        SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
            Op0, Demanded0, DemandedElts, TLO.DAG, Depth + 1);
        SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
            Op1, Demanded1, DemandedElts, TLO.DAG, Depth + 1);
        if (DemandedOp0 || DemandedOp1) {
          DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
          DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
          SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0,
                                          DemandedOp1, Op2);
          return TLO.CombineTo(Op, NewOp);
        }
      }
    }

    if (isPowerOf2_32(BitWidth)) {
      // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
      // iff we're guaranteed not to use Op0.
      // TODO: Add FSHL equivalent?
      if (!IsFSHL && !DemandedBits.isAllOnes() &&
          (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT))) {
        KnownBits KnownAmt =
            TLO.DAG.computeKnownBits(Op2, DemandedElts, Depth + 1);
        unsigned MaxShiftAmt =
            KnownAmt.getMaxValue().getLimitedValue(BitWidth - 1);
        // Check we don't demand any shifted bits outside Op1.
        if (DemandedBits.countl_zero() >= MaxShiftAmt) {
          EVT AmtVT = Op2.getValueType();
          SDValue NewAmt =
              TLO.DAG.getNode(ISD::AND, dl, AmtVT, Op2,
                              TLO.DAG.getConstant(BitWidth - 1, dl, AmtVT));
          SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, Op1, NewAmt);
          return TLO.CombineTo(Op, NewOp);
        }
      }

      // For pow-2 bitwidths we only demand the bottom modulo amt bits.
      APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
      if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts, Known2, TLO,
                               Depth + 1))
        return true;
    }
    break;
  }
  case ISD::ROTL:
  case ISD::ROTR: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    bool IsROTL = (Op.getOpcode() == ISD::ROTL);

    // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
    if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
      return TLO.CombineTo(Op, Op0);

    if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
      unsigned Amt = SA->getAPIntValue().urem(BitWidth);
      unsigned RevAmt = BitWidth - Amt;

      // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
      // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
      APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
      if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
                               Depth + 1))
        return true;

      // rot*(x, 0) --> x
      if (Amt == 0)
        return TLO.CombineTo(Op, Op0);

      // See if we don't demand either half of the rotated bits.
      if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
          DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
        Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
        return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
      }
      if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
          DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
        Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
        return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
      }
    }

    // For pow-2 bitwidths we only demand the bottom modulo amt bits.
    if (isPowerOf2_32(BitWidth)) {
      APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
      if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
                               Depth + 1))
        return true;
    }
    break;
  }
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX: {
    unsigned Opc = Op.getOpcode();
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    // If we're only demanding signbits, then we can simplify to OR/AND node.
    unsigned BitOp =
        (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
    unsigned NumSignBits =
        std::min(TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1),
                 TLO.DAG.ComputeNumSignBits(Op1, DemandedElts, Depth + 1));
    unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
    if (NumSignBits >= NumDemandedUpperBits)
      return TLO.CombineTo(Op, TLO.DAG.getNode(BitOp, SDLoc(Op), VT, Op0, Op1));

    // Check if one arg is always less/greater than (or equal) to the other arg.
    KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
    KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
    switch (Opc) {
    case ISD::SMIN:
      if (std::optional<bool> IsSLE = KnownBits::sle(Known0, Known1))
        return TLO.CombineTo(Op, *IsSLE ? Op0 : Op1);
      if (std::optional<bool> IsSLT = KnownBits::slt(Known0, Known1))
        return TLO.CombineTo(Op, *IsSLT ? Op0 : Op1);
      Known = KnownBits::smin(Known0, Known1);
      break;
    case ISD::SMAX:
      if (std::optional<bool> IsSGE = KnownBits::sge(Known0, Known1))
        return TLO.CombineTo(Op, *IsSGE ? Op0 : Op1);
      if (std::optional<bool> IsSGT = KnownBits::sgt(Known0, Known1))
        return TLO.CombineTo(Op, *IsSGT ? Op0 : Op1);
      Known = KnownBits::smax(Known0, Known1);
      break;
    case ISD::UMIN:
      if (std::optional<bool> IsULE = KnownBits::ule(Known0, Known1))
        return TLO.CombineTo(Op, *IsULE ? Op0 : Op1);
      if (std::optional<bool> IsULT = KnownBits::ult(Known0, Known1))
        return TLO.CombineTo(Op, *IsULT ? Op0 : Op1);
      Known = KnownBits::umin(Known0, Known1);
      break;
    case ISD::UMAX:
      if (std::optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
        return TLO.CombineTo(Op, *IsUGE ? Op0 : Op1);
      if (std::optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
        return TLO.CombineTo(Op, *IsUGT ? Op0 : Op1);
      Known = KnownBits::umax(Known0, Known1);
      break;
    }
    break;
  }
  case ISD::BITREVERSE: {
    SDValue Src = Op.getOperand(0);
    APInt DemandedSrcBits = DemandedBits.reverseBits();
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;
    Known = Known2.reverseBits();
    break;
  }
  case ISD::BSWAP: {
    SDValue Src = Op.getOperand(0);

    // If the only bits demanded come from one byte of the bswap result,
    // just shift the input byte into position to eliminate the bswap.
    unsigned NLZ = DemandedBits.countl_zero();
    unsigned NTZ = DemandedBits.countr_zero();

    // Round NTZ down to the next byte.  If we have 11 trailing zeros, then
    // we need all the bits down to bit 8.  Likewise, round NLZ.  If we
    // have 14 leading zeros, round to 8.
    NLZ = alignDown(NLZ, 8);
    NTZ = alignDown(NTZ, 8);
    // If we need exactly one byte, we can do this transformation.
    if (BitWidth - NLZ - NTZ == 8) {
      // Replace this with either a left or right shift to get the byte into
      // the right place.
      unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
      if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
        unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
        SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShiftAmount, VT, dl);
        SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    APInt DemandedSrcBits = DemandedBits.byteSwap();
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;
    Known = Known2.byteSwap();
    break;
  }
  case ISD::CTPOP: {
    // If only 1 bit is demanded, replace with PARITY as long as we're before
    // op legalization.
    // FIXME: Limit to scalars for now.
    if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
                                               Op.getOperand(0)));

    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    break;
  }
  case ISD::PDEP: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    unsigned DemandedBitsLZ = OriginalDemandedBits.countl_zero();
    APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);

    // If the demanded bits has leading zeroes, we don't demand those from the
    // mask.
    if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1))
      return true;

    // The number of possible 1s in the mask determines the number of LSBs of
    // operand 0 used. Undemanded bits from the mask don't matter so filter
    // them before counting.
    KnownBits Known2;
    uint64_t Count = (~Known.Zero & LoMask).popcount();
    APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count));
    if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1))
      return true;

    // Zeroes are retained from the mask, but not ones.
    Known.One.clearAllBits();
    // The result will have at least as many trailing zeros as the non-mask
    // operand since bits can only map to the same or higher bit position.
    Known.Zero.setLowBits(Known2.countMinTrailingZeros());
    break;
  }
  case ISD::SIGN_EXTEND_INREG: {
    SDValue Op0 = Op.getOperand(0);
    EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    unsigned ExVTBits = ExVT.getScalarSizeInBits();

    // If we only care about the highest bit, don't bother shifting right.
    if (DemandedBits.isSignMask()) {
      unsigned MinSignedBits =
          TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
      bool AlreadySignExtended = ExVTBits >= MinSignedBits;
      // However if the input is already sign extended we expect the sign
      // extension to be dropped altogether later and do not simplify.
      if (!AlreadySignExtended) {
        // Compute the correct shift amount type, which must be getShiftAmountTy
        // for scalar types after legalization.
        SDValue ShiftAmt =
            TLO.DAG.getShiftAmountConstant(BitWidth - ExVTBits, VT, dl);
        return TLO.CombineTo(Op,
                             TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
      }
    }

    // If none of the extended bits are demanded, eliminate the sextinreg.
    if (DemandedBits.getActiveBits() <= ExVTBits)
      return TLO.CombineTo(Op, Op0);

    APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

    // Since the sign extended bits are demanded, we know that the sign
    // bit is demanded.
    InputDemandedBits.setBit(ExVTBits - 1);

    if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;

    // If the sign bit of the input is known set or clear, then we know the
    // top bits of the result.

    // If the input sign bit is known zero, convert this into a zero extension.
    if (Known.Zero[ExVTBits - 1])
      return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

    APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
    if (Known.One[ExVTBits - 1]) { // Input sign bit known set
      Known.One.setBitsFrom(ExVTBits);
      Known.Zero &= Mask;
    } else { // Input sign bit unknown
      Known.Zero &= Mask;
      Known.One &= Mask;
    }
    break;
  }
  case ISD::BUILD_PAIR: {
    EVT HalfVT = Op.getOperand(0).getValueType();
    unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

    APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
    APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

    KnownBits KnownLo, KnownHi;

    if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
      return true;

    if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
      return true;

    Known = KnownHi.concat(KnownLo);
    break;
  }
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    if (VT.isScalableVector())
      return false;
    [[fallthrough]];
  case ISD::ZERO_EXTEND: {
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    unsigned InBits = SrcVT.getScalarSizeInBits();
    unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
    bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

    // If none of the top bits are demanded, convert this into an any_extend.
    if (DemandedBits.getActiveBits() <= InBits) {
      // If we only need the non-extended bits of the bottom element
      // then we can just bitcast to the result.
      if (IsLE && IsVecInReg && DemandedElts == 1 &&
          VT.getSizeInBits() == SrcVT.getSizeInBits())
        return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

      unsigned Opc =
          IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
      if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
        return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
    }

    APInt InDemandedBits = DemandedBits.trunc(InBits);
    APInt InDemandedElts = DemandedElts.zext(InElts);
    if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                             Depth + 1)) {
      Op->dropFlags(SDNodeFlags::NonNeg);
      return true;
    }
    assert(Known.getBitWidth() == InBits && "Src width has changed?");
    Known = Known.zext(BitWidth);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
            Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
    break;
  }
  case ISD::SIGN_EXTEND_VECTOR_INREG:
    if (VT.isScalableVector())
      return false;
    [[fallthrough]];
  case ISD::SIGN_EXTEND: {
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    unsigned InBits = SrcVT.getScalarSizeInBits();
    unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
    bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

    APInt InDemandedElts = DemandedElts.zext(InElts);
    APInt InDemandedBits = DemandedBits.trunc(InBits);

    // Since some of the sign extended bits are demanded, we know that the sign
    // bit is demanded.
    InDemandedBits.setBit(InBits - 1);

    // If none of the top bits are demanded, convert this into an any_extend.
    if (DemandedBits.getActiveBits() <= InBits) {
      // If we only need the non-extended bits of the bottom element
      // then we can just bitcast to the result.
      if (IsLE && IsVecInReg && DemandedElts == 1 &&
          VT.getSizeInBits() == SrcVT.getSizeInBits())
        return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

      // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
      if (getBooleanContents(VT) != ZeroOrNegativeOneBooleanContent ||
          TLO.DAG.ComputeNumSignBits(Src, InDemandedElts, Depth + 1) !=
              InBits) {
        unsigned Opc =
            IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
        if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
          return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
      }
    }

    if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(Known.getBitWidth() == InBits && "Src width has changed?");

    // If the sign bit is known one, the top bits match.
    Known = Known.sext(BitWidth);

    // If the sign bit is known zero, convert this to a zero extend.
    if (Known.isNonNegative()) {
      unsigned Opc =
          IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
      if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) {
        SDNodeFlags Flags;
        if (!IsVecInReg)
          Flags |= SDNodeFlags::NonNeg;
        return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags));
      }
    }

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
            Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
    break;
  }
  case ISD::ANY_EXTEND_VECTOR_INREG:
    if (VT.isScalableVector())
      return false;
    [[fallthrough]];
  case ISD::ANY_EXTEND: {
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    unsigned InBits = SrcVT.getScalarSizeInBits();
    unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
    bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

    // If we only need the bottom element then we can just bitcast.
    // TODO: Handle ANY_EXTEND?
    if (IsLE && IsVecInReg && DemandedElts == 1 &&
        VT.getSizeInBits() == SrcVT.getSizeInBits())
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

    APInt InDemandedBits = DemandedBits.trunc(InBits);
    APInt InDemandedElts = DemandedElts.zext(InElts);
    if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(Known.getBitWidth() == InBits && "Src width has changed?");
    Known = Known.anyext(BitWidth);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
            Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
    break;
  }
  case ISD::TRUNCATE: {
    SDValue Src = Op.getOperand(0);

    // Simplify the input, using demanded bit information, and compute the known
    // zero/one bits live out.
    unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
    APInt TruncMask = DemandedBits.zext(OperandBitWidth);
    if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
                             Depth + 1)) {
      // Disable the nsw and nuw flags. We can no longer guarantee that we
      // won't wrap after simplification.
      Op->dropFlags(SDNodeFlags::NoWrap);
      return true;
    }
    Known = Known.trunc(BitWidth);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
            Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

    // If the input is only used by this truncate, see if we can shrink it based
    // on the known demanded bits.
    switch (Src.getOpcode()) {
    default:
      break;
    case ISD::SRL:
      // Shrink SRL by a constant if none of the high bits shifted in are
      // demanded.
      if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
        // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
        // undesirable.
        break;

      if (Src.getNode()->hasOneUse()) {
        if (isTruncateFree(Src, VT) &&
            !isTruncateFree(Src.getValueType(), VT)) {
          // If truncate is only free at trunc(srl), do not turn it into
          // srl(trunc). The check is done by first check the truncate is free
          // at Src's opcode(srl), then check the truncate is not done by
          // referencing sub-register. In test, if both trunc(srl) and
          // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
          // trunc(srl)'s trunc is free, trunc(srl) is better.
          break;
        }

        std::optional<unsigned> ShAmtC =
            TLO.DAG.getValidShiftAmount(Src, DemandedElts, Depth + 2);
        if (!ShAmtC || *ShAmtC >= BitWidth)
          break;
        unsigned ShVal = *ShAmtC;

        APInt HighBits =
            APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
        HighBits.lshrInPlace(ShVal);
        HighBits = HighBits.trunc(BitWidth);
        if (!(HighBits & DemandedBits)) {
          // None of the shifted in bits are needed.  Add a truncate of the
          // shift input, then shift it.
          SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(ShVal, VT, dl);
          SDValue NewTrunc =
              TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
          return TLO.CombineTo(
              Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
        }
      }
      break;
    }

    break;
  }
  case ISD::AssertZext: {
    // AssertZext demands all of the high bits, plus any of the low bits
    // demanded by its users.
    EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
    APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
    if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
                             TLO, Depth + 1))
      return true;

    Known.Zero |= ~InMask;
    Known.One &= (~Known.Zero);
    break;
  }
  case ISD::EXTRACT_VECTOR_ELT: {
    SDValue Src = Op.getOperand(0);
    SDValue Idx = Op.getOperand(1);
    ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
    unsigned EltBitWidth = Src.getScalarValueSizeInBits();

    if (SrcEltCnt.isScalable())
      return false;

    // Demand the bits from every vector element without a constant index.
    unsigned NumSrcElts = SrcEltCnt.getFixedValue();
    APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
    if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
      if (CIdx->getAPIntValue().ult(NumSrcElts))
        DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

    // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
    // anything about the extended bits.
    APInt DemandedSrcBits = DemandedBits;
    if (BitWidth > EltBitWidth)
      DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
                             Depth + 1))
      return true;

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
      if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
              Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
        SDValue NewOp =
            TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    Known = Known2;
    if (BitWidth > EltBitWidth)
      Known = Known.anyext(BitWidth);
    break;
  }
  case ISD::BITCAST: {
    if (VT.isScalableVector())
      return false;
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();
    unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

    // If this is an FP->Int bitcast and if the sign bit is the only
    // thing demanded, turn this into a FGETSIGN.
    if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
        DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
        SrcVT.isFloatingPoint()) {
      if (isOperationLegalOrCustom(ISD::FGETSIGN, VT)) {
        // Make a FGETSIGN + SHL to move the sign bit into the appropriate
        // place.  We expect the SHL to be eliminated by other optimizations.
        SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, VT, Src);
        unsigned ShVal = Op.getValueSizeInBits() - 1;
        SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShVal, VT, dl);
        return TLO.CombineTo(Op,
                             TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
      }
    }

    // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
    // Demand the elt/bit if any of the original elts/bits are demanded.
    if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
      unsigned Scale = BitWidth / NumSrcEltBits;
      unsigned NumSrcElts = SrcVT.getVectorNumElements();
      APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
      for (unsigned i = 0; i != Scale; ++i) {
        unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
        unsigned BitOffset = EltOffset * NumSrcEltBits;
        DemandedSrcBits |= DemandedBits.extractBits(NumSrcEltBits, BitOffset);
      }
      // Recursive calls below may turn not demanded elements into poison, so we
      // need to demand all smaller source elements that maps to a demanded
      // destination element.
      APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);

      APInt KnownSrcUndef, KnownSrcZero;
      if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                     KnownSrcZero, TLO, Depth + 1))
        return true;

      KnownBits KnownSrcBits;
      if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                               KnownSrcBits, TLO, Depth + 1))
        return true;
    } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
      // TODO - bigendian once we have test coverage.
      unsigned Scale = NumSrcEltBits / BitWidth;
      unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
      APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
      APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
      for (unsigned i = 0; i != NumElts; ++i)
        if (DemandedElts[i]) {
          unsigned Offset = (i % Scale) * BitWidth;
          DemandedSrcBits.insertBits(DemandedBits, Offset);
          DemandedSrcElts.setBit(i / Scale);
        }

      if (SrcVT.isVector()) {
        APInt KnownSrcUndef, KnownSrcZero;
        if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                       KnownSrcZero, TLO, Depth + 1))
          return true;
      }

      KnownBits KnownSrcBits;
      if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                               KnownSrcBits, TLO, Depth + 1))
        return true;

      // Attempt to avoid multi-use ops if we don't need anything from them.
      if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
        if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
                Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
          SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc);
          return TLO.CombineTo(Op, NewOp);
        }
      }
    }

    // If this is a bitcast, let computeKnownBits handle it.  Only do this on a
    // recursive call where Known may be useful to the caller.
    if (Depth > 0) {
      Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
      return false;
    }
    break;
  }
  case ISD::MUL:
    if (DemandedBits.isPowerOf2()) {
      // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
      // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
      // odd (has LSB set), then the left-shifted low bit of X is the answer.
      unsigned CTZ = DemandedBits.countr_zero();
      ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(1), DemandedElts);
      if (C && C->getAPIntValue().countr_zero() == CTZ) {
        SDValue AmtC = TLO.DAG.getShiftAmountConstant(CTZ, VT, dl);
        SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC);
        return TLO.CombineTo(Op, Shl);
      }
    }
    // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
    // X * X is odd iff X is odd.
    // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
    if (Op.getOperand(0) == Op.getOperand(1) && DemandedBits.ult(4)) {
      SDValue One = TLO.DAG.getConstant(1, dl, VT);
      SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One);
      return TLO.CombineTo(Op, And1);
    }
    [[fallthrough]];
  case ISD::PTRADD:
    if (Op.getOperand(0).getValueType() != Op.getOperand(1).getValueType())
      break;
    // PTRADD behaves like ADD if pointers are represented as integers.
    [[fallthrough]];
  case ISD::ADD:
  case ISD::SUB: {
    // Add, Sub, and Mul don't demand any bits in positions beyond that
    // of the highest bit demanded of them.
    SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
    SDNodeFlags Flags = Op.getNode()->getFlags();
    unsigned DemandedBitsLZ = DemandedBits.countl_zero();
    APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
    KnownBits KnownOp0, KnownOp1;
    auto GetDemandedBitsLHSMask = [&](APInt Demanded,
                                      const KnownBits &KnownRHS) {
      if (Op.getOpcode() == ISD::MUL)
        Demanded.clearHighBits(KnownRHS.countMinTrailingZeros());
      return Demanded;
    };
    if (SimplifyDemandedBits(Op1, LoMask, DemandedElts, KnownOp1, TLO,
                             Depth + 1) ||
        SimplifyDemandedBits(Op0, GetDemandedBitsLHSMask(LoMask, KnownOp1),
                             DemandedElts, KnownOp0, TLO, Depth + 1) ||
        // See if the operation should be performed at a smaller bit width.
        ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
      // Disable the nsw and nuw flags. We can no longer guarantee that we
      // won't wrap after simplification.
      Op->dropFlags(SDNodeFlags::NoWrap);
      return true;
    }

    // neg x with only low bit demanded is simply x.
    if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
        isNullConstant(Op0))
      return TLO.CombineTo(Op, Op1);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
      SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
          Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
      if (DemandedOp0 || DemandedOp1) {
        Op0 = DemandedOp0 ? DemandedOp0 : Op0;
        Op1 = DemandedOp1 ? DemandedOp1 : Op1;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1,
                                        Flags & ~SDNodeFlags::NoWrap);
        return TLO.CombineTo(Op, NewOp);
      }
    }

    // If we have a constant operand, we may be able to turn it into -1 if we
    // do not demand the high bits. This can make the constant smaller to
    // encode, allow more general folding, or match specialized instruction
    // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
    // is probably not useful (and could be detrimental).
    ConstantSDNode *C = isConstOrConstSplat(Op1);
    APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
    if (C && !C->isAllOnes() && !C->isOne() &&
        (C->getAPIntValue() | HighMask).isAllOnes()) {
      SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
      // Disable the nsw and nuw flags. We can no longer guarantee that we
      // won't wrap after simplification.
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1,
                                      Flags & ~SDNodeFlags::NoWrap);
      return TLO.CombineTo(Op, NewOp);
    }

    // Match a multiply with a disguised negated-power-of-2 and convert to a
    // an equivalent shift-left amount.
    // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
    auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
      if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
        return 0;

      // Don't touch opaque constants. Also, ignore zero and power-of-2
      // multiplies. Those will get folded later.
      ConstantSDNode *MulC = isConstOrConstSplat(Mul.getOperand(1));
      if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
          !MulC->getAPIntValue().isPowerOf2()) {
        APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
        if (UnmaskedC.isNegatedPowerOf2())
          return (-UnmaskedC).logBase2();
      }
      return 0;
    };

    auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
                       unsigned ShlAmt) {
      SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(ShlAmt, VT, dl);
      SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC);
      SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl);
      return TLO.CombineTo(Op, Res);
    };

    if (isOperationLegalOrCustom(ISD::SHL, VT)) {
      if (Op.getOpcode() == ISD::ADD) {
        // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
        if (unsigned ShAmt = getShiftLeftAmt(Op0))
          return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt);
        // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
        if (unsigned ShAmt = getShiftLeftAmt(Op1))
          return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
      }
      if (Op.getOpcode() == ISD::SUB) {
        // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
        if (unsigned ShAmt = getShiftLeftAmt(Op1))
          return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
      }
    }

    if (Op.getOpcode() == ISD::MUL) {
      Known = KnownBits::mul(KnownOp0, KnownOp1);
    } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
      Known = KnownBits::computeForAddSub(
          Op.getOpcode() != ISD::SUB, Flags.hasNoSignedWrap(),
          Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1);
    }
    break;
  }
  case ISD::FABS: {
    SDValue Op0 = Op.getOperand(0);
    APInt SignMask = APInt::getSignMask(BitWidth);

    if (!DemandedBits.intersects(SignMask))
      return TLO.CombineTo(Op, Op0);

    if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;

    if (Known.isNonNegative())
      return TLO.CombineTo(Op, Op0);
    if (Known.isNegative())
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::FNEG, dl, VT, Op0, Op->getFlags()));

    Known.Zero |= SignMask;
    Known.One &= ~SignMask;

    break;
  }
  case ISD::FCOPYSIGN: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
    unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
    APInt SignMask0 = APInt::getSignMask(BitWidth0);
    APInt SignMask1 = APInt::getSignMask(BitWidth1);

    if (!DemandedBits.intersects(SignMask0))
      return TLO.CombineTo(Op, Op0);

    if (SimplifyDemandedBits(Op0, ~SignMask0 & DemandedBits, DemandedElts,
                             Known, TLO, Depth + 1) ||
        SimplifyDemandedBits(Op1, SignMask1, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;

    if (Known2.isNonNegative())
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::FABS, dl, VT, Op0, Op->getFlags()));

    if (Known2.isNegative())
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::FNEG, dl, VT,
                              TLO.DAG.getNode(ISD::FABS, SDLoc(Op0), VT, Op0)));

    Known.Zero &= ~SignMask0;
    Known.One &= ~SignMask0;
    break;
  }
  case ISD::FNEG: {
    SDValue Op0 = Op.getOperand(0);
    APInt SignMask = APInt::getSignMask(BitWidth);

    if (!DemandedBits.intersects(SignMask))
      return TLO.CombineTo(Op, Op0);

    if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;

    if (!Known.isSignUnknown()) {
      Known.Zero ^= SignMask;
      Known.One ^= SignMask;
    }

    break;
  }
  default:
    // We also ask the target about intrinsics (which could be specific to it).
    if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
      // TODO: Probably okay to remove after audit; here to reduce change size
      // in initial enablement patch for scalable vectors
      if (Op.getValueType().isScalableVector())
        break;
      if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
                                            Known, TLO, Depth))
        return true;
      break;
    }

    // Just use computeKnownBits to compute output bits.
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    break;
  }

  // If we know the value of all of the demanded bits, return this as a
  // constant.
  if (!isTargetCanonicalConstantNode(Op) &&
      DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
    // Avoid folding to a constant if any OpaqueConstant is involved.
    if (llvm::any_of(Op->ops(), [](SDValue V) {
          auto *C = dyn_cast<ConstantSDNode>(V);
          return C && C->isOpaque();
        }))
      return false;
    if (VT.isInteger())
      return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
    if (VT.isFloatingPoint())
      return TLO.CombineTo(
          Op, TLO.DAG.getConstantFP(APFloat(VT.getFltSemantics(), Known.One),
                                    dl, VT));
  }

  // A multi use 'all demanded elts' simplify failed to find any knownbits.
  // Try again just for the original demanded elts.
  // Ensure we do this AFTER constant folding above.
  if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
    Known = TLO.DAG.computeKnownBits(Op, OriginalDemandedElts, Depth);

  return false;
}

bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
                                                const APInt &DemandedElts,
                                                DAGCombinerInfo &DCI) const {
  SelectionDAG &DAG = DCI.DAG;
  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                        !DCI.isBeforeLegalizeOps());

  APInt KnownUndef, KnownZero;
  bool Simplified =
      SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
  if (Simplified) {
    DCI.AddToWorklist(Op.getNode());
    DCI.CommitTargetLoweringOpt(TLO);
  }

  return Simplified;
}

/// Given a vector binary operation and known undefined elements for each input
/// operand, compute whether each element of the output is undefined.
static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
                                         const APInt &UndefOp0,
                                         const APInt &UndefOp1) {
  EVT VT = BO.getValueType();
  assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
         "Vector binop only");

  EVT EltVT = VT.getVectorElementType();
  unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
  assert(UndefOp0.getBitWidth() == NumElts &&
         UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");

  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
                                   const APInt &UndefVals) {
    if (UndefVals[Index])
      return DAG.getUNDEF(EltVT);

    if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
      // Try hard to make sure that the getNode() call is not creating temporary
      // nodes. Ignore opaque integers because they do not constant fold.
      SDValue Elt = BV->getOperand(Index);
      auto *C = dyn_cast<ConstantSDNode>(Elt);
      if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
        return Elt;
    }

    return SDValue();
  };

  APInt KnownUndef = APInt::getZero(NumElts);
  for (unsigned i = 0; i != NumElts; ++i) {
    // If both inputs for this element are either constant or undef and match
    // the element type, compute the constant/undef result for this element of
    // the vector.
    // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
    // not handle FP constants. The code within getNode() should be refactored
    // to avoid the danger of creating a bogus temporary node here.
    SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
    SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
    if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
      if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
        KnownUndef.setBit(i);
  }
  return KnownUndef;
}

bool TargetLowering::SimplifyDemandedVectorElts(
    SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
    APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
    bool AssumeSingleUse) const {
  EVT VT = Op.getValueType();
  unsigned Opcode = Op.getOpcode();
  APInt DemandedElts = OriginalDemandedElts;
  unsigned NumElts = DemandedElts.getBitWidth();
  assert(VT.isVector() && "Expected vector op");

  KnownUndef = KnownZero = APInt::getZero(NumElts);

  if (!shouldSimplifyDemandedVectorElts(Op, TLO))
    return false;

  // TODO: For now we assume we know nothing about scalable vectors.
  if (VT.isScalableVector())
    return false;

  assert(VT.getVectorNumElements() == NumElts &&
         "Mask size mismatches value type element count!");

  // Undef operand.
  if (Op.isUndef()) {
    KnownUndef.setAllBits();
    return false;
  }

  // If Op has other users, assume that all elements are needed.
  if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
    DemandedElts.setAllBits();

  // Not demanding any elements from Op.
  if (DemandedElts == 0) {
    KnownUndef.setAllBits();
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
  }

  // Limit search depth.
  if (Depth >= SelectionDAG::MaxRecursionDepth)
    return false;

  SDLoc DL(Op);
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();

  // Helper for demanding the specified elements and all the bits of both binary
  // operands.
  auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
    SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
                                                           TLO.DAG, Depth + 1);
    SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
                                                           TLO.DAG, Depth + 1);
    if (NewOp0 || NewOp1) {
      SDValue NewOp =
          TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0,
                          NewOp1 ? NewOp1 : Op1, Op->getFlags());
      return TLO.CombineTo(Op, NewOp);
    }
    return false;
  };

  switch (Opcode) {
  case ISD::SCALAR_TO_VECTOR: {
    if (!DemandedElts[0]) {
      KnownUndef.setAllBits();
      return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
    }
    KnownUndef.setHighBits(NumElts - 1);
    break;
  }
  case ISD::BITCAST: {
    SDValue Src = Op.getOperand(0);
    EVT SrcVT = Src.getValueType();

    if (!SrcVT.isVector()) {
      // TODO - bigendian once we have test coverage.
      if (IsLE) {
        APInt DemandedSrcBits = APInt::getZero(SrcVT.getSizeInBits());
        unsigned EltSize = VT.getScalarSizeInBits();
        for (unsigned I = 0; I != NumElts; ++I) {
          if (DemandedElts[I]) {
            unsigned Offset = I * EltSize;
            DemandedSrcBits.setBits(Offset, Offset + EltSize);
          }
        }
        KnownBits Known;
        if (SimplifyDemandedBits(Src, DemandedSrcBits, Known, TLO, Depth + 1))
          return true;
      }
      break;
    }

    // Fast handling of 'identity' bitcasts.
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    if (NumSrcElts == NumElts)
      return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
                                        KnownZero, TLO, Depth + 1);

    APInt SrcDemandedElts, SrcZero, SrcUndef;

    // Bitcast from 'large element' src vector to 'small element' vector, we
    // must demand a source element if any DemandedElt maps to it.
    if ((NumElts % NumSrcElts) == 0) {
      unsigned Scale = NumElts / NumSrcElts;
      SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
      if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                     TLO, Depth + 1))
        return true;

      // Try calling SimplifyDemandedBits, converting demanded elts to the bits
      // of the large element.
      // TODO - bigendian once we have test coverage.
      if (IsLE) {
        unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
        APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
        for (unsigned i = 0; i != NumElts; ++i)
          if (DemandedElts[i]) {
            unsigned Ofs = (i % Scale) * EltSizeInBits;
            SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
          }

        KnownBits Known;
        if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
                                 TLO, Depth + 1))
          return true;

        // The bitcast has split each wide element into a number of
        // narrow subelements. We have just computed the Known bits
        // for wide elements. See if element splitting results in
        // some subelements being zero. Only for demanded elements!
        for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
          if (!Known.Zero.extractBits(EltSizeInBits, SubElt * EltSizeInBits)
                   .isAllOnes())
            continue;
          for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
            unsigned Elt = Scale * SrcElt + SubElt;
            if (DemandedElts[Elt])
              KnownZero.setBit(Elt);
          }
        }
      }

      // If the src element is zero/undef then all the output elements will be -
      // only demanded elements are guaranteed to be correct.
      for (unsigned i = 0; i != NumSrcElts; ++i) {
        if (SrcDemandedElts[i]) {
          if (SrcZero[i])
            KnownZero.setBits(i * Scale, (i + 1) * Scale);
          if (SrcUndef[i])
            KnownUndef.setBits(i * Scale, (i + 1) * Scale);
        }
      }
    }

    // Bitcast from 'small element' src vector to 'large element' vector, we
    // demand all smaller source elements covered by the larger demanded element
    // of this vector.
    if ((NumSrcElts % NumElts) == 0) {
      unsigned Scale = NumSrcElts / NumElts;
      SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
      if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                     TLO, Depth + 1))
        return true;

      // If all the src elements covering an output element are zero/undef, then
      // the output element will be as well, assuming it was demanded.
      for (unsigned i = 0; i != NumElts; ++i) {
        if (DemandedElts[i]) {
          if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
            KnownZero.setBit(i);
          if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
            KnownUndef.setBit(i);
        }
      }
    }
    break;
  }
  case ISD::FREEZE: {
    SDValue N0 = Op.getOperand(0);
    if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
            N0, DemandedElts, UndefPoisonKind::UndefOrPoison, Depth + 1))
      return TLO.CombineTo(Op, N0);

    // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
    // freeze(op(x, ...)) -> op(freeze(x), ...).
    if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1)
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
                              TLO.DAG.getFreeze(N0.getOperand(0))));
    break;
  }
  case ISD::BUILD_VECTOR: {
    // Check all elements and simplify any unused elements with UNDEF.
    if (!DemandedElts.isAllOnes()) {
      // Don't simplify BROADCASTS.
      if (llvm::any_of(Op->op_values(),
                       [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
        SmallVector<SDValue, 32> Ops(Op->ops());
        bool Updated = false;
        for (unsigned i = 0; i != NumElts; ++i) {
          if (!DemandedElts[i] && !Ops[i].isUndef()) {
            Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
            KnownUndef.setBit(i);
            Updated = true;
          }
        }
        if (Updated)
          return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
      }
    }
    for (unsigned i = 0; i != NumElts; ++i) {
      SDValue SrcOp = Op.getOperand(i);
      if (SrcOp.isUndef()) {
        KnownUndef.setBit(i);
      } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
                 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
        KnownZero.setBit(i);
      }
    }
    break;
  }
  case ISD::CONCAT_VECTORS: {
    EVT SubVT = Op.getOperand(0).getValueType();
    unsigned NumSubVecs = Op.getNumOperands();
    unsigned NumSubElts = SubVT.getVectorNumElements();
    for (unsigned i = 0; i != NumSubVecs; ++i) {
      SDValue SubOp = Op.getOperand(i);
      APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
      APInt SubUndef, SubZero;
      if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
                                     Depth + 1))
        return true;
      KnownUndef.insertBits(SubUndef, i * NumSubElts);
      KnownZero.insertBits(SubZero, i * NumSubElts);
    }

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedElts.isAllOnes()) {
      bool FoundNewSub = false;
      SmallVector<SDValue, 2> DemandedSubOps;
      for (unsigned i = 0; i != NumSubVecs; ++i) {
        SDValue SubOp = Op.getOperand(i);
        APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
        SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
            SubOp, SubElts, TLO.DAG, Depth + 1);
        DemandedSubOps.push_back(NewSubOp ? NewSubOp : SubOp);
        FoundNewSub = NewSubOp ? true : FoundNewSub;
      }
      if (FoundNewSub) {
        SDValue NewOp =
            TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedSubOps);
        return TLO.CombineTo(Op, NewOp);
      }
    }
    break;
  }
  case ISD::INSERT_SUBVECTOR: {
    // Demand any elements from the subvector and the remainder from the src it
    // is inserted into.
    SDValue Src = Op.getOperand(0);
    SDValue Sub = Op.getOperand(1);
    uint64_t Idx = Op.getConstantOperandVal(2);
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
    APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
    APInt DemandedSrcElts = DemandedElts;
    DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);

    // If none of the sub operand elements are demanded, bypass the insert.
    if (!DemandedSubElts)
      return TLO.CombineTo(Op, Src);

    APInt SubUndef, SubZero;
    if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
                                   Depth + 1))
      return true;

    // If none of the src operand elements are demanded, replace it with undef.
    if (!DemandedSrcElts && !Src.isUndef())
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
                                               TLO.DAG.getUNDEF(VT), Sub,
                                               Op.getOperand(2)));

    if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
                                   TLO, Depth + 1))
      return true;
    KnownUndef.insertBits(SubUndef, Idx);
    KnownZero.insertBits(SubZero, Idx);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
      SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
          Src, DemandedSrcElts, TLO.DAG, Depth + 1);
      SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
          Sub, DemandedSubElts, TLO.DAG, Depth + 1);
      if (NewSrc || NewSub) {
        NewSrc = NewSrc ? NewSrc : Src;
        NewSub = NewSub ? NewSub : Sub;
        SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
                                        NewSub, Op.getOperand(2));
        return TLO.CombineTo(Op, NewOp);
      }
    }
    break;
  }
  case ISD::EXTRACT_SUBVECTOR: {
    // Offset the demanded elts by the subvector index.
    SDValue Src = Op.getOperand(0);
    if (Src.getValueType().isScalableVector())
      break;
    uint64_t Idx = Op.getConstantOperandVal(1);
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);

    APInt SrcUndef, SrcZero;
    if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
                                   Depth + 1))
      return true;
    KnownUndef = SrcUndef.extractBits(NumElts, Idx);
    KnownZero = SrcZero.extractBits(NumElts, Idx);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Src, DemandedSrcElts,
                                                           TLO.DAG, Depth + 1);
    if (NewSrc) {
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
                                      Op.getOperand(1));
      return TLO.CombineTo(Op, NewOp);
    }
    break;
  }
  case ISD::INSERT_VECTOR_ELT: {
    SDValue Vec = Op.getOperand(0);
    SDValue Scl = Op.getOperand(1);
    auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

    // For a legal, constant insertion index, if we don't need this insertion
    // then strip it, else remove it from the demanded elts.
    if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
      unsigned Idx = CIdx->getZExtValue();
      if (!DemandedElts[Idx])
        return TLO.CombineTo(Op, Vec);

      APInt DemandedVecElts(DemandedElts);
      DemandedVecElts.clearBit(Idx);
      if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
                                     KnownZero, TLO, Depth + 1))
        return true;

      KnownUndef.setBitVal(Idx, Scl.isUndef());

      KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
      break;
    }

    APInt VecUndef, VecZero;
    if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
                                   Depth + 1))
      return true;
    // Without knowing the insertion index we can't set KnownUndef/KnownZero.
    break;
  }
  case ISD::VSELECT: {
    SDValue Sel = Op.getOperand(0);
    SDValue LHS = Op.getOperand(1);
    SDValue RHS = Op.getOperand(2);

    // Try to transform the select condition based on the current demanded
    // elements.
    APInt UndefSel, ZeroSel;
    if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO,
                                   Depth + 1))
      return true;

    // See if we can simplify either vselect operand.
    APInt DemandedLHS(DemandedElts);
    APInt DemandedRHS(DemandedElts);
    APInt UndefLHS, ZeroLHS;
    APInt UndefRHS, ZeroRHS;
    if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
                                   Depth + 1))
      return true;
    if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
                                   Depth + 1))
      return true;

    KnownUndef = UndefLHS & UndefRHS;
    KnownZero = ZeroLHS & ZeroRHS;

    // If we know that the selected element is always zero, we don't need the
    // select value element.
    APInt DemandedSel = DemandedElts & ~KnownZero;
    if (DemandedSel != DemandedElts)
      if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO,
                                     Depth + 1))
        return true;

    break;
  }
  case ISD::VECTOR_SHUFFLE: {
    SDValue LHS = Op.getOperand(0);
    SDValue RHS = Op.getOperand(1);
    ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

    // Collect demanded elements from shuffle operands..
    APInt DemandedLHS(NumElts, 0);
    APInt DemandedRHS(NumElts, 0);
    for (unsigned i = 0; i != NumElts; ++i) {
      int M = ShuffleMask[i];
      if (M < 0 || !DemandedElts[i])
        continue;
      assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
      if (M < (int)NumElts)
        DemandedLHS.setBit(M);
      else
        DemandedRHS.setBit(M - NumElts);
    }

    // If either side isn't demanded, replace it by UNDEF. We handle this
    // explicitly here to also simplify in case of multiple uses (on the
    // contrary to the SimplifyDemandedVectorElts calls below).
    bool FoldLHS = !DemandedLHS && !LHS.isUndef();
    bool FoldRHS = !DemandedRHS && !RHS.isUndef();
    if (FoldLHS || FoldRHS) {
      LHS = FoldLHS ? TLO.DAG.getUNDEF(LHS.getValueType()) : LHS;
      RHS = FoldRHS ? TLO.DAG.getUNDEF(RHS.getValueType()) : RHS;
      SDValue NewOp =
          TLO.DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, ShuffleMask);
      return TLO.CombineTo(Op, NewOp);
    }

    // See if we can simplify either shuffle operand.
    APInt UndefLHS, ZeroLHS;
    APInt UndefRHS, ZeroRHS;
    if (SimplifyDemandedVectorElts(LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
                                   Depth + 1))
      return true;
    if (SimplifyDemandedVectorElts(RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
                                   Depth + 1))
      return true;

    // Simplify mask using undef elements from LHS/RHS.
    bool Updated = false;
    bool IdentityLHS = true, IdentityRHS = true;
    SmallVector<int, 32> NewMask(ShuffleMask);
    for (unsigned i = 0; i != NumElts; ++i) {
      int &M = NewMask[i];
      if (M < 0)
        continue;
      if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
          (M >= (int)NumElts && UndefRHS[M - NumElts])) {
        Updated = true;
        M = -1;
      }
      IdentityLHS &= (M < 0) || (M == (int)i);
      IdentityRHS &= (M < 0) || ((M - NumElts) == i);
    }

    // Update legal shuffle masks based on demanded elements if it won't reduce
    // to Identity which can cause premature removal of the shuffle mask.
    if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
      SDValue LegalShuffle =
          buildLegalVectorShuffle(VT, DL, LHS, RHS, NewMask, TLO.DAG);
      if (LegalShuffle)
        return TLO.CombineTo(Op, LegalShuffle);
    }

    // Propagate undef/zero elements from LHS/RHS.
    for (unsigned i = 0; i != NumElts; ++i) {
      int M = ShuffleMask[i];
      if (M < 0) {
        KnownUndef.setBit(i);
      } else if (M < (int)NumElts) {
        if (UndefLHS[M])
          KnownUndef.setBit(i);
        if (ZeroLHS[M])
          KnownZero.setBit(i);
      } else {
        if (UndefRHS[M - NumElts])
          KnownUndef.setBit(i);
        if (ZeroRHS[M - NumElts])
          KnownZero.setBit(i);
      }
    }
    break;
  }
  case ISD::ANY_EXTEND_VECTOR_INREG:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
  case ISD::ZERO_EXTEND_VECTOR_INREG: {
    APInt SrcUndef, SrcZero;
    SDValue Src = Op.getOperand(0);
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
    APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
    if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
                                   Depth + 1))
      return true;
    KnownZero = SrcZero.zextOrTrunc(NumElts);
    KnownUndef = SrcUndef.zextOrTrunc(NumElts);

    if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
        Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
        DemandedSrcElts == 1) {
      // aext - if we just need the bottom element then we can bitcast.
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
    }

    if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
      // zext(undef) upper bits are guaranteed to be zero.
      if (DemandedElts.isSubsetOf(KnownUndef))
        return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
      KnownUndef.clearAllBits();

      // zext - if we just need the bottom element then we can mask:
      // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
      if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
          Op->isOnlyUserOf(Src.getNode()) &&
          Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
        SDLoc DL(Op);
        EVT SrcVT = Src.getValueType();
        EVT SrcSVT = SrcVT.getScalarType();

        // If we're after type legalization and SrcSVT is not legal, use the
        // promoted type for creating constants to avoid creating nodes with
        // illegal types.
        if (TLO.LegalTypes())
          SrcSVT = getLegalTypeToTransformTo(*TLO.DAG.getContext(), SrcSVT);

        SmallVector<SDValue> MaskElts;
        MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
        MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
        SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
        if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
                ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
          Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
          return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
        }
      }
    }
    break;
  }

  // TODO: There are more binop opcodes that could be handled here - MIN,
  // MAX, saturated math, etc.
  case ISD::ADD: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);
    if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
      APInt UndefLHS, ZeroLHS;
      if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
                                     Depth + 1, /*AssumeSingleUse*/ true))
        return true;
    }
    [[fallthrough]];
  }
  case ISD::AVGCEILS:
  case ISD::AVGCEILU:
  case ISD::AVGFLOORS:
  case ISD::AVGFLOORU:
  case ISD::OR:
  case ISD::XOR:
  case ISD::SUB:
  case ISD::FADD:
  case ISD::FSUB:
  case ISD::FMUL:
  case ISD::FDIV:
  case ISD::FREM: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    APInt UndefRHS, ZeroRHS;
    if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
                                   Depth + 1))
      return true;
    APInt UndefLHS, ZeroLHS;
    if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
                                   Depth + 1))
      return true;

    KnownZero = ZeroLHS & ZeroRHS;
    KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    // TODO - use KnownUndef to relax the demandedelts?
    if (!DemandedElts.isAllOnes())
      if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
        return true;
    break;
  }
  case ISD::SHL:
  case ISD::SRL:
  case ISD::SRA:
  case ISD::ROTL:
  case ISD::ROTR: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    APInt UndefRHS, ZeroRHS;
    if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
                                   Depth + 1))
      return true;
    APInt UndefLHS, ZeroLHS;
    if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
                                   Depth + 1))
      return true;

    KnownZero = ZeroLHS;
    KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

    // Attempt to avoid multi-use ops if we don't need anything from them.
    // TODO - use KnownUndef to relax the demandedelts?
    if (!DemandedElts.isAllOnes())
      if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
        return true;
    break;
  }
  case ISD::MUL:
  case ISD::MULHU:
  case ISD::MULHS:
  case ISD::AND: {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    APInt SrcUndef, SrcZero;
    if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
                                   Depth + 1))
      return true;
    // FIXME: If we know that a demanded element was zero in Op1 we don't need
    // to demand it in Op0 - its guaranteed to be zero. There is however a
    // restriction, as we must not make any of the originally demanded elements
    // more poisonous. We could reduce amount of elements demanded, but then we
    // also need a to inform SimplifyDemandedVectorElts that some elements must
    // not be made more poisonous.
    if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
                                   TLO, Depth + 1))
      return true;

    KnownUndef &= DemandedElts;
    KnownZero &= DemandedElts;

    // If every element pair has a zero/undef/poison then just fold to zero.
    // fold (and x, undef/poison) -> 0  /  (and x, 0) -> 0
    // fold (mul x, undef/poison) -> 0  /  (mul x, 0) -> 0
    if (DemandedElts.isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));

    // If either side has a zero element, then the result element is zero, even
    // if the other is an UNDEF.
    // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
    // and then handle 'and' nodes with the rest of the binop opcodes.
    KnownZero |= SrcZero;
    KnownUndef &= SrcUndef;
    KnownUndef &= ~KnownZero;

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!DemandedElts.isAllOnes())
      if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
        return true;
    break;
  }
  case ISD::TRUNCATE:
  case ISD::SIGN_EXTEND:
  case ISD::ZERO_EXTEND:
    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                   KnownZero, TLO, Depth + 1))
      return true;

    if (!DemandedElts.isAllOnes())
      if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
              Op.getOperand(0), DemandedElts, TLO.DAG, Depth + 1))
        return TLO.CombineTo(Op, TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp));

    if (Op.getOpcode() == ISD::ZERO_EXTEND) {
      // zext(undef) upper bits are guaranteed to be zero.
      if (DemandedElts.isSubsetOf(KnownUndef))
        return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
      KnownUndef.clearAllBits();
    }
    break;
  case ISD::SINT_TO_FP:
  case ISD::UINT_TO_FP:
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT:
    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                   KnownZero, TLO, Depth + 1))
      return true;
    // Don't fall through to generic undef -> undef handling.
    return false;
  default: {
    if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
      if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
                                                  KnownZero, TLO, Depth))
        return true;
    } else {
      KnownBits Known;
      APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
      if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
                               TLO, Depth, AssumeSingleUse))
        return true;
    }
    break;
  }
  }
  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");

  // Constant fold all undef cases.
  // TODO: Handle zero cases as well.
  if (DemandedElts.isSubsetOf(KnownUndef))
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

  return false;
}

/// Determine which of the bits specified in Mask are known to be either zero or
/// one and return them in the Known.
void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
                                                   KnownBits &Known,
                                                   const APInt &DemandedElts,
                                                   const SelectionDAG &DAG,
                                                   unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use MaskedValueIsZero if you don't know whether Op"
         " is a target node!");
  Known.resetAll();
}

void TargetLowering::computeKnownBitsForTargetInstr(
    GISelValueTracking &Analysis, Register R, KnownBits &Known,
    const APInt &DemandedElts, const MachineRegisterInfo &MRI,
    unsigned Depth) const {
  Known.resetAll();
}

void TargetLowering::computeKnownFPClassForTargetInstr(
    GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
    const APInt &DemandedElts, const MachineRegisterInfo &MRI,
    unsigned Depth) const {
  Known.resetAll();
}

void TargetLowering::computeKnownBitsForFrameIndex(
  const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
  // The low bits are known zero if the pointer is aligned.
  Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
}

Align TargetLowering::computeKnownAlignForTargetInstr(
    GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
    unsigned Depth) const {
  return Align(1);
}

/// This method can be implemented by targets that want to expose additional
/// information about sign bits to the DAG Combiner.
unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
                                                         const APInt &,
                                                         const SelectionDAG &,
                                                         unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use ComputeNumSignBits if you don't know whether Op"
         " is a target node!");
  return 1;
}

unsigned TargetLowering::computeNumSignBitsForTargetInstr(
    GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
    const MachineRegisterInfo &MRI, unsigned Depth) const {
  return 1;
}

bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
    SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
    TargetLoweringOpt &TLO, unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use SimplifyDemandedVectorElts if you don't know whether Op"
         " is a target node!");
  return false;
}

bool TargetLowering::SimplifyDemandedBitsForTargetNode(
    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
    KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use SimplifyDemandedBits if you don't know whether Op"
         " is a target node!");
  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
  return false;
}

SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
    SelectionDAG &DAG, unsigned Depth) const {
  assert(
      (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
       Op.getOpcode() == ISD::INTRINSIC_VOID) &&
      "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
      " is a target node!");
  return SDValue();
}

SDValue
TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
                                        SDValue N1, MutableArrayRef<int> Mask,
                                        SelectionDAG &DAG) const {
  bool LegalMask = isShuffleMaskLegal(Mask, VT);
  if (!LegalMask) {
    std::swap(N0, N1);
    ShuffleVectorSDNode::commuteMask(Mask);
    LegalMask = isShuffleMaskLegal(Mask, VT);
  }

  if (!LegalMask)
    return SDValue();

  return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
}

const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
  return nullptr;
}

bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
    UndefPoisonKind Kind, unsigned Depth) const {
  assert(
      (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
       Op.getOpcode() == ISD::INTRINSIC_VOID) &&
      "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
      " is a target node!");

  // If Op can't create undef/poison and none of its operands are undef/poison
  // then Op is never undef/poison.
  return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
                                              /*ConsiderFlags*/ true, Depth) &&
         all_of(Op->ops(), [&](SDValue V) {
           return DAG.isGuaranteedNotToBeUndefOrPoison(V, Kind, Depth + 1);
         });
}

bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
    UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use canCreateUndefOrPoison if you don't know whether Op"
         " is a target node!");
  // Be conservative and return true.
  return true;
}

void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
                                                      KnownFPClass &Known,
                                                      const APInt &DemandedElts,
                                                      const SelectionDAG &DAG,
                                                      unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use computeKnownFPClass if you don't know whether Op"
         " is a target node!");
}

bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
                                                  const APInt &DemandedElts,
                                                  const SelectionDAG &DAG,
                                                  bool SNaN,
                                                  unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use isKnownNeverNaN if you don't know whether Op"
         " is a target node!");
  return false;
}

bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
                                               const APInt &DemandedElts,
                                               APInt &UndefElts,
                                               const SelectionDAG &DAG,
                                               unsigned Depth) const {
  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_VOID) &&
         "Should use isSplatValue if you don't know whether Op"
         " is a target node!");
  return false;
}

// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
// work with truncating build vectors and vectors with elements of less than
// 8 bits.
bool TargetLowering::isConstTrueVal(SDValue N) const {
  if (!N)
    return false;

  unsigned EltWidth;
  APInt CVal;
  if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
                                               /*AllowTruncation=*/true)) {
    CVal = CN->getAPIntValue();
    EltWidth = N.getValueType().getScalarSizeInBits();
  } else
    return false;

  // If this is a truncating splat, truncate the splat value.
  // Otherwise, we may fail to match the expected values below.
  if (EltWidth < CVal.getBitWidth())
    CVal = CVal.trunc(EltWidth);

  switch (getBooleanContents(N.getValueType())) {
  case UndefinedBooleanContent:
    return CVal[0];
  case ZeroOrOneBooleanContent:
    return CVal.isOne();
  case ZeroOrNegativeOneBooleanContent:
    return CVal.isAllOnes();
  }

  llvm_unreachable("Invalid boolean contents");
}

bool TargetLowering::isConstFalseVal(SDValue N) const {
  if (!N)
    return false;

  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
  if (!CN) {
    const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
    if (!BV)
      return false;

    // Only interested in constant splats, we don't care about undef
    // elements in identifying boolean constants and getConstantSplatNode
    // returns NULL if all ops are undef;
    CN = BV->getConstantSplatNode();
    if (!CN)
      return false;
  }

  if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
    return !CN->getAPIntValue()[0];

  return CN->isZero();
}

bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
                                       bool SExt) const {
  if (VT == MVT::i1)
    return N->isOne();

  TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
  switch (Cnt) {
  case TargetLowering::ZeroOrOneBooleanContent:
    // An extended value of 1 is always true, unless its original type is i1,
    // in which case it will be sign extended to -1.
    return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
  case TargetLowering::UndefinedBooleanContent:
  case TargetLowering::ZeroOrNegativeOneBooleanContent:
    return N->isAllOnes() && SExt;
  }
  llvm_unreachable("Unexpected enumeration.");
}

/// This helper function of SimplifySetCC tries to optimize the comparison when
/// either operand of the SetCC node is a bitwise-and instruction.
SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
                                         ISD::CondCode Cond, const SDLoc &DL,
                                         DAGCombinerInfo &DCI) const {
  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
    std::swap(N0, N1);

  SelectionDAG &DAG = DCI.DAG;
  EVT OpVT = N0.getValueType();
  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
      (Cond != ISD::SETEQ && Cond != ISD::SETNE))
    return SDValue();

  // (X & Y) != 0 --> zextOrTrunc(X & Y)
  // iff everything but LSB is known zero:
  if (Cond == ISD::SETNE && isNullConstant(N1) &&
      (getBooleanContents(OpVT) == TargetLowering::UndefinedBooleanContent ||
       getBooleanContents(OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
    unsigned NumEltBits = OpVT.getScalarSizeInBits();
    APInt UpperBits = APInt::getHighBitsSet(NumEltBits, NumEltBits - 1);
    if (DAG.MaskedValueIsZero(N0, UpperBits))
      return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT);
  }

  // Try to eliminate a power-of-2 mask constant by converting to a signbit
  // test in a narrow type that we can truncate to with no cost. Examples:
  // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
  // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
  // TODO: This conservatively checks for type legality on the source and
  //       destination types. That may inhibit optimizations, but it also
  //       allows setcc->shift transforms that may be more beneficial.
  auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
  if (AndC && isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
      isTypeLegal(OpVT) && N0.hasOneUse()) {
    EVT NarrowVT = EVT::getIntegerVT(*DAG.getContext(),
                                     AndC->getAPIntValue().getActiveBits());
    if (isTruncateFree(OpVT, NarrowVT) && isTypeLegal(NarrowVT)) {
      SDValue Trunc = DAG.getZExtOrTrunc(N0.getOperand(0), DL, NarrowVT);
      SDValue Zero = DAG.getConstant(0, DL, NarrowVT);
      return DAG.getSetCC(DL, VT, Trunc, Zero,
                          Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
    }
  }

  // Match these patterns in any of their permutations:
  // (X & Y) == Y
  // (X & Y) != Y
  SDValue X, Y;
  if (N0.getOperand(0) == N1) {
    X = N0.getOperand(1);
    Y = N0.getOperand(0);
  } else if (N0.getOperand(1) == N1) {
    X = N0.getOperand(0);
    Y = N0.getOperand(1);
  } else {
    return SDValue();
  }

  // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
  // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
  // its liable to create and infinite loop.
  SDValue Zero = DAG.getConstant(0, DL, OpVT);
  if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
      DAG.isKnownToBeAPowerOfTwo(Y)) {
    // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
    // Note that where Y is variable and is known to have at most one bit set
    // (for example, if it is Z & 1) we cannot do this; the expressions are not
    // equivalent when Y == 0.
    assert(OpVT.isInteger());
    Cond = ISD::getSetCCInverse(Cond, OpVT);
    if (DCI.isBeforeLegalizeOps() ||
        isCondCodeLegal(Cond, N0.getSimpleValueType()))
      return DAG.getSetCC(DL, VT, N0, Zero, Cond);
  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
    // If the target supports an 'and-not' or 'and-complement' logic operation,
    // try to use that to make a comparison operation more efficient.
    // But don't do this transform if the mask is a single bit because there are
    // more efficient ways to deal with that case (for example, 'bt' on x86 or
    // 'rlwinm' on PPC).

    // Bail out if the compare operand that we want to turn into a zero is
    // already a zero (otherwise, infinite loop).
    if (isNullConstant(Y))
      return SDValue();

    // Transform this into: ~X & Y == 0.
    SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
    SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
    return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
  }

  return SDValue();
}

/// This helper function of SimplifySetCC tries to optimize the comparison when
/// either operand of the SetCC node is a bitwise-or instruction.
/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
                                        ISD::CondCode Cond, const SDLoc &DL,
                                        DAGCombinerInfo &DCI) const {
  if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
    std::swap(N0, N1);

  SelectionDAG &DAG = DCI.DAG;
  EVT OpVT = N0.getValueType();
  if (!N0.hasOneUse() || !OpVT.isInteger() ||
      (Cond != ISD::SETEQ && Cond != ISD::SETNE))
    return SDValue();

  // (X | Y) == Y
  // (X | Y) != Y
  SDValue X;
  if (sd_match(N0, m_Or(m_Value(X), m_Specific(N1))) && hasAndNotCompare(X)) {
    // If the target supports an 'and-not' or 'and-complement' logic operation,
    // try to use that to make a comparison operation more efficient.

    // Bail out if the compare operand that we want to turn into a zero is
    // already a zero (otherwise, infinite loop).
    if (isNullConstant(N1))
      return SDValue();

    // Transform this into: X & ~Y ==/!= 0.
    SDValue NotY = DAG.getNOT(SDLoc(N1), N1, OpVT);
    SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, X, NotY);
    return DAG.getSetCC(DL, VT, NewAnd, DAG.getConstant(0, DL, OpVT), Cond);
  }

  return SDValue();
}

/// There are multiple IR patterns that could be checking whether certain
/// truncation of a signed number would be lossy or not. The pattern which is
/// best at IR level, may not lower optimally. Thus, we want to unfold it.
/// We are looking for the following pattern: (KeptBits is a constant)
///   (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
/// KeptBits also can't be 1, that would have been folded to  %x dstcond 0
/// We will unfold it into the natural trunc+sext pattern:
///   ((%x << C) a>> C) dstcond %x
/// Where  C = bitwidth(x) - KeptBits  and  C u< bitwidth(x)
SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
    EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
    const SDLoc &DL) const {
  // We must be comparing with a constant.
  ConstantSDNode *C1;
  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
    return SDValue();

  // N0 should be:  add %x, (1 << (KeptBits-1))
  if (N0->getOpcode() != ISD::ADD)
    return SDValue();

  // And we must be 'add'ing a constant.
  ConstantSDNode *C01;
  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
    return SDValue();

  SDValue X = N0->getOperand(0);
  EVT XVT = X.getValueType();

  // Validate constants ...

  APInt I1 = C1->getAPIntValue();

  ISD::CondCode NewCond;
  if (Cond == ISD::CondCode::SETULT) {
    NewCond = ISD::CondCode::SETEQ;
  } else if (Cond == ISD::CondCode::SETULE) {
    NewCond = ISD::CondCode::SETEQ;
    // But need to 'canonicalize' the constant.
    I1 += 1;
  } else if (Cond == ISD::CondCode::SETUGT) {
    NewCond = ISD::CondCode::SETNE;
    // But need to 'canonicalize' the constant.
    I1 += 1;
  } else if (Cond == ISD::CondCode::SETUGE) {
    NewCond = ISD::CondCode::SETNE;
  } else
    return SDValue();

  APInt I01 = C01->getAPIntValue();

  auto checkConstants = [&I1, &I01]() -> bool {
    // Both of them must be power-of-two, and the constant from setcc is bigger.
    return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
  };

  if (checkConstants()) {
    // Great, e.g. got  icmp ult i16 (add i16 %x, 128), 256
  } else {
    // What if we invert constants? (and the target predicate)
    I1.negate();
    I01.negate();
    assert(XVT.isInteger());
    NewCond = getSetCCInverse(NewCond, XVT);
    if (!checkConstants())
      return SDValue();
    // Great, e.g. got  icmp uge i16 (add i16 %x, -128), -256
  }

  // They are power-of-two, so which bit is set?
  const unsigned KeptBits = I1.logBase2();
  const unsigned KeptBitsMinusOne = I01.logBase2();

  // Magic!
  if (KeptBits != (KeptBitsMinusOne + 1))
    return SDValue();
  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");

  // We don't want to do this in every single case.
  SelectionDAG &DAG = DCI.DAG;
  if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
    return SDValue();

  // Unfold into:  sext_inreg(%x) cond %x
  // Where 'cond' will be either 'eq' or 'ne'.
  SDValue SExtInReg = DAG.getNode(
      ISD::SIGN_EXTEND_INREG, DL, XVT, X,
      DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), KeptBits)));
  return DAG.getSetCC(DL, SCCVT, SExtInReg, X, NewCond);
}

// (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
    EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
    DAGCombinerInfo &DCI, const SDLoc &DL) const {
  assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
         "Should be a comparison with 0.");
  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
         "Valid only for [in]equality comparisons.");

  unsigned NewShiftOpcode;
  SDValue X, C, Y;

  SelectionDAG &DAG = DCI.DAG;

  // Look for '(C l>>/<< Y)'.
  auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
    // The shift should be one-use.
    if (!V.hasOneUse())
      return false;
    unsigned OldShiftOpcode = V.getOpcode();
    switch (OldShiftOpcode) {
    case ISD::SHL:
      NewShiftOpcode = ISD::SRL;
      break;
    case ISD::SRL:
      NewShiftOpcode = ISD::SHL;
      break;
    default:
      return false; // must be a logical shift.
    }
    // We should be shifting a constant.
    // FIXME: best to use isConstantOrConstantVector().
    C = V.getOperand(0);
    ConstantSDNode *CC =
        isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
    if (!CC)
      return false;
    Y = V.getOperand(1);

    ConstantSDNode *XC =
        isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
    return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
        X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
  };

  // LHS of comparison should be an one-use 'and'.
  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
    return SDValue();

  X = N0.getOperand(0);
  SDValue Mask = N0.getOperand(1);

  // 'and' is commutative!
  if (!Match(Mask)) {
    std::swap(X, Mask);
    if (!Match(Mask))
      return SDValue();
  }

  EVT VT = X.getValueType();

  // Produce:
  // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
  SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
  SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
  return T2;
}

/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
/// handle the commuted versions of these patterns.
SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
                                           ISD::CondCode Cond, const SDLoc &DL,
                                           DAGCombinerInfo &DCI) const {
  unsigned BOpcode = N0.getOpcode();
  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
         "Unexpected binop");
  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");

  // (X + Y) == X --> Y == 0
  // (X - Y) == X --> Y == 0
  // (X ^ Y) == X --> Y == 0
  SelectionDAG &DAG = DCI.DAG;
  EVT OpVT = N0.getValueType();
  SDValue X = N0.getOperand(0);
  SDValue Y = N0.getOperand(1);
  if (X == N1)
    return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

  if (Y != N1)
    return SDValue();

  // (X + Y) == Y --> X == 0
  // (X ^ Y) == Y --> X == 0
  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
    return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

  // The shift would not be valid if the operands are boolean (i1).
  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
    return SDValue();

  // (X - Y) == Y --> X == Y << 1
  SDValue One = DAG.getShiftAmountConstant(1, OpVT, DL);
  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
  if (!DCI.isCalledByLegalizer())
    DCI.AddToWorklist(YShl1.getNode());
  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
}

static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
                                      SDValue N0, const APInt &C1,
                                      ISD::CondCode Cond, const SDLoc &dl,
                                      SelectionDAG &DAG) {
  // Look through truncs that don't change the value of a ctpop.
  // FIXME: Add vector support? Need to be careful with setcc result type below.
  SDValue CTPOP = N0;
  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
      N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
    CTPOP = N0.getOperand(0);

  if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
    return SDValue();

  EVT CTVT = CTPOP.getValueType();
  SDValue CTOp = CTPOP.getOperand(0);

  // Expand a power-of-2-or-zero comparison based on ctpop:
  // (ctpop x) u< 2 -> (x & x-1) == 0
  // (ctpop x) u> 1 -> (x & x-1) != 0
  if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
    // Keep the CTPOP if it is a cheap vector op.
    if (CTVT.isVector() && TLI.isCtpopFast(CTVT))
      return SDValue();

    unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
    if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
      return SDValue();
    if (C1 == 0 && (Cond == ISD::SETULT))
      return SDValue(); // This is handled elsewhere.

    unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

    SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
    SDValue Result = CTOp;
    for (unsigned i = 0; i < Passes; i++) {
      SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
      Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
    }
    ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
    return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
  }

  // Expand a power-of-2 comparison based on ctpop
  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
    // Keep the CTPOP if it is cheap.
    if (TLI.isCtpopFast(CTVT))
      return SDValue();

    SDValue Zero = DAG.getConstant(0, dl, CTVT);
    SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
    assert(CTVT.isInteger());
    SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);

    // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
    // check before emitting a potentially unnecessary op.
    if (DAG.isKnownNeverZero(CTOp)) {
      // (ctpop x) == 1 --> (x & x-1) == 0
      // (ctpop x) != 1 --> (x & x-1) != 0
      SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
      SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
      return RHS;
    }

    // (ctpop x) == 1 --> (x ^ x-1) >  x-1
    // (ctpop x) != 1 --> (x ^ x-1) <= x-1
    SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add);
    ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
    return DAG.getSetCC(dl, VT, Xor, Add, CmpCond);
  }

  return SDValue();
}

static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
                                   ISD::CondCode Cond, const SDLoc &dl,
                                   SelectionDAG &DAG) {
  if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
    return SDValue();

  auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
  if (!C1 || !(C1->isZero() || C1->isAllOnes()))
    return SDValue();

  auto getRotateSource = [](SDValue X) {
    if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
      return X.getOperand(0);
    return SDValue();
  };

  // Peek through a rotated value compared against 0 or -1:
  // (rot X, Y) == 0/-1 --> X == 0/-1
  // (rot X, Y) != 0/-1 --> X != 0/-1
  if (SDValue R = getRotateSource(N0))
    return DAG.getSetCC(dl, VT, R, N1, Cond);

  // Peek through an 'or' of a rotated value compared against 0:
  // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
  // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
  //
  // TODO: Add the 'and' with -1 sibling.
  // TODO: Recurse through a series of 'or' ops to find the rotate.
  EVT OpVT = N0.getValueType();
  if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
    if (SDValue R = getRotateSource(N0.getOperand(0))) {
      SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1));
      return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
    }
    if (SDValue R = getRotateSource(N0.getOperand(1))) {
      SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0));
      return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
    }
  }

  return SDValue();
}

static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
                                        ISD::CondCode Cond, const SDLoc &dl,
                                        SelectionDAG &DAG) {
  // If we are testing for all-bits-clear, we might be able to do that with
  // less shifting since bit-order does not matter.
  if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
    return SDValue();

  auto *C1 = isConstOrConstSplat(N1, /* AllowUndefs */ true);
  if (!C1 || !C1->isZero())
    return SDValue();

  if (!N0.hasOneUse() ||
      (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
    return SDValue();

  unsigned BitWidth = N0.getScalarValueSizeInBits();
  auto *ShAmtC = isConstOrConstSplat(N0.getOperand(2));
  if (!ShAmtC)
    return SDValue();

  uint64_t ShAmt = ShAmtC->getAPIntValue().urem(BitWidth);
  if (ShAmt == 0)
    return SDValue();

  // Canonicalize fshr as fshl to reduce pattern-matching.
  if (N0.getOpcode() == ISD::FSHR)
    ShAmt = BitWidth - ShAmt;

  // Match an 'or' with a specific operand 'Other' in either commuted variant.
  SDValue X, Y;
  auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
    if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
      return false;
    if (Or.getOperand(0) == Other) {
      X = Or.getOperand(0);
      Y = Or.getOperand(1);
      return true;
    }
    if (Or.getOperand(1) == Other) {
      X = Or.getOperand(1);
      Y = Or.getOperand(0);
      return true;
    }
    return false;
  };

  EVT OpVT = N0.getValueType();
  EVT ShAmtVT = N0.getOperand(2).getValueType();
  SDValue F0 = N0.getOperand(0);
  SDValue F1 = N0.getOperand(1);
  if (matchOr(F0, F1)) {
    // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
    SDValue NewShAmt = DAG.getConstant(ShAmt, dl, ShAmtVT);
    SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt);
    SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
    return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
  }
  if (matchOr(F1, F0)) {
    // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
    SDValue NewShAmt = DAG.getConstant(BitWidth - ShAmt, dl, ShAmtVT);
    SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt);
    SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X);
    return DAG.getSetCC(dl, VT, NewOr, N1, Cond);
  }

  return SDValue();
}

/// Try to simplify a setcc built with the specified operands and cc. If it is
/// unable to simplify it, return a null SDValue.
SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
                                      ISD::CondCode Cond, bool foldBooleans,
                                      DAGCombinerInfo &DCI,
                                      const SDLoc &dl) const {
  SelectionDAG &DAG = DCI.DAG;
  const DataLayout &Layout = DAG.getDataLayout();
  EVT OpVT = N0.getValueType();
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

  // Constant fold or commute setcc.
  if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
    return Fold;

  bool N0ConstOrSplat =
      isConstOrConstSplat(N0, /*AllowUndefs*/ false, /*AllowTruncate*/ true);
  bool N1ConstOrSplat =
      isConstOrConstSplat(N1, /*AllowUndefs*/ false, /*AllowTruncate*/ true);

  // Canonicalize toward having the constant on the RHS.
  // TODO: Handle non-splat vector constants. All undef causes trouble.
  // FIXME: We can't yet fold constant scalable vector splats, so avoid an
  // infinite loop here when we encounter one.
  ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
  if (N0ConstOrSplat && !N1ConstOrSplat &&
      (DCI.isBeforeLegalizeOps() ||
       isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
    return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

  // If we have a subtract with the same 2 non-constant operands as this setcc
  // -- but in reverse order -- then try to commute the operands of this setcc
  // to match. A matching pair of setcc (cmp) and sub may be combined into 1
  // instruction on some targets.
  if (!N0ConstOrSplat && !N1ConstOrSplat &&
      (DCI.isBeforeLegalizeOps() ||
       isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
      DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
      !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
    return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

  if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
    return V;

  if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
    return V;

  if (auto *N1C = isConstOrConstSplat(N1)) {
    const APInt &C1 = N1C->getAPIntValue();

    // Optimize some CTPOP cases.
    if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
      return V;

    // For equality to 0 of a no-wrap multiply, decompose and test each op:
    // X * Y == 0 --> (X == 0) || (Y == 0)
    // X * Y != 0 --> (X != 0) && (Y != 0)
    // TODO: This bails out if minsize is set, but if the target doesn't have a
    //       single instruction multiply for this type, it would likely be
    //       smaller to decompose.
    if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
        (N0->getFlags().hasNoUnsignedWrap() ||
         N0->getFlags().hasNoSignedWrap()) &&
        !Attr.hasFnAttr(Attribute::MinSize)) {
      SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
      SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond);
      unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
      return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero);
    }

    // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
    // equality comparison, then we're just comparing whether X itself is
    // zero.
    if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
        N0.getOperand(0).getOpcode() == ISD::CTLZ &&
        llvm::has_single_bit<uint32_t>(N0.getScalarValueSizeInBits())) {
      if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
        if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
            ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
          if ((C1 == 0) == (Cond == ISD::SETEQ)) {
            // (srl (ctlz x), 5) == 0  -> X != 0
            // (srl (ctlz x), 5) != 1  -> X != 0
            Cond = ISD::SETNE;
          } else {
            // (srl (ctlz x), 5) != 0  -> X == 0
            // (srl (ctlz x), 5) == 1  -> X == 0
            Cond = ISD::SETEQ;
          }
          SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
          return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
                              Cond);
        }
      }
    }
  }

  // setcc X, 0, setlt --> X  (when X is all sign bits)
  // setcc X, 0, setne --> X  (when X is all sign bits)
  //
  // When we know that X has 0 or -1 in each element (or scalar), this
  // comparison will produce X. This is only true when boolean contents are
  // represented via 0s and -1s.
  if (VT == OpVT &&
      // Check that the result of setcc is 0 and -1.
      getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent &&
      // Match only for checks X < 0 and X != 0
      (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(N1) &&
      // The identity holds iff we know all sign bits for all lanes.
      DAG.ComputeNumSignBits(N0) == N0.getScalarValueSizeInBits())
    return N0;

  // FIXME: Support vectors.
  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
    const APInt &C1 = N1C->getAPIntValue();

    // (zext x) == C --> x == (trunc C)
    // (sext x) == C --> x == (trunc C)
    if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        DCI.isBeforeLegalize() && N0->hasOneUse()) {
      unsigned MinBits = N0.getValueSizeInBits();
      SDValue PreExt;
      bool Signed = false;
      if (N0->getOpcode() == ISD::ZERO_EXTEND) {
        // ZExt
        MinBits = N0->getOperand(0).getValueSizeInBits();
        PreExt = N0->getOperand(0);
      } else if (N0->getOpcode() == ISD::AND) {
        // DAGCombine turns costly ZExts into ANDs
        if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
          if ((C->getAPIntValue()+1).isPowerOf2()) {
            MinBits = C->getAPIntValue().countr_one();
            PreExt = N0->getOperand(0);
          }
      } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
        // SExt
        MinBits = N0->getOperand(0).getValueSizeInBits();
        PreExt = N0->getOperand(0);
        Signed = true;
      } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
        // ZEXTLOAD / SEXTLOAD
        if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
          MinBits = LN0->getMemoryVT().getSizeInBits();
          PreExt = N0;
        } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
          Signed = true;
          MinBits = LN0->getMemoryVT().getSizeInBits();
          PreExt = N0;
        }
      }

      // Figure out how many bits we need to preserve this constant.
      unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();

      // Make sure we're not losing bits from the constant.
      if (MinBits > 0 &&
          MinBits < C1.getBitWidth() &&
          MinBits >= ReqdBits) {
        EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
        if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
          // Will get folded away.
          SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
          if (MinBits == 1 && C1 == 1)
            // Invert the condition.
            return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
                                Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
          SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
          return DAG.getSetCC(dl, VT, Trunc, C, Cond);
        }

        // If truncating the setcc operands is not desirable, we can still
        // simplify the expression in some cases:
        // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
        // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
        // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
        // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
        // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
        // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
        SDValue TopSetCC = N0->getOperand(0);
        unsigned N0Opc = N0->getOpcode();
        bool SExt = (N0Opc == ISD::SIGN_EXTEND);
        if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
            TopSetCC.getOpcode() == ISD::SETCC &&
            (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
            (isConstFalseVal(N1) ||
             isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

          bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
                         (!N1C->isZero() && Cond == ISD::SETNE);

          if (!Inverse)
            return TopSetCC;

          ISD::CondCode InvCond = ISD::getSetCCInverse(
              cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
              TopSetCC.getOperand(0).getValueType());
          return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
                                      TopSetCC.getOperand(1),
                                      InvCond);
        }
      }
    }

    // If the LHS is '(and load, const)', the RHS is 0, the test is for
    // equality or unsigned, and all 1 bits of the const are in the same
    // partial word, see if we can shorten the load.
    if (DCI.isBeforeLegalize() &&
        !ISD::isSignedIntSetCC(Cond) &&
        N0.getOpcode() == ISD::AND && C1 == 0 &&
        N0.getNode()->hasOneUse() &&
        isa<LoadSDNode>(N0.getOperand(0)) &&
        N0.getOperand(0).getNode()->hasOneUse() &&
        isa<ConstantSDNode>(N0.getOperand(1))) {
      auto *Lod = cast<LoadSDNode>(N0.getOperand(0));
      APInt bestMask;
      unsigned bestWidth = 0, bestOffset = 0;
      if (Lod->isSimple() && Lod->isUnindexed() &&
          (Lod->getMemoryVT().isByteSized() ||
           isPaddedAtMostSignificantBitsWhenStored(Lod->getMemoryVT()))) {
        unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
        unsigned origWidth = N0.getValueSizeInBits();
        unsigned maskWidth = origWidth;
        // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
        // 8 bits, but have to be careful...
        if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
          origWidth = Lod->getMemoryVT().getSizeInBits();
        const APInt &Mask = N0.getConstantOperandAPInt(1);
        // Only consider power-of-2 widths (and at least one byte) as candiates
        // for the narrowed load.
        for (unsigned width = 8; width < origWidth; width *= 2) {
          EVT newVT = EVT::getIntegerVT(*DAG.getContext(), width);
          APInt newMask = APInt::getLowBitsSet(maskWidth, width);
          // Avoid accessing any padding here for now (we could use memWidth
          // instead of origWidth here otherwise).
          unsigned maxOffset = origWidth - width;
          for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
            if (Mask.isSubsetOf(newMask)) {
              unsigned ptrOffset =
                  Layout.isLittleEndian() ? offset : memWidth - width - offset;
              unsigned IsFast = 0;
              assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
              Align NewAlign = commonAlignment(Lod->getAlign(), ptrOffset / 8);
              if (shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT,
                                        ptrOffset / 8) &&
                  allowsMemoryAccess(
                      *DAG.getContext(), Layout, newVT, Lod->getAddressSpace(),
                      NewAlign, Lod->getMemOperand()->getFlags(), &IsFast) &&
                  IsFast) {
                bestOffset = ptrOffset / 8;
                bestMask = Mask.lshr(offset);
                bestWidth = width;
                break;
              }
            }
            newMask <<= 8;
          }
          if (bestWidth)
            break;
        }
      }
      if (bestWidth) {
        EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
        SDValue Ptr = Lod->getBasePtr();
        if (bestOffset != 0)
          Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(bestOffset));
        SDValue NewLoad =
            DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
                        Lod->getPointerInfo().getWithOffset(bestOffset),
                        Lod->getBaseAlign());
        SDValue And =
            DAG.getNode(ISD::AND, dl, newVT, NewLoad,
                        DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
        return DAG.getSetCC(dl, VT, And, DAG.getConstant(0LL, dl, newVT), Cond);
      }
    }

    // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
    if (N0.getOpcode() == ISD::ZERO_EXTEND) {
      unsigned InSize = N0.getOperand(0).getValueSizeInBits();

      // If the comparison constant has bits in the upper part, the
      // zero-extended value could never match.
      if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
                                              C1.getBitWidth() - InSize))) {
        switch (Cond) {
        case ISD::SETUGT:
        case ISD::SETUGE:
        case ISD::SETEQ:
          return DAG.getConstant(0, dl, VT);
        case ISD::SETULT:
        case ISD::SETULE:
        case ISD::SETNE:
          return DAG.getConstant(1, dl, VT);
        case ISD::SETGT:
        case ISD::SETGE:
          // True if the sign bit of C1 is set.
          return DAG.getConstant(C1.isNegative(), dl, VT);
        case ISD::SETLT:
        case ISD::SETLE:
          // True if the sign bit of C1 isn't set.
          return DAG.getConstant(C1.isNonNegative(), dl, VT);
        default:
          break;
        }
      }

      // Otherwise, we can perform the comparison with the low bits.
      switch (Cond) {
      case ISD::SETEQ:
      case ISD::SETNE:
      case ISD::SETUGT:
      case ISD::SETUGE:
      case ISD::SETULT:
      case ISD::SETULE: {
        EVT newVT = N0.getOperand(0).getValueType();
        // FIXME: Should use isNarrowingProfitable.
        if (DCI.isBeforeLegalizeOps() ||
            (isOperationLegal(ISD::SETCC, newVT) &&
             isCondCodeLegal(Cond, newVT.getSimpleVT()) &&
             isTypeDesirableForOp(ISD::SETCC, newVT))) {
          EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
          SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

          SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
                                          NewConst, Cond);
          return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
        }
        break;
      }
      default:
        break; // todo, be more careful with signed comparisons
      }
    } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
               (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
               !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
                                      OpVT)) {
      EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
      unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
      EVT ExtDstTy = N0.getValueType();
      unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

      // If the constant doesn't fit into the number of bits for the source of
      // the sign extension, it is impossible for both sides to be equal.
      if (C1.getSignificantBits() > ExtSrcTyBits)
        return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

      assert(ExtDstTy == N0.getOperand(0).getValueType() &&
             ExtDstTy != ExtSrcTy && "Unexpected types!");
      APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
      SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
                                   DAG.getConstant(Imm, dl, ExtDstTy));
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(ZextOp.getNode());
      // Otherwise, make this a use of a zext.
      return DAG.getSetCC(dl, VT, ZextOp,
                          DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
    } else if ((N1C->isZero() || N1C->isOne()) &&
               (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
      // SETCC (X), [0|1], [EQ|NE]  -> X if X is known 0/1. i1 types are
      // excluded as they are handled below whilst checking for foldBooleans.
      if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
          isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
          (N0.getValueType() == MVT::i1 ||
           getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
          DAG.MaskedValueIsZero(
              N0, APInt::getBitsSetFrom(N0.getValueSizeInBits(), 1))) {
        bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
        if (TrueWhenTrue)
          return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
        // Invert the condition.
        if (N0.getOpcode() == ISD::SETCC) {
          ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
          CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
          if (DCI.isBeforeLegalizeOps() ||
              isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
            return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
        }
      }

      if ((N0.getOpcode() == ISD::XOR ||
           (N0.getOpcode() == ISD::AND &&
            N0.getOperand(0).getOpcode() == ISD::XOR &&
            N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
          isOneConstant(N0.getOperand(1))) {
        // If this is (X^1) == 0/1, swap the RHS and eliminate the xor.  We
        // can only do this if the top bits are known zero.
        unsigned BitWidth = N0.getValueSizeInBits();
        if (DAG.MaskedValueIsZero(N0,
                                  APInt::getHighBitsSet(BitWidth,
                                                        BitWidth-1))) {
          // Okay, get the un-inverted input value.
          SDValue Val;
          if (N0.getOpcode() == ISD::XOR) {
            Val = N0.getOperand(0);
          } else {
            assert(N0.getOpcode() == ISD::AND &&
                    N0.getOperand(0).getOpcode() == ISD::XOR);
            // ((X^1)&1)^1 -> X & 1
            Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
                              N0.getOperand(0).getOperand(0),
                              N0.getOperand(1));
          }

          return DAG.getSetCC(dl, VT, Val, N1,
                              Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
        }
      } else if (N1C->isOne()) {
        SDValue Op0 = N0;
        if (Op0.getOpcode() == ISD::TRUNCATE)
          Op0 = Op0.getOperand(0);

        if ((Op0.getOpcode() == ISD::XOR) &&
            Op0.getOperand(0).getOpcode() == ISD::SETCC &&
            Op0.getOperand(1).getOpcode() == ISD::SETCC) {
          SDValue XorLHS = Op0.getOperand(0);
          SDValue XorRHS = Op0.getOperand(1);
          // Ensure that the input setccs return an i1 type or 0/1 value.
          if (Op0.getValueType() == MVT::i1 ||
              (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
                      ZeroOrOneBooleanContent &&
               getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
                        ZeroOrOneBooleanContent)) {
            // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
            Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
            return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
          }
        }
        if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
          // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
          if (Op0.getValueType().bitsGT(VT))
            Op0 = DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
                          DAG.getConstant(1, dl, VT));
          else if (Op0.getValueType().bitsLT(VT))
            Op0 = DAG.getNode(ISD::AND, dl, VT,
                        DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
                        DAG.getConstant(1, dl, VT));

          return DAG.getSetCC(dl, VT, Op0,
                              DAG.getConstant(0, dl, Op0.getValueType()),
                              Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
        }
        if (Op0.getOpcode() == ISD::AssertZext &&
            cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
          return DAG.getSetCC(dl, VT, Op0,
                              DAG.getConstant(0, dl, Op0.getValueType()),
                              Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
      }
    }

    // Given:
    //   icmp eq/ne (urem %x, %y), 0
    // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
    //   icmp eq/ne %x, 0
    if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
        (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
      KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
      KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
      if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
        return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
    }

    // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
    //  and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
    if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
        N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
        N1C->isAllOnes()) {
      return DAG.getSetCC(dl, VT, N0.getOperand(0),
                          DAG.getConstant(0, dl, OpVT),
                          Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
    }

    // fold (setcc (trunc x) c) -> (setcc x c)
    if (N0.getOpcode() == ISD::TRUNCATE &&
        ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Cond)) ||
         (N0->getFlags().hasNoSignedWrap() &&
          !ISD::isUnsignedIntSetCC(Cond))) &&
        isTypeDesirableForOp(ISD::SETCC, N0.getOperand(0).getValueType())) {
      EVT NewVT = N0.getOperand(0).getValueType();
      SDValue NewConst = DAG.getConstant(
          (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Cond))
              ? C1.sext(NewVT.getSizeInBits())
              : C1.zext(NewVT.getSizeInBits()),
          dl, NewVT);
      return DAG.getSetCC(dl, VT, N0.getOperand(0), NewConst, Cond);
    }

    if (SDValue V =
            optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
      return V;
  }

  // These simplifications apply to splat vectors as well.
  // TODO: Handle more splat vector cases.
  if (auto *N1C = isConstOrConstSplat(N1)) {
    const APInt &C1 = N1C->getAPIntValue();

    APInt MinVal, MaxVal;
    unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
    if (ISD::isSignedIntSetCC(Cond)) {
      MinVal = APInt::getSignedMinValue(OperandBitSize);
      MaxVal = APInt::getSignedMaxValue(OperandBitSize);
    } else {
      MinVal = APInt::getMinValue(OperandBitSize);
      MaxVal = APInt::getMaxValue(OperandBitSize);
    }

    // Canonicalize GE/LE comparisons to use GT/LT comparisons.
    if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
      // X >= MIN --> true
      if (C1 == MinVal)
        return DAG.getBoolConstant(true, dl, VT, OpVT);

      if (!VT.isVector()) { // TODO: Support this for vectors.
        // X >= C0 --> X > (C0 - 1)
        APInt C = C1 - 1;
        ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
        if ((DCI.isBeforeLegalizeOps() ||
             isCondCodeLegal(NewCC, OpVT.getSimpleVT())) &&
            (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                  isLegalICmpImmediate(C.getSExtValue())))) {
          return DAG.getSetCC(dl, VT, N0,
                              DAG.getConstant(C, dl, N1.getValueType()),
                              NewCC);
        }
      }
    }

    if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
      // X <= MAX --> true
      if (C1 == MaxVal)
        return DAG.getBoolConstant(true, dl, VT, OpVT);

      // X <= C0 --> X < (C0 + 1)
      if (!VT.isVector()) { // TODO: Support this for vectors.
        APInt C = C1 + 1;
        ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
        if ((DCI.isBeforeLegalizeOps() ||
             isCondCodeLegal(NewCC, OpVT.getSimpleVT())) &&
            (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                  isLegalICmpImmediate(C.getSExtValue())))) {
          return DAG.getSetCC(dl, VT, N0,
                              DAG.getConstant(C, dl, N1.getValueType()),
                              NewCC);
        }
      }
    }

    if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
      if (C1 == MinVal)
        return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

      // TODO: Support this for vectors after legalize ops.
      if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
        // Canonicalize setlt X, Max --> setne X, Max
        if (C1 == MaxVal)
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

        // If we have setult X, 1, turn it into seteq X, 0
        if (C1 == MinVal+1)
          return DAG.getSetCC(dl, VT, N0,
                              DAG.getConstant(MinVal, dl, N0.getValueType()),
                              ISD::SETEQ);
      }
    }

    if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
      if (C1 == MaxVal)
        return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

      // TODO: Support this for vectors after legalize ops.
      if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
        // Canonicalize setgt X, Min --> setne X, Min
        if (C1 == MinVal)
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

        // If we have setugt X, Max-1, turn it into seteq X, Max
        if (C1 == MaxVal-1)
          return DAG.getSetCC(dl, VT, N0,
                              DAG.getConstant(MaxVal, dl, N0.getValueType()),
                              ISD::SETEQ);
      }
    }

    if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
      // (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
      if (C1.isZero())
        if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
                VT, N0, N1, Cond, DCI, dl))
          return CC;

      // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
      // For example, when high 32-bits of i64 X are known clear:
      // all bits clear: (X | (Y<<32)) ==  0 --> (X | Y) ==  0
      // all bits set:   (X | (Y<<32)) == -1 --> (X & Y) == -1
      bool CmpZero = N1C->isZero();
      bool CmpNegOne = N1C->isAllOnes();
      if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
        // Match or(lo,shl(hi,bw/2)) pattern.
        auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
          unsigned EltBits = V.getScalarValueSizeInBits();
          if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
            return false;
          SDValue LHS = V.getOperand(0);
          SDValue RHS = V.getOperand(1);
          APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
          // Unshifted element must have zero upperbits.
          if (RHS.getOpcode() == ISD::SHL &&
              isa<ConstantSDNode>(RHS.getOperand(1)) &&
              RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
              DAG.MaskedValueIsZero(LHS, HiBits)) {
            Lo = LHS;
            Hi = RHS.getOperand(0);
            return true;
          }
          if (LHS.getOpcode() == ISD::SHL &&
              isa<ConstantSDNode>(LHS.getOperand(1)) &&
              LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
              DAG.MaskedValueIsZero(RHS, HiBits)) {
            Lo = RHS;
            Hi = LHS.getOperand(0);
            return true;
          }
          return false;
        };

        auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
          unsigned EltBits = N0.getScalarValueSizeInBits();
          unsigned HalfBits = EltBits / 2;
          APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
          SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
          SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
          SDValue NewN0 =
              DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
          SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
          return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
        };

        SDValue Lo, Hi;
        if (IsConcat(N0, Lo, Hi))
          return MergeConcat(Lo, Hi);

        if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
          SDValue Lo0, Lo1, Hi0, Hi1;
          if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
              IsConcat(N0.getOperand(1), Lo1, Hi1)) {
            return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
                               DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
          }
        }
      }
    }

    // If we have "setcc X, C0", check to see if we can shrink the immediate
    // by changing cc.
    // TODO: Support this for vectors after legalize ops.
    if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
      // SETUGT X, SINTMAX  -> SETLT X, 0
      // SETUGE X, SINTMIN -> SETLT X, 0
      if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
          (Cond == ISD::SETUGE && C1.isMinSignedValue()))
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(0, dl, N1.getValueType()),
                            ISD::SETLT);

      // SETULT X, SINTMIN  -> SETGT X, -1
      // SETULE X, SINTMAX  -> SETGT X, -1
      if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
          (Cond == ISD::SETULE && C1.isMaxSignedValue()))
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getAllOnesConstant(dl, N1.getValueType()),
                            ISD::SETGT);
    }
  }

  // Back to non-vector simplifications.
  // TODO: Can we do these for vector splats?
  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
    const APInt &C1 = N1C->getAPIntValue();
    EVT ShValTy = N0.getValueType();

    // Fold bit comparisons when we can. This will result in an
    // incorrect value when boolean false is negative one, unless
    // the bitsize is 1 in which case the false value is the same
    // in practice regardless of the representation.
    if ((VT.getSizeInBits() == 1 ||
         getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
        (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
        N0.getOpcode() == ISD::AND) {
      if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
        if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0  -->  (X & 8) >> 3
          // Perform the xform if the AND RHS is a single bit.
          unsigned ShCt = AndRHS->getAPIntValue().logBase2();
          if (AndRHS->getAPIntValue().isPowerOf2() &&
              !shouldAvoidTransformToShift(ShValTy, ShCt)) {
            return DAG.getNode(
                ISD::TRUNCATE, dl, VT,
                DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                            DAG.getShiftAmountConstant(ShCt, ShValTy, dl)));
          }
        } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
          // (X & 8) == 8  -->  (X & 8) >> 3
          // Perform the xform if C1 is a single bit.
          unsigned ShCt = C1.logBase2();
          if (C1.isPowerOf2() && !shouldAvoidTransformToShift(ShValTy, ShCt)) {
            return DAG.getNode(
                ISD::TRUNCATE, dl, VT,
                DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                            DAG.getShiftAmountConstant(ShCt, ShValTy, dl)));
          }
        }
      }
    }

    if (C1.getSignificantBits() <= 64 &&
        !isLegalICmpImmediate(C1.getSExtValue())) {
      // (X & -256) == 256 -> (X >> 8) == 1
      if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
          N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
        if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
          const APInt &AndRHSC = AndRHS->getAPIntValue();
          if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(AndRHSC)) {
            unsigned ShiftBits = AndRHSC.countr_zero();
            if (!shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
              // If using an unsigned shift doesn't yield a legal compare
              // immediate, try using sra instead.
              APInt NewC = C1.lshr(ShiftBits);
              if (NewC.getSignificantBits() <= 64 &&
                  !isLegalICmpImmediate(NewC.getSExtValue())) {
                APInt SignedC = C1.ashr(ShiftBits);
                if (SignedC.getSignificantBits() <= 64 &&
                    isLegalICmpImmediate(SignedC.getSExtValue())) {
                  SDValue Shift = DAG.getNode(
                      ISD::SRA, dl, ShValTy, N0.getOperand(0),
                      DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
                  SDValue CmpRHS = DAG.getConstant(SignedC, dl, ShValTy);
                  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
                }
              }
              SDValue Shift = DAG.getNode(
                  ISD::SRL, dl, ShValTy, N0.getOperand(0),
                  DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
              SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
              return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
            }
          }
        }
      } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
                 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
        bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
        // X <  0x100000000 -> (X >> 32) <  1
        // X >= 0x100000000 -> (X >> 32) >= 1
        // X <= 0x0ffffffff -> (X >> 32) <  1
        // X >  0x0ffffffff -> (X >> 32) >= 1
        unsigned ShiftBits;
        APInt NewC = C1;
        ISD::CondCode NewCond = Cond;
        if (AdjOne) {
          ShiftBits = C1.countr_one();
          NewC = NewC + 1;
          NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
        } else {
          ShiftBits = C1.countr_zero();
        }
        NewC.lshrInPlace(ShiftBits);
        if (ShiftBits && NewC.getSignificantBits() <= 64 &&
            isLegalICmpImmediate(NewC.getSExtValue()) &&
            !shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
          SDValue Shift =
              DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                          DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
          SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
          return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
        }
      }
    }
  }

  if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
    auto *CFP = cast<ConstantFPSDNode>(N1);
    assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");

    // Otherwise, we know the RHS is not a NaN.  Simplify the node to drop the
    // constant if knowing that the operand is non-nan is enough.  We prefer to
    // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
    // materialize 0.0.
    if (Cond == ISD::SETO || Cond == ISD::SETUO)
      return DAG.getSetCC(dl, VT, N0, N0, Cond);

    // setcc (fneg x), C -> setcc swap(pred) x, -C
    if (N0.getOpcode() == ISD::FNEG) {
      ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
      if (DCI.isBeforeLegalizeOps() ||
          isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
        SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
        return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
      }
    }

    // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
    if (isOperationLegalOrCustom(ISD::IS_FPCLASS, N0.getValueType()) &&
        !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(0))) {
      bool IsFabs = N0.getOpcode() == ISD::FABS;
      SDValue Op = IsFabs ? N0.getOperand(0) : N0;
      if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
        FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
                                             : (IsFabs ? fcInf : fcPosInf);
        if (Cond == ISD::SETUEQ)
          Flag |= fcNan;
        return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op,
                           DAG.getTargetConstant(Flag, dl, MVT::i32));
      }
    }

    // If the condition is not legal, see if we can find an equivalent one
    // which is legal.
    if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
      // If the comparison was an awkward floating-point == or != and one of
      // the comparison operands is infinity or negative infinity, convert the
      // condition to a less-awkward <= or >=.
      if (CFP->getValueAPF().isInfinity()) {
        bool IsNegInf = CFP->getValueAPF().isNegative();
        ISD::CondCode NewCond = ISD::SETCC_INVALID;
        switch (Cond) {
        case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
        case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
        case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
        case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
        default: break;
        }
        if (NewCond != ISD::SETCC_INVALID &&
            isCondCodeLegal(NewCond, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, NewCond);
      }
    }
  }

  if (N0 == N1) {
    // The sext(setcc()) => setcc() optimization relies on the appropriate
    // constant being emitted.
    assert(!N0.getValueType().isInteger() &&
           "Integer types should be handled by FoldSetCC");

    bool EqTrue = ISD::isTrueWhenEqual(Cond);
    unsigned UOF = ISD::getUnorderedFlavor(Cond);
    if (UOF == 2) // FP operators that are undefined on NaNs.
      return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
    if (UOF == unsigned(EqTrue))
      return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
    // Otherwise, we can't fold it.  However, we can simplify it to SETUO/SETO
    // if it is not already.
    ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
    if (NewCond != Cond &&
        (DCI.isBeforeLegalizeOps() ||
                            isCondCodeLegal(NewCond, N0.getSimpleValueType())))
      return DAG.getSetCC(dl, VT, N0, N1, NewCond);
  }

  // ~X > ~Y --> Y > X
  // ~X < ~Y --> Y < X
  // ~X < C --> X > ~C
  // ~X > C --> X < ~C
  if ((isSignedIntSetCC(Cond) || isUnsignedIntSetCC(Cond)) &&
      N0.getValueType().isInteger()) {
    if (isBitwiseNot(N0)) {
      if (isBitwiseNot(N1))
        return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond);

      if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
          !DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(0))) {
        SDValue Not = DAG.getNOT(dl, N1, OpVT);
        return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond);
      }
    }
  }

  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      N0.getValueType().isInteger()) {
    if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
        N0.getOpcode() == ISD::XOR) {
      // Simplify (X+Y) == (X+Z) -->  Y == Z
      if (N0.getOpcode() == N1.getOpcode()) {
        if (N0.getOperand(0) == N1.getOperand(0))
          return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
        if (N0.getOperand(1) == N1.getOperand(1))
          return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
        if (isCommutativeBinOp(N0.getOpcode())) {
          // If X op Y == Y op X, try other combinations.
          if (N0.getOperand(0) == N1.getOperand(1))
            return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
                                Cond);
          if (N0.getOperand(1) == N1.getOperand(0))
            return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
                                Cond);
        }
      }

      // If RHS is a legal immediate value for a compare instruction, we need
      // to be careful about increasing register pressure needlessly.
      bool LegalRHSImm = false;

      if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
        if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
          // Turn (X+C1) == C2 --> X == C2-C1
          if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
            return DAG.getSetCC(
                dl, VT, N0.getOperand(0),
                DAG.getConstant(RHSC->getAPIntValue() - LHSR->getAPIntValue(),
                                dl, N0.getValueType()),
                Cond);

          // Turn (X^C1) == C2 --> X == C1^C2
          if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
            return DAG.getSetCC(
                dl, VT, N0.getOperand(0),
                DAG.getConstant(LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
                                dl, N0.getValueType()),
                Cond);
        }

        // Turn (C1-X) == C2 --> X == C1-C2
        if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
          if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
            return DAG.getSetCC(
                dl, VT, N0.getOperand(1),
                DAG.getConstant(SUBC->getAPIntValue() - RHSC->getAPIntValue(),
                                dl, N0.getValueType()),
                Cond);

        // Could RHSC fold directly into a compare?
        if (RHSC->getValueType(0).getSizeInBits() <= 64)
          LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
      }

      // (X+Y) == X --> Y == 0 and similar folds.
      // Don't do this if X is an immediate that can fold into a cmp
      // instruction and X+Y has other uses. It could be an induction variable
      // chain, and the transform would increase register pressure.
      if (!LegalRHSImm || N0.hasOneUse())
        if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
          return V;
    }

    if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
        N1.getOpcode() == ISD::XOR)
      if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
        return V;

    if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
      return V;

    if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, dl, DCI))
      return V;
  }

  // Fold remainder of division by a constant.
  if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
      N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    // When division is cheap or optimizing for minimum size,
    // fall through to DIVREM creation by skipping this fold.
    if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
      if (N0.getOpcode() == ISD::UREM) {
        if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
          return Folded;
      } else if (N0.getOpcode() == ISD::SREM) {
        if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
          return Folded;
      }
    }
  }

  // Fold away ALL boolean setcc's.
  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
    SDValue Temp;
    switch (Cond) {
    default: llvm_unreachable("Unknown integer setcc!");
    case ISD::SETEQ:  // X == Y  -> ~(X^Y)
      Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
      N0 = DAG.getNOT(dl, Temp, OpVT);
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(Temp.getNode());
      break;
    case ISD::SETNE:  // X != Y   -->  (X^Y)
      N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
      break;
    case ISD::SETGT:  // X >s Y   -->  X == 0 & Y == 1  -->  ~X & Y
    case ISD::SETULT: // X <u Y   -->  X == 0 & Y == 1  -->  ~X & Y
      Temp = DAG.getNOT(dl, N0, OpVT);
      N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(Temp.getNode());
      break;
    case ISD::SETLT:  // X <s Y   --> X == 1 & Y == 0  -->  ~Y & X
    case ISD::SETUGT: // X >u Y   --> X == 1 & Y == 0  -->  ~Y & X
      Temp = DAG.getNOT(dl, N1, OpVT);
      N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(Temp.getNode());
      break;
    case ISD::SETULE: // X <=u Y  --> X == 0 | Y == 1  -->  ~X | Y
    case ISD::SETGE:  // X >=s Y  --> X == 0 | Y == 1  -->  ~X | Y
      Temp = DAG.getNOT(dl, N0, OpVT);
      N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(Temp.getNode());
      break;
    case ISD::SETUGE: // X >=u Y  --> X == 1 | Y == 0  -->  ~Y | X
    case ISD::SETLE:  // X <=s Y  --> X == 1 | Y == 0  -->  ~Y | X
      Temp = DAG.getNOT(dl, N1, OpVT);
      N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
      break;
    }
    if (VT.getScalarType() != MVT::i1) {
      if (!DCI.isCalledByLegalizer())
        DCI.AddToWorklist(N0.getNode());
      // FIXME: If running after legalize, we probably can't do this.
      ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
      N0 = DAG.getNode(ExtendCode, dl, VT, N0);
    }
    return N0;
  }

  // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
  if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
      N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
      ((!ISD::isSignedIntSetCC(Cond) && N0->getFlags().hasNoUnsignedWrap() &&
        N1->getFlags().hasNoUnsignedWrap()) ||
       (!ISD::isUnsignedIntSetCC(Cond) && N0->getFlags().hasNoSignedWrap() &&
        N1->getFlags().hasNoSignedWrap())) &&
      isTypeDesirableForOp(ISD::SETCC, N0.getOperand(0).getValueType())) {
    return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
  }

  // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
  // TODO: Remove that .isVector() check
  if (VT.isVector() && isZeroOrZeroSplat(N1) && N0.getOpcode() == ISD::SUB &&
      N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Cond)) {
    return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), Cond);
  }

  // Could not fold it.
  return SDValue();
}

/// Returns true (and the GlobalValue and the offset) if the node is a
/// GlobalAddress + offset.
bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
                                    int64_t &Offset) const {

  SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
    GA = GASD->getGlobal();
    Offset += GASD->getOffset();
    return true;
  }

  if (N->isAnyAdd()) {
    SDValue N1 = N->getOperand(0);
    SDValue N2 = N->getOperand(1);
    if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
      if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
        Offset += V->getSExtValue();
        return true;
      }
    } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
      if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
        Offset += V->getSExtValue();
        return true;
      }
    }
  }

  return false;
}

SDValue TargetLowering::PerformDAGCombine(SDNode *N,
                                          DAGCombinerInfo &DCI) const {
  // Default implementation: no optimization.
  return SDValue();
}

//===----------------------------------------------------------------------===//
//  Inline Assembler Implementation Methods
//===----------------------------------------------------------------------===//

TargetLowering::ConstraintType
TargetLowering::getConstraintType(StringRef Constraint) const {
  unsigned S = Constraint.size();

  if (S == 1) {
    switch (Constraint[0]) {
    default: break;
    case 'r':
      return C_RegisterClass;
    case 'm': // memory
    case 'o': // offsetable
    case 'V': // not offsetable
      return C_Memory;
    case 'p': // Address.
      return C_Address;
    case 'n': // Simple Integer
    case 'E': // Floating Point Constant
    case 'F': // Floating Point Constant
      return C_Immediate;
    case 'i': // Simple Integer or Relocatable Constant
    case 's': // Relocatable Constant
    case 'X': // Allow ANY value.
    case 'I': // Target registers.
    case 'J':
    case 'K':
    case 'L':
    case 'M':
    case 'N':
    case 'O':
    case 'P':
    case '<':
    case '>':
      return C_Other;
    }
  }

  if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
    if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
      return C_Memory;
    return C_Register;
  }
  return C_Unknown;
}

/// Try to replace an X constraint, which matches anything, with another that
/// has more specific requirements based on the type of the corresponding
/// operand.
const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
  if (ConstraintVT.isInteger())
    return "r";
  if (ConstraintVT.isFloatingPoint())
    return "f"; // works for many targets
  return nullptr;
}

SDValue TargetLowering::LowerAsmOutputForConstraint(
    SDValue &Chain, SDValue &Glue, const SDLoc &DL,
    const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
  return SDValue();
}

/// Lower the specified operand into the Ops vector.
/// If it is invalid, don't add anything to Ops.
void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
                                                  StringRef Constraint,
                                                  std::vector<SDValue> &Ops,
                                                  SelectionDAG &DAG) const {

  if (Constraint.size() > 1)
    return;

  char ConstraintLetter = Constraint[0];
  switch (ConstraintLetter) {
  default: break;
  case 'X':    // Allows any operand
  case 'i':    // Simple Integer or Relocatable Constant
  case 'n':    // Simple Integer
  case 's': {  // Relocatable Constant

    ConstantSDNode *C;
    uint64_t Offset = 0;

    // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
    // etc., since getelementpointer is variadic. We can't use
    // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
    // while in this case the GA may be furthest from the root node which is
    // likely an ISD::ADD.
    while (true) {
      if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
        // gcc prints these as sign extended.  Sign extend value to 64 bits
        // now; without this it would get ZExt'd later in
        // ScheduleDAGSDNodes::EmitNode, which is very generic.
        bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
        BooleanContent BCont = getBooleanContents(MVT::i64);
        ISD::NodeType ExtOpc =
            IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
        int64_t ExtVal =
            ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
        Ops.push_back(
            DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
        return;
      }
      if (ConstraintLetter != 'n') {
        if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
          Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
                                                   GA->getValueType(0),
                                                   Offset + GA->getOffset()));
          return;
        }
        if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
          Ops.push_back(DAG.getTargetBlockAddress(
              BA->getBlockAddress(), BA->getValueType(0),
              Offset + BA->getOffset(), BA->getTargetFlags()));
          return;
        }
        if (isa<BasicBlockSDNode>(Op)) {
          Ops.push_back(Op);
          return;
        }
      }
      const unsigned OpCode = Op.getOpcode();
      if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
        if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
          Op = Op.getOperand(1);
        // Subtraction is not commutative.
        else if (OpCode == ISD::ADD &&
                 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
          Op = Op.getOperand(0);
        else
          return;
        Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
        continue;
      }
      return;
    }
    break;
  }
  }
}

void TargetLowering::CollectTargetIntrinsicOperands(
    const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
}

std::pair<unsigned, const TargetRegisterClass *>
TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
                                             StringRef Constraint,
                                             MVT VT) const {
  if (!Constraint.starts_with("{"))
    return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
  assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");

  // Remove the braces from around the name.
  StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

  std::pair<unsigned, const TargetRegisterClass *> R =
      std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

  // Figure out which register class contains this reg.
  for (const TargetRegisterClass *RC : RI->regclasses()) {
    // If none of the value types for this register class are valid, we
    // can't use it.  For example, 64-bit reg classes on 32-bit targets.
    if (!isLegalRC(*RI, *RC))
      continue;

    for (const MCPhysReg &PR : *RC) {
      if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
        std::pair<unsigned, const TargetRegisterClass *> S =
            std::make_pair(PR, RC);

        // If this register class has the requested value type, return it,
        // otherwise keep searching and return the first class found
        // if no other is found which explicitly has the requested type.
        if (RI->isTypeLegalForClass(*RC, VT))
          return S;
        if (!R.second)
          R = S;
      }
    }
  }

  return R;
}

//===----------------------------------------------------------------------===//
// Constraint Selection.

/// Return true of this is an input operand that is a matching constraint like
/// "4".
bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
  assert(!ConstraintCode.empty() && "No known constraint!");
  return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
}

/// If this is an input matching constraint, this method returns the output
/// operand it matches.
unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
  assert(!ConstraintCode.empty() && "No known constraint!");
  return atoi(ConstraintCode.c_str());
}

/// Split up the constraint string from the inline assembly value into the
/// specific constraints and their prefixes, and also tie in the associated
/// operand values.
/// If this returns an empty vector, and if the constraint string itself
/// isn't empty, there was an error parsing.
TargetLowering::AsmOperandInfoVector
TargetLowering::ParseConstraints(const DataLayout &DL,
                                 const TargetRegisterInfo *TRI,
                                 const CallBase &Call) const {
  /// Information about all of the constraints.
  AsmOperandInfoVector ConstraintOperands;
  const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
  unsigned maCount = 0; // Largest number of multiple alternative constraints.

  // Do a prepass over the constraints, canonicalizing them, and building up the
  // ConstraintOperands list.
  unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
  unsigned ResNo = 0; // ResNo - The result number of the next output.
  unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.

  for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
    ConstraintOperands.emplace_back(std::move(CI));
    AsmOperandInfo &OpInfo = ConstraintOperands.back();

    // Update multiple alternative constraint count.
    if (OpInfo.multipleAlternatives.size() > maCount)
      maCount = OpInfo.multipleAlternatives.size();

    OpInfo.ConstraintVT = MVT::Other;

    // Compute the value type for each operand.
    switch (OpInfo.Type) {
    case InlineAsm::isOutput: {
      // Indirect outputs just consume an argument.
      if (OpInfo.isIndirect) {
        OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
        break;
      }

      // The return value of the call is this value.  As such, there is no
      // corresponding argument.
      assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
      EVT VT;
      if (auto *STy = dyn_cast<StructType>(Call.getType())) {
        VT = getAsmOperandValueType(DL, STy->getElementType(ResNo));
      } else {
        assert(ResNo == 0 && "Asm only has one result!");
        VT = getAsmOperandValueType(DL, Call.getType());
      }
      OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
      ++ResNo;
      break;
    }
    case InlineAsm::isInput:
      OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
      break;
    case InlineAsm::isLabel:
      OpInfo.CallOperandVal = cast<CallBrInst>(&Call)->getIndirectDest(LabelNo);
      ++LabelNo;
      continue;
    case InlineAsm::isClobber:
      // Nothing to do.
      break;
    }

    if (OpInfo.CallOperandVal) {
      llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
      if (OpInfo.isIndirect) {
        OpTy = Call.getParamElementType(ArgNo);
        assert(OpTy && "Indirect operand must have elementtype attribute");
      }

      // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
      if (StructType *STy = dyn_cast<StructType>(OpTy))
        if (STy->getNumElements() == 1)
          OpTy = STy->getElementType(0);

      // If OpTy is not a single value, it may be a struct/union that we
      // can tile with integers.
      if (!OpTy->isSingleValueType() && OpTy->isSized()) {
        unsigned BitSize = DL.getTypeSizeInBits(OpTy);
        switch (BitSize) {
        default: break;
        case 1:
        case 8:
        case 16:
        case 32:
        case 64:
        case 128:
          OpTy = IntegerType::get(OpTy->getContext(), BitSize);
          break;
        }
      }

      EVT VT = getAsmOperandValueType(DL, OpTy, true);
      OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
      ArgNo++;
    }
  }

  // If we have multiple alternative constraints, select the best alternative.
  if (!ConstraintOperands.empty()) {
    if (maCount) {
      unsigned bestMAIndex = 0;
      int bestWeight = -1;
      // weight:  -1 = invalid match, and 0 = so-so match to 5 = good match.
      int weight = -1;
      unsigned maIndex;
      // Compute the sums of the weights for each alternative, keeping track
      // of the best (highest weight) one so far.
      for (maIndex = 0; maIndex < maCount; ++maIndex) {
        int weightSum = 0;
        for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
             cIndex != eIndex; ++cIndex) {
          AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
          if (OpInfo.Type == InlineAsm::isClobber)
            continue;

          // If this is an output operand with a matching input operand,
          // look up the matching input. If their types mismatch, e.g. one
          // is an integer, the other is floating point, or their sizes are
          // different, flag it as an maCantMatch.
          if (OpInfo.hasMatchingInput()) {
            AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
            if (OpInfo.ConstraintVT != Input.ConstraintVT) {
              if ((OpInfo.ConstraintVT.isInteger() !=
                   Input.ConstraintVT.isInteger()) ||
                  (OpInfo.ConstraintVT.getSizeInBits() !=
                   Input.ConstraintVT.getSizeInBits())) {
                weightSum = -1; // Can't match.
                break;
              }
            }
          }
          weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
          if (weight == -1) {
            weightSum = -1;
            break;
          }
          weightSum += weight;
        }
        // Update best.
        if (weightSum > bestWeight) {
          bestWeight = weightSum;
          bestMAIndex = maIndex;
        }
      }

      // Now select chosen alternative in each constraint.
      for (AsmOperandInfo &cInfo : ConstraintOperands)
        if (cInfo.Type != InlineAsm::isClobber)
          cInfo.selectAlternative(bestMAIndex);
    }
  }

  // Check and hook up tied operands, choose constraint code to use.
  for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
       cIndex != eIndex; ++cIndex) {
    AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

    // If this is an output operand with a matching input operand, look up the
    // matching input. If their types mismatch, e.g. one is an integer, the
    // other is floating point, or their sizes are different, flag it as an
    // error.
    if (OpInfo.hasMatchingInput()) {
      AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

      if (OpInfo.ConstraintVT != Input.ConstraintVT) {
        std::pair<unsigned, const TargetRegisterClass *> MatchRC =
            getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
                                         OpInfo.ConstraintVT);
        std::pair<unsigned, const TargetRegisterClass *> InputRC =
            getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
                                         Input.ConstraintVT);
        const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
                                    OpInfo.ConstraintVT.isFloatingPoint();
        const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
                                   Input.ConstraintVT.isFloatingPoint();
        if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
            (MatchRC.second != InputRC.second)) {
          report_fatal_error("Unsupported asm: input constraint"
                             " with a matching output constraint of"
                             " incompatible type!");
        }
      }
    }
  }

  return ConstraintOperands;
}

/// Return a number indicating our preference for chosing a type of constraint
/// over another, for the purpose of sorting them. Immediates are almost always
/// preferrable (when they can be emitted). A higher return value means a
/// stronger preference for one constraint type relative to another.
/// FIXME: We should prefer registers over memory but doing so may lead to
/// unrecoverable register exhaustion later.
/// https://github.com/llvm/llvm-project/issues/20571
static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
  switch (CT) {
  case TargetLowering::C_Immediate:
  case TargetLowering::C_Other:
    return 4;
  case TargetLowering::C_Memory:
  case TargetLowering::C_Address:
    return 3;
  case TargetLowering::C_RegisterClass:
    return 2;
  case TargetLowering::C_Register:
    return 1;
  case TargetLowering::C_Unknown:
    return 0;
  }
  llvm_unreachable("Invalid constraint type");
}

/// Examine constraint type and operand type and determine a weight value.
/// This object must already have been set up with the operand type
/// and the current alternative constraint selected.
TargetLowering::ConstraintWeight
  TargetLowering::getMultipleConstraintMatchWeight(
    AsmOperandInfo &info, int maIndex) const {
  InlineAsm::ConstraintCodeVector *rCodes;
  if (maIndex >= (int)info.multipleAlternatives.size())
    rCodes = &info.Codes;
  else
    rCodes = &info.multipleAlternatives[maIndex].Codes;
  ConstraintWeight BestWeight = CW_Invalid;

  // Loop over the options, keeping track of the most general one.
  for (const std::string &rCode : *rCodes) {
    ConstraintWeight weight =
        getSingleConstraintMatchWeight(info, rCode.c_str());
    if (weight > BestWeight)
      BestWeight = weight;
  }

  return BestWeight;
}

/// Examine constraint type and operand type and determine a weight value.
/// This object must already have been set up with the operand type
/// and the current alternative constraint selected.
TargetLowering::ConstraintWeight
  TargetLowering::getSingleConstraintMatchWeight(
    AsmOperandInfo &info, const char *constraint) const {
  ConstraintWeight weight = CW_Invalid;
  Value *CallOperandVal = info.CallOperandVal;
    // If we don't have a value, we can't do a match,
    // but allow it at the lowest weight.
  if (!CallOperandVal)
    return CW_Default;
  // Look at the constraint type.
  switch (*constraint) {
    case 'i': // immediate integer.
    case 'n': // immediate integer with a known value.
      if (isa<ConstantInt>(CallOperandVal))
        weight = CW_Constant;
      break;
    case 's': // non-explicit intregal immediate.
      if (isa<GlobalValue>(CallOperandVal))
        weight = CW_Constant;
      break;
    case 'E': // immediate float if host format.
    case 'F': // immediate float.
      if (isa<ConstantFP>(CallOperandVal))
        weight = CW_Constant;
      break;
    case '<': // memory operand with autodecrement.
    case '>': // memory operand with autoincrement.
    case 'm': // memory operand.
    case 'o': // offsettable memory operand
    case 'V': // non-offsettable memory operand
      weight = CW_Memory;
      break;
    case 'r': // general register.
    case 'g': // general register, memory operand or immediate integer.
              // note: Clang converts "g" to "imr".
      if (CallOperandVal->getType()->isIntegerTy())
        weight = CW_Register;
      break;
    case 'X': // any operand.
  default:
    weight = CW_Default;
    break;
  }
  return weight;
}

/// If there are multiple different constraints that we could pick for this
/// operand (e.g. "imr") try to pick the 'best' one.
/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
/// into seven classes:
///    Register      -> one specific register
///    RegisterClass -> a group of regs
///    Memory        -> memory
///    Address       -> a symbolic memory reference
///    Immediate     -> immediate values
///    Other         -> magic values (such as "Flag Output Operands")
///    Unknown       -> something we don't recognize yet and can't handle
/// Ideally, we would pick the most specific constraint possible: if we have
/// something that fits into a register, we would pick it.  The problem here
/// is that if we have something that could either be in a register or in
/// memory that use of the register could cause selection of *other*
/// operands to fail: they might only succeed if we pick memory.  Because of
/// this the heuristic we use is:
///
///  1) If there is an 'other' constraint, and if the operand is valid for
///     that constraint, use it.  This makes us take advantage of 'i'
///     constraints when available.
///  2) Otherwise, pick the most general constraint present.  This prefers
///     'm' over 'r', for example.
///
TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
    TargetLowering::AsmOperandInfo &OpInfo) const {
  ConstraintGroup Ret;

  Ret.reserve(OpInfo.Codes.size());
  for (StringRef Code : OpInfo.Codes) {
    TargetLowering::ConstraintType CType = getConstraintType(Code);

    // Indirect 'other' or 'immediate' constraints are not allowed.
    if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
                               CType == TargetLowering::C_Register ||
                               CType == TargetLowering::C_RegisterClass))
      continue;

    // Things with matching constraints can only be registers, per gcc
    // documentation.  This mainly affects "g" constraints.
    if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
      continue;

    Ret.emplace_back(Code, CType);
  }

  llvm::stable_sort(Ret, [](ConstraintPair a, ConstraintPair b) {
    return getConstraintPiority(a.second) > getConstraintPiority(b.second);
  });

  return Ret;
}

/// If we have an immediate, see if we can lower it. Return true if we can,
/// false otherwise.
static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
                                     SDValue Op, SelectionDAG *DAG,
                                     const TargetLowering &TLI) {

  assert((P.second == TargetLowering::C_Other ||
          P.second == TargetLowering::C_Immediate) &&
         "need immediate or other");

  if (!Op.getNode())
    return false;

  std::vector<SDValue> ResultOps;
  TLI.LowerAsmOperandForConstraint(Op, P.first, ResultOps, *DAG);
  return !ResultOps.empty();
}

/// Determines the constraint code and constraint type to use for the specific
/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
                                            SDValue Op,
                                            SelectionDAG *DAG) const {
  assert(!OpInfo.Codes.empty() && "Must have at least one constraint");

  // Single-letter constraints ('r') are very common.
  if (OpInfo.Codes.size() == 1) {
    OpInfo.ConstraintCode = OpInfo.Codes[0];
    OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
  } else {
    ConstraintGroup G = getConstraintPreferences(OpInfo);
    if (G.empty())
      return;

    unsigned BestIdx = 0;
    for (const unsigned E = G.size();
         BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
                         G[BestIdx].second == TargetLowering::C_Immediate);
         ++BestIdx) {
      if (lowerImmediateIfPossible(G[BestIdx], Op, DAG, *this))
        break;
      // If we're out of constraints, just pick the first one.
      if (BestIdx + 1 == E) {
        BestIdx = 0;
        break;
      }
    }

    OpInfo.ConstraintCode = G[BestIdx].first;
    OpInfo.ConstraintType = G[BestIdx].second;
  }

  // 'X' matches anything.
  if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
    // Constants are handled elsewhere.  For Functions, the type here is the
    // type of the result, which is not what we want to look at; leave them
    // alone.
    Value *v = OpInfo.CallOperandVal;
    if (isa<ConstantInt>(v) || isa<Function>(v)) {
      return;
    }

    if (isa<BasicBlock>(v) || isa<BlockAddress>(v)) {
      OpInfo.ConstraintCode = "i";
      return;
    }

    // Otherwise, try to resolve it to something we know about by looking at
    // the actual operand type.
    if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
      OpInfo.ConstraintCode = Repl;
      OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
    }
  }
}

/// Given an exact SDIV by a constant, create a multiplication
/// with the multiplicative inverse of the constant.
/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
                              const SDLoc &dl, SelectionDAG &DAG,
                              SmallVectorImpl<SDNode *> &Created) {
  SDValue Op0 = N->getOperand(0);
  SDValue Op1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();

  bool UseSRA = false;
  SmallVector<SDValue, 16> Shifts, Factors;

  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
    if (C->isZero())
      return false;

    EVT CT = C->getValueType(0);
    APInt Divisor = C->getAPIntValue();
    unsigned Shift = Divisor.countr_zero();
    if (Shift) {
      Divisor.ashrInPlace(Shift);
      UseSRA = true;
    }
    APInt Factor = Divisor.multiplicativeInverse();
    Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
    Factors.push_back(DAG.getConstant(Factor, dl, CT));
    return true;
  };

  // Collect all magic values from the build vector.
  if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
    return SDValue();

  SDValue Shift, Factor;
  if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
    Shift = DAG.getBuildVector(ShVT, dl, Shifts);
    Factor = DAG.getBuildVector(VT, dl, Factors);
  } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(Shifts.size() == 1 && Factors.size() == 1 &&
           "Expected matchUnaryPredicate to return one element for scalable "
           "vectors");
    Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
    Factor = DAG.getSplatVector(VT, dl, Factors[0]);
  } else {
    assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
    Shift = Shifts[0];
    Factor = Factors[0];
  }

  SDValue Res = Op0;
  if (UseSRA) {
    Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, SDNodeFlags::Exact);
    Created.push_back(Res.getNode());
  }

  return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
}

/// Given an exact UDIV by a constant, create a multiplication
/// with the multiplicative inverse of the constant.
/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
                              const SDLoc &dl, SelectionDAG &DAG,
                              SmallVectorImpl<SDNode *> &Created) {
  EVT VT = N->getValueType(0);
  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();

  bool UseSRL = false;
  SmallVector<SDValue, 16> Shifts, Factors;

  auto BuildUDIVPattern = [&](ConstantSDNode *C) {
    if (C->isZero())
      return false;

    EVT CT = C->getValueType(0);
    APInt Divisor = C->getAPIntValue();
    unsigned Shift = Divisor.countr_zero();
    if (Shift) {
      Divisor.lshrInPlace(Shift);
      UseSRL = true;
    }
    // Calculate the multiplicative inverse modulo BW.
    APInt Factor = Divisor.multiplicativeInverse();
    Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
    Factors.push_back(DAG.getConstant(Factor, dl, CT));
    return true;
  };

  SDValue Op1 = N->getOperand(1);

  // Collect all magic values from the build vector.
  if (!ISD::matchUnaryPredicate(Op1, BuildUDIVPattern))
    return SDValue();

  SDValue Shift, Factor;
  if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
    Shift = DAG.getBuildVector(ShVT, dl, Shifts);
    Factor = DAG.getBuildVector(VT, dl, Factors);
  } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(Shifts.size() == 1 && Factors.size() == 1 &&
           "Expected matchUnaryPredicate to return one element for scalable "
           "vectors");
    Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
    Factor = DAG.getSplatVector(VT, dl, Factors[0]);
  } else {
    assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
    Shift = Shifts[0];
    Factor = Factors[0];
  }

  SDValue Res = N->getOperand(0);
  if (UseSRL) {
    Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, SDNodeFlags::Exact);
    Created.push_back(Res.getNode());
  }

  return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
}

SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
                              SelectionDAG &DAG,
                              SmallVectorImpl<SDNode *> &Created) const {
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (isIntDivCheap(N->getValueType(0), Attr))
    return SDValue(N, 0); // Lower SDIV as SDIV
  return SDValue();
}

SDValue
TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
                              SelectionDAG &DAG,
                              SmallVectorImpl<SDNode *> &Created) const {
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (isIntDivCheap(N->getValueType(0), Attr))
    return SDValue(N, 0); // Lower SREM as SREM
  return SDValue();
}

/// Build sdiv by power-of-2 with conditional move instructions
/// Ref: "Hacker's Delight" by Henry Warren 10-1
/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
///   bgez x, label
///   add x, x, 2**k-1
/// label:
///   sra res, x, k
///   neg res, res (when the divisor is negative)
SDValue TargetLowering::buildSDIVPow2WithCMov(
    SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
    SmallVectorImpl<SDNode *> &Created) const {
  unsigned Lg2 = Divisor.countr_zero();
  EVT VT = N->getValueType(0);

  SDLoc DL(N);
  SDValue N0 = N->getOperand(0);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2);
  SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);

  // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue Cmp = DAG.getSetCC(DL, CCVT, N0, Zero, ISD::SETLT);
  SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
  SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);

  Created.push_back(Cmp.getNode());
  Created.push_back(Add.getNode());
  Created.push_back(CMov.getNode());

  // Divide by pow2.
  SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, CMov,
                            DAG.getShiftAmountConstant(Lg2, VT, DL));

  // If we're dividing by a positive value, we're done.  Otherwise, we must
  // negate the result.
  if (Divisor.isNonNegative())
    return SRA;

  Created.push_back(SRA.getNode());
  return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA);
}

/// Given an ISD::SDIV node expressing a divide by constant,
/// return a DAG expression to select that will generate the same value by
/// multiplying by a magic number.
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
                                  bool IsAfterLegalization,
                                  bool IsAfterLegalTypes,
                                  SmallVectorImpl<SDNode *> &Created) const {
  SDLoc dl(N);

  // If the sdiv has an 'exact' bit we can use a simpler lowering.
  if (N->getFlags().hasExact())
    return BuildExactSDIV(*this, N, dl, DAG, Created);

  EVT VT = N->getValueType(0);
  EVT SVT = VT.getScalarType();
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();
  unsigned EltBits = VT.getScalarSizeInBits();
  EVT MulVT;

  // Check to see if we can do this.
  // FIXME: We should be more aggressive here.
  EVT QueryVT = VT;
  if (VT.isVector()) {
    // If the vector type will be legalized to a vector type with the same
    // element type, allow the transform before type legalization if MULHS or
    // SMUL_LOHI are supported.
    QueryVT = getLegalTypeToTransformTo(*DAG.getContext(), VT);
    if (!QueryVT.isVector() ||
        QueryVT.getVectorElementType() != VT.getVectorElementType())
      return SDValue();
  } else if (!isTypeLegal(VT)) {
    // Limit this to simple scalars for now.
    if (!VT.isSimple())
      return SDValue();

    // If this type will be promoted to a large enough type with a legal
    // multiply operation, we can go ahead and do this transform.
    if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
      return SDValue();

    MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
    if (MulVT.getSizeInBits() < (2 * EltBits) ||
        !isOperationLegal(ISD::MUL, MulVT))
      return SDValue();
  }

  bool HasMULHS =
      isOperationLegalOrCustom(ISD::MULHS, QueryVT, IsAfterLegalization);
  bool HasSMUL_LOHI =
      isOperationLegalOrCustom(ISD::SMUL_LOHI, QueryVT, IsAfterLegalization);

  if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
    // If type twice as wide legal, widen and use a mul plus a shift.
    EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
    // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
    // custom lowered. This is very expensive so avoid it at all costs for
    // constant divisors.
    if ((!IsAfterLegalTypes && isOperationExpand(ISD::SDIV, VT) &&
         isOperationCustom(ISD::SDIVREM, VT.getScalarType())) ||
        isOperationLegalOrCustom(ISD::MUL, WideVT))
      MulVT = WideVT;
  }

  if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
    return SDValue();

  // If we're after type legalization and SVT is not legal, use the
  // promoted type for creating constants to avoid creating nodes with
  // illegal types.
  if (IsAfterLegalTypes && VT.isVector()) {
    SVT = getTypeToTransformTo(*DAG.getContext(), SVT);
    if (SVT.bitsLT(VT.getScalarType()))
      return SDValue();
    ShSVT = getTypeToTransformTo(*DAG.getContext(), ShSVT);
    if (ShSVT.bitsLT(ShVT.getScalarType()))
      return SDValue();
  }
  const unsigned SVTBits = SVT.getSizeInBits();

  SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

  auto BuildSDIVPattern = [&](ConstantSDNode *C) {
    if (C->isZero())
      return false;
    // Truncate the divisor to the target scalar type in case it was promoted
    // during type legalization.
    APInt Divisor = C->getAPIntValue().trunc(EltBits);
    SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
    int NumeratorFactor = 0;
    int ShiftMask = -1;

    if (Divisor.isOne() || Divisor.isAllOnes()) {
      // If d is +1/-1, we just multiply the numerator by +1/-1.
      NumeratorFactor = Divisor.getSExtValue();
      magics.Magic = 0;
      magics.ShiftAmount = 0;
      ShiftMask = 0;
    } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
      // If d > 0 and m < 0, add the numerator.
      NumeratorFactor = 1;
    } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
      // If d < 0 and m > 0, subtract the numerator.
      NumeratorFactor = -1;
    }

    MagicFactors.push_back(
        DAG.getConstant(magics.Magic.zext(SVTBits), dl, SVT));
    Factors.push_back(DAG.getSignedConstant(NumeratorFactor, dl, SVT));
    Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
    ShiftMasks.push_back(DAG.getSignedConstant(ShiftMask, dl, SVT));
    return true;
  };

  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);

  // Collect the shifts / magic values from each element.
  if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern, /*AllowUndefs=*/false,
                                /*AllowTruncation=*/true))
    return SDValue();

  SDValue MagicFactor, Factor, Shift, ShiftMask;
  if (N1.getOpcode() == ISD::BUILD_VECTOR) {
    MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
    Factor = DAG.getBuildVector(VT, dl, Factors);
    Shift = DAG.getBuildVector(ShVT, dl, Shifts);
    ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
  } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
           Shifts.size() == 1 && ShiftMasks.size() == 1 &&
           "Expected matchUnaryPredicate to return one element for scalable "
           "vectors");
    MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
    Factor = DAG.getSplatVector(VT, dl, Factors[0]);
    Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
    ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
  } else {
    assert(isa<ConstantSDNode>(N1) && "Expected a constant");
    MagicFactor = MagicFactors[0];
    Factor = Factors[0];
    Shift = Shifts[0];
    ShiftMask = ShiftMasks[0];
  }

  // Multiply the numerator (operand 0) by the magic value.
  auto GetMULHS = [&](SDValue X, SDValue Y) {
    if (HasMULHS)
      return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
    if (HasSMUL_LOHI) {
      SDValue LoHi =
          DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
      return LoHi.getValue(1);
    }

    X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
    Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
    Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
    Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
                    DAG.getShiftAmountConstant(EltBits, MulVT, dl));
    return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
  };

  SDValue Q = GetMULHS(N0, MagicFactor);
  if (!Q)
    return SDValue();

  Created.push_back(Q.getNode());

  // (Optionally) Add/subtract the numerator using Factor.
  Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
  Created.push_back(Factor.getNode());
  Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
  Created.push_back(Q.getNode());

  // Shift right algebraic by shift value.
  Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
  Created.push_back(Q.getNode());

  // Extract the sign bit, mask it and add it to the quotient.
  SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
  SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
  Created.push_back(T.getNode());
  T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
  Created.push_back(T.getNode());
  return DAG.getNode(ISD::ADD, dl, VT, Q, T);
}

/// Given an ISD::UDIV node expressing a divide by constant,
/// return a DAG expression to select that will generate the same value by
/// multiplying by a magic number.
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
                                  bool IsAfterLegalization,
                                  bool IsAfterLegalTypes,
                                  SmallVectorImpl<SDNode *> &Created) const {
  SDLoc dl(N);

  // If the udiv has an 'exact' bit we can use a simpler lowering.
  if (N->getFlags().hasExact())
    return BuildExactUDIV(*this, N, dl, DAG, Created);

  EVT VT = N->getValueType(0);
  EVT SVT = VT.getScalarType();
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();
  unsigned EltBits = VT.getScalarSizeInBits();
  EVT MulVT;

  // Check to see if we can do this.
  // FIXME: We should be more aggressive here.
  EVT QueryVT = VT;
  if (VT.isVector()) {
    // If the vector type will be legalized to a vector type with the same
    // element type, allow the transform before type legalization if MULHU or
    // UMUL_LOHI are supported.
    QueryVT = getLegalTypeToTransformTo(*DAG.getContext(), VT);
    if (!QueryVT.isVector() ||
        QueryVT.getVectorElementType() != VT.getVectorElementType())
      return SDValue();
  } else if (!isTypeLegal(VT)) {
    // Limit this to simple scalars for now.
    if (!VT.isSimple())
      return SDValue();

    // If this type will be promoted to a large enough type with a legal
    // multiply operation, we can go ahead and do this transform.
    if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
      return SDValue();

    MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
    if (MulVT.getSizeInBits() < (2 * EltBits) ||
        !isOperationLegal(ISD::MUL, MulVT))
      return SDValue();
  }

  bool HasMULHU =
      isOperationLegalOrCustom(ISD::MULHU, QueryVT, IsAfterLegalization);
  bool HasUMUL_LOHI =
      isOperationLegalOrCustom(ISD::UMUL_LOHI, QueryVT, IsAfterLegalization);

  if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
    // If type twice as wide legal, widen and use a mul plus a shift.
    EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
    // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
    // custom lowered. This is very expensive so avoid it at all costs for
    // constant divisors.
    if ((!IsAfterLegalTypes && isOperationExpand(ISD::UDIV, VT) &&
         isOperationCustom(ISD::UDIVREM, VT.getScalarType())) ||
        isOperationLegalOrCustom(ISD::MUL, WideVT))
      MulVT = WideVT;
  }

  if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
    return SDValue();

  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);

  // Try to use leading zeros of the dividend to reduce the multiplier and
  // avoid expensive fixups.
  unsigned KnownLeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();

  // If we're after type legalization and SVT is not legal, use the
  // promoted type for creating constants to avoid creating nodes with
  // illegal types.
  if (IsAfterLegalTypes && VT.isVector()) {
    SVT = getTypeToTransformTo(*DAG.getContext(), SVT);
    if (SVT.bitsLT(VT.getScalarType()))
      return SDValue();
    ShSVT = getTypeToTransformTo(*DAG.getContext(), ShSVT);
    if (ShSVT.bitsLT(ShVT.getScalarType()))
      return SDValue();
  }
  const unsigned SVTBits = SVT.getSizeInBits();

  // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
  // UMUL_LOHI is supported.
  const EVT WideSVT = MVT::i64;
  const bool HasWideMULHU =
      VT == MVT::i32 &&
      isOperationLegalOrCustom(ISD::MULHU, WideSVT, IsAfterLegalization);
  const bool HasWideUMUL_LOHI =
      VT == MVT::i32 &&
      isOperationLegalOrCustom(ISD::UMUL_LOHI, WideSVT, IsAfterLegalization);
  const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);

  bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
  bool UseWiden = false;
  SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

  auto BuildUDIVPattern = [&](ConstantSDNode *C) {
    if (C->isZero())
      return false;
    // Truncate the divisor to the target scalar type in case it was promoted
    // during type legalization.
    APInt Divisor = C->getAPIntValue().trunc(EltBits);

    SDValue PreShift, MagicFactor, NPQFactor, PostShift;

    // Magic algorithm doesn't work for division by 1. We need to emit a select
    // at the end.
    if (Divisor.isOne()) {
      PreShift = PostShift = DAG.getUNDEF(ShSVT);
      MagicFactor = NPQFactor = DAG.getUNDEF(SVT);
    } else {
      UnsignedDivisionByConstantInfo magics =
          UnsignedDivisionByConstantInfo::get(
              Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()),
              /*AllowEvenDivisorOptimization=*/true,
              /*AllowWidenOptimization=*/AllowWiden);

      if (magics.Widen) {
        UseWiden = true;
        MagicFactor = DAG.getConstant(magics.Magic, dl, WideSVT);
      } else {
        MagicFactor = DAG.getConstant(magics.Magic.zext(SVTBits), dl, SVT);
      }

      assert(magics.PreShift < Divisor.getBitWidth() &&
             "We shouldn't generate an undefined shift!");
      assert(magics.PostShift < Divisor.getBitWidth() &&
             "We shouldn't generate an undefined shift!");
      assert((!magics.IsAdd || magics.PreShift == 0) &&
             "Unexpected pre-shift");
      PreShift = DAG.getConstant(magics.PreShift, dl, ShSVT);
      PostShift = DAG.getConstant(magics.PostShift, dl, ShSVT);
      NPQFactor = DAG.getConstant(
          magics.IsAdd ? APInt::getOneBitSet(SVTBits, EltBits - 1)
                       : APInt::getZero(SVTBits),
          dl, SVT);
      UseNPQ |= magics.IsAdd;
      UsePreShift |= magics.PreShift != 0;
      UsePostShift |= magics.PostShift != 0;
    }

    PreShifts.push_back(PreShift);
    MagicFactors.push_back(MagicFactor);
    NPQFactors.push_back(NPQFactor);
    PostShifts.push_back(PostShift);
    return true;
  };

  // Collect the shifts/magic values from each element.
  if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern, /*AllowUndefs=*/false,
                                /*AllowTruncation=*/true))
    return SDValue();

  SDValue PreShift, PostShift, MagicFactor, NPQFactor;
  if (N1.getOpcode() == ISD::BUILD_VECTOR) {
    PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
    MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
    NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
    PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
  } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
           NPQFactors.size() == 1 && PostShifts.size() == 1 &&
           "Expected matchUnaryPredicate to return one for scalable vectors");
    PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
    MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
    NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
    PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
  } else {
    assert(isa<ConstantSDNode>(N1) && "Expected a constant");
    PreShift = PreShifts[0];
    MagicFactor = MagicFactors[0];
    PostShift = PostShifts[0];
  }

  if (UseWiden) {
    // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
    SDValue WideN0 = DAG.getNode(ISD::ZERO_EXTEND, dl, WideSVT, N0);

    // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
    // WideSVT bits
    SDValue High;
    if (HasWideMULHU) {
      High = DAG.getNode(ISD::MULHU, dl, WideSVT, WideN0, MagicFactor);
    } else {
      assert(HasWideUMUL_LOHI);
      SDValue LoHi =
          DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(WideSVT, WideSVT),
                      WideN0, MagicFactor);
      High = LoHi.getValue(1);
    }

    Created.push_back(High.getNode());
    return DAG.getNode(ISD::TRUNCATE, dl, VT, High);
  }

  SDValue Q = N0;
  if (UsePreShift) {
    Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
    Created.push_back(Q.getNode());
  }

  auto GetMULHU = [&](SDValue X, SDValue Y) {
    if (HasMULHU)
      return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
    if (HasUMUL_LOHI) {
      SDValue LoHi =
          DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
      return LoHi.getValue(1);
    }

    X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
    Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
    Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
    Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
                    DAG.getShiftAmountConstant(EltBits, MulVT, dl));
    return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
  };

  // Multiply the numerator (operand 0) by the magic value.
  Q = GetMULHU(Q, MagicFactor);
  if (!Q)
    return SDValue();

  Created.push_back(Q.getNode());

  if (UseNPQ) {
    SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
    Created.push_back(NPQ.getNode());

    // For vectors we might have a mix of non-NPQ/NPQ paths, so use
    // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
    if (VT.isVector())
      NPQ = GetMULHU(NPQ, NPQFactor);
    else
      NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

    Created.push_back(NPQ.getNode());

    Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
    Created.push_back(Q.getNode());
  }

  if (UsePostShift) {
    Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
    Created.push_back(Q.getNode());
  }

  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

  SDValue One = DAG.getConstant(1, dl, VT);
  SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
  return DAG.getSelect(dl, VT, IsOne, N0, Q);
}

/// If all values in Values that *don't* match the predicate are same 'splat'
/// value, then replace all values with that splat value.
/// Else, if AlternativeReplacement was provided, then replace all values that
/// do match predicate with AlternativeReplacement value.
static void
turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
                          std::function<bool(SDValue)> Predicate,
                          SDValue AlternativeReplacement = SDValue()) {
  SDValue Replacement;
  // Is there a value for which the Predicate does *NOT* match? What is it?
  auto SplatValue = llvm::find_if_not(Values, Predicate);
  if (SplatValue != Values.end()) {
    // Does Values consist only of SplatValue's and values matching Predicate?
    if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
          return Value == *SplatValue || Predicate(Value);
        })) // Then we shall replace values matching predicate with SplatValue.
      Replacement = *SplatValue;
  }
  if (!Replacement) {
    // Oops, we did not find the "baseline" splat value.
    if (!AlternativeReplacement)
      return; // Nothing to do.
    // Let's replace with provided value then.
    Replacement = AlternativeReplacement;
  }
  std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
}

/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
/// where the divisor and comparison target are constants,
/// return a DAG expression that will generate the same comparison result
/// using only multiplications, additions and shifts/rotations.
/// Ref: "Hacker's Delight" 10-17.
SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                        SDValue CompTargetNode,
                                        ISD::CondCode Cond,
                                        DAGCombinerInfo &DCI,
                                        const SDLoc &DL) const {
  SmallVector<SDNode *, 5> Built;
  if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                         DCI, DL, Built)) {
    for (SDNode *N : Built)
      DCI.AddToWorklist(N);
    return Folded;
  }

  return SDValue();
}

SDValue
TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                  SDValue CompTargetNode, ISD::CondCode Cond,
                                  DAGCombinerInfo &DCI, const SDLoc &DL,
                                  SmallVectorImpl<SDNode *> &Created) const {
  // fold (seteq/ne (urem N, D), C) ->
  //      (setule/ugt (rotr (mul (sub N, C), P), K), Q)
  // - D must be constant, with D = D0 * 2^K where D0 is odd
  // - P is the multiplicative inverse of D0 modulo 2^W
  // - Q = floor(((2^W) - 1) / D)
  // where W is the width of the common type of N and D.
  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
         "Only applicable for (in)equality comparisons.");

  SelectionDAG &DAG = DCI.DAG;

  EVT VT = REMNode.getValueType();
  EVT SVT = VT.getScalarType();
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();

  // If MUL is unavailable, we cannot proceed in any case.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
    return SDValue();

  bool ComparingWithAllZeros = true;
  bool AllComparisonsWithNonZerosAreTautological = true;
  bool HadTautologicalLanes = false;
  bool AllLanesAreTautological = true;
  bool HadEvenDivisor = false;
  bool AllDivisorsArePowerOfTwo = true;
  bool HadTautologicalInvertedLanes = false;
  SmallVector<SDValue, 16> PAmts, KAmts, QAmts;

  auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
    // Division by 0 is UB. Leave it to be constant-folded elsewhere.
    if (CDiv->isZero())
      return false;

    const APInt &D = CDiv->getAPIntValue();
    const APInt &Cmp = CCmp->getAPIntValue();

    ComparingWithAllZeros &= Cmp.isZero();

    // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
    // if C2 is not less than C1, the comparison is always false.
    // But we will only be able to produce the comparison that will give the
    // opposive tautological answer. So this lane would need to be fixed up.
    bool TautologicalInvertedLane = D.ule(Cmp);
    HadTautologicalInvertedLanes |= TautologicalInvertedLane;

    // If all lanes are tautological (either all divisors are ones, or divisor
    // is not greater than the constant we are comparing with),
    // we will prefer to avoid the fold.
    bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
    HadTautologicalLanes |= TautologicalLane;
    AllLanesAreTautological &= TautologicalLane;

    // If we are comparing with non-zero, we need'll need to subtract said
    // comparison value from the LHS. But there is no point in doing that if
    // every lane where we are comparing with non-zero is tautological..
    if (!Cmp.isZero())
      AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

    // Decompose D into D0 * 2^K
    unsigned K = D.countr_zero();
    assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
    APInt D0 = D.lshr(K);

    // D is even if it has trailing zeros.
    HadEvenDivisor |= (K != 0);
    // D is a power-of-two if D0 is one.
    // If all divisors are power-of-two, we will prefer to avoid the fold.
    AllDivisorsArePowerOfTwo &= D0.isOne();

    // P = inv(D0, 2^W)
    // 2^W requires W + 1 bits, so we have to extend and then truncate.
    unsigned W = D.getBitWidth();
    APInt P = D0.multiplicativeInverse();
    assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");

    // Q = floor((2^W - 1) u/ D)
    // R = ((2^W - 1) u% D)
    APInt Q, R;
    APInt::udivrem(APInt::getAllOnes(W), D, Q, R);

    // If we are comparing with zero, then that comparison constant is okay,
    // else it may need to be one less than that.
    if (Cmp.ugt(R))
      Q -= 1;

    assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
           "We are expecting that K is always less than all-ones for ShSVT");

    // If the lane is tautological the result can be constant-folded.
    if (TautologicalLane) {
      // Set P and K amount to a bogus values so we can try to splat them.
      P = 0;
      KAmts.push_back(DAG.getAllOnesConstant(DL, ShSVT));
      // And ensure that comparison constant is tautological,
      // it will always compare true/false.
      Q.setAllBits();
    } else {
      KAmts.push_back(DAG.getConstant(K, DL, ShSVT));
    }

    PAmts.push_back(DAG.getConstant(P, DL, SVT));
    QAmts.push_back(DAG.getConstant(Q, DL, SVT));
    return true;
  };

  SDValue N = REMNode.getOperand(0);
  SDValue D = REMNode.getOperand(1);

  // Collect the values from each element.
  if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
    return SDValue();

  // If all lanes are tautological, the result can be constant-folded.
  if (AllLanesAreTautological)
    return SDValue();

  // If this is a urem by a powers-of-two, avoid the fold since it can be
  // best implemented as a bit test.
  if (AllDivisorsArePowerOfTwo)
    return SDValue();

  SDValue PVal, KVal, QVal;
  if (D.getOpcode() == ISD::BUILD_VECTOR) {
    if (HadTautologicalLanes) {
      // Try to turn PAmts into a splat, since we don't care about the values
      // that are currently '0'. If we can't, just keep '0'`s.
      turnVectorIntoSplatVector(PAmts, isNullConstant);
      // Try to turn KAmts into a splat, since we don't care about the values
      // that are currently '-1'. If we can't, change them to '0'`s.
      turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                                DAG.getConstant(0, DL, ShSVT));
    }

    PVal = DAG.getBuildVector(VT, DL, PAmts);
    KVal = DAG.getBuildVector(ShVT, DL, KAmts);
    QVal = DAG.getBuildVector(VT, DL, QAmts);
  } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
           "Expected matchBinaryPredicate to return one element for "
           "SPLAT_VECTORs");
    PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
    KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
    QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
  } else {
    PVal = PAmts[0];
    KVal = KAmts[0];
    QVal = QAmts[0];
  }

  if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
    if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
      return SDValue(); // FIXME: Could/should use `ISD::ADD`?
    assert(CompTargetNode.getValueType() == N.getValueType() &&
           "Expecting that the types on LHS and RHS of comparisons match.");
    N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
  }

  // (mul N, P)
  SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
  Created.push_back(Op0.getNode());

  // Rotate right only if any divisor was even. We avoid rotates for all-odd
  // divisors as a performance improvement, since rotating by 0 is a no-op.
  if (HadEvenDivisor) {
    // We need ROTR to do this.
    if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
      return SDValue();
    // UREM: (rotr (mul N, P), K)
    Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
    Created.push_back(Op0.getNode());
  }

  // UREM: (setule/setugt (rotr (mul N, P), K), Q)
  SDValue NewCC =
      DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                   ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
  if (!HadTautologicalInvertedLanes)
    return NewCC;

  // If any lanes previously compared always-false, the NewCC will give
  // always-true result for them, so we need to fixup those lanes.
  // Or the other way around for inequality predicate.
  assert(VT.isVector() && "Can/should only get here for vectors.");
  Created.push_back(NewCC.getNode());

  // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
  // if C2 is not less than C1, the comparison is always false.
  // But we have produced the comparison that will give the
  // opposive tautological answer. So these lanes would need to be fixed up.
  SDValue TautologicalInvertedChannels =
      DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
  Created.push_back(TautologicalInvertedChannels.getNode());

  // NOTE: we avoid letting illegal types through even if we're before legalize
  // ops – legalization has a hard time producing good code for this.
  if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
    // If we have a vector select, let's replace the comparison results in the
    // affected lanes with the correct tautological result.
    SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
                                              DL, SETCCVT, SETCCVT);
    return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
                       Replacement, NewCC);
  }

  // Else, we can just invert the comparison result in the appropriate lanes.
  //
  // NOTE: see the note above VSELECT above.
  if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
    return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
                       TautologicalInvertedChannels);

  return SDValue(); // Don't know how to lower.
}

/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
/// where the divisor is constant and the comparison target is zero,
/// return a DAG expression that will generate the same comparison result
/// using only multiplications, additions and shifts/rotations.
/// Ref: "Hacker's Delight" 10-17.
SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                        SDValue CompTargetNode,
                                        ISD::CondCode Cond,
                                        DAGCombinerInfo &DCI,
                                        const SDLoc &DL) const {
  SmallVector<SDNode *, 7> Built;
  if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                         DCI, DL, Built)) {
    assert(Built.size() <= 7 && "Max size prediction failed.");
    for (SDNode *N : Built)
      DCI.AddToWorklist(N);
    return Folded;
  }

  return SDValue();
}

SDValue
TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                  SDValue CompTargetNode, ISD::CondCode Cond,
                                  DAGCombinerInfo &DCI, const SDLoc &DL,
                                  SmallVectorImpl<SDNode *> &Created) const {
  // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
  // Fold:
  //   (seteq/ne (srem N, D), 0)
  // To:
  //   (setule/ugt (rotr (add (mul N, P), A), K), Q)
  //
  // - D must be constant, with D = D0 * 2^K where D0 is odd
  // - P is the multiplicative inverse of D0 modulo 2^W
  // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
  // - Q = floor((2 * A) / (2^K))
  // where W is the width of the common type of N and D.
  //
  // When D is a power of two (and thus D0 is 1), the normal
  // formula for A and Q don't apply, because the derivation
  // depends on D not dividing 2^(W-1), and thus theorem ZRS
  // does not apply. This specifically fails when N = INT_MIN.
  //
  // Instead, for power-of-two D, we use:
  // - A = 0
  // | -> No offset needed. We're effectively treating it the same as urem.
  // - Q = 2^(W-K) - 1
  // |-> Test that the top K bits are zero after rotation
  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
         "Only applicable for (in)equality comparisons.");

  SelectionDAG &DAG = DCI.DAG;

  EVT VT = REMNode.getValueType();
  EVT SVT = VT.getScalarType();
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  EVT ShSVT = ShVT.getScalarType();

  // If we are after ops legalization, and MUL is unavailable, we can not
  // proceed.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
    return SDValue();

  // TODO: Could support comparing with non-zero too.
  ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
  if (!CompTarget || !CompTarget->isZero())
    return SDValue();

  bool HadOneDivisor = false;
  bool AllDivisorsAreOnes = true;
  bool HadEvenDivisor = false;
  bool AllDivisorsArePowerOfTwo = true;
  SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

  auto BuildSREMPattern = [&](ConstantSDNode *C) {
    // Division by 0 is UB. Leave it to be constant-folded elsewhere.
    if (C->isZero())
      return false;

    // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

    // WARNING: this fold is only valid for positive divisors!
    // `rem %X, -C` is equivalent to `rem %X, C`
    APInt D = C->getAPIntValue().abs();

    // If all divisors are ones, we will prefer to avoid the fold.
    HadOneDivisor |= D.isOne();
    AllDivisorsAreOnes &= D.isOne();

    // Decompose D into D0 * 2^K
    unsigned K = D.countr_zero();
    assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
    APInt D0 = D.lshr(K);

    // D is even if it has trailing zeros.
    HadEvenDivisor |= (K != 0);

    // D is a power-of-two if D0 is one. This includes INT_MIN.
    // If all divisors are power-of-two, we will prefer to avoid the fold.
    AllDivisorsArePowerOfTwo &= D0.isOne();

    // P = inv(D0, 2^W)
    // 2^W requires W + 1 bits, so we have to extend and then truncate.
    unsigned W = D.getBitWidth();
    APInt P = D0.multiplicativeInverse();
    assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");

    // A = floor((2^(W - 1) - 1) / D0) & -2^K
    APInt A = APInt::getSignedMaxValue(W).udiv(D0);
    A.clearLowBits(K);

    // Q = floor((2 * A) / (2^K))
    APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));

    assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
           "We are expecting that A is always less than all-ones for SVT");
    assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
           "We are expecting that K is always less than all-ones for ShSVT");

    // If D was a power of two, apply the alternate constant derivation.
    if (D0.isOne()) {
      // A = 0
      A = APInt(W, 0);
      // - Q = 2^(W-K) - 1
      Q = APInt::getLowBitsSet(W, W - K);
    }

    // If the divisor is 1 the result can be constant-folded.
    if (D.isOne()) {
      // Set P, A and K to a bogus values so we can try to splat them.
      P = 0;
      A.setAllBits();
      KAmts.push_back(DAG.getAllOnesConstant(DL, ShSVT));

      // x ?% 1 == 0  <-->  true  <-->  x u<= -1
      Q.setAllBits();
    } else {
      KAmts.push_back(DAG.getConstant(K, DL, ShSVT));
    }

    PAmts.push_back(DAG.getConstant(P, DL, SVT));
    AAmts.push_back(DAG.getConstant(A, DL, SVT));
    QAmts.push_back(DAG.getConstant(Q, DL, SVT));
    return true;
  };

  SDValue N = REMNode.getOperand(0);
  SDValue D = REMNode.getOperand(1);

  // Collect the values from each element.
  if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
    return SDValue();

  // If this is a srem by a one, avoid the fold since it can be constant-folded.
  if (AllDivisorsAreOnes)
    return SDValue();

  // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
  // since it can be best implemented as a bit test.
  if (AllDivisorsArePowerOfTwo)
    return SDValue();

  SDValue PVal, AVal, KVal, QVal;
  if (D.getOpcode() == ISD::BUILD_VECTOR) {
    if (HadOneDivisor) {
      // Try to turn PAmts into a splat, since we don't care about the values
      // that are currently '0'. If we can't, just keep '0'`s.
      turnVectorIntoSplatVector(PAmts, isNullConstant);
      // Try to turn AAmts into a splat, since we don't care about the
      // values that are currently '-1'. If we can't, change them to '0'`s.
      turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
                                DAG.getConstant(0, DL, SVT));
      // Try to turn KAmts into a splat, since we don't care about the values
      // that are currently '-1'. If we can't, change them to '0'`s.
      turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                                DAG.getConstant(0, DL, ShSVT));
    }

    PVal = DAG.getBuildVector(VT, DL, PAmts);
    AVal = DAG.getBuildVector(VT, DL, AAmts);
    KVal = DAG.getBuildVector(ShVT, DL, KAmts);
    QVal = DAG.getBuildVector(VT, DL, QAmts);
  } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
    assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
           QAmts.size() == 1 &&
           "Expected matchUnaryPredicate to return one element for scalable "
           "vectors");
    PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
    AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
    KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
    QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
  } else {
    assert(isa<ConstantSDNode>(D) && "Expected a constant");
    PVal = PAmts[0];
    AVal = AAmts[0];
    KVal = KAmts[0];
    QVal = QAmts[0];
  }

  // (mul N, P)
  SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
  Created.push_back(Op0.getNode());

  // We need ADD to do this.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
    return SDValue();

  // (add (mul N, P), A)
  Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
  Created.push_back(Op0.getNode());

  // Rotate right only if any divisor was even. We avoid rotates for all-odd
  // divisors as a performance improvement, since rotating by 0 is a no-op.
  if (HadEvenDivisor) {
    // We need ROTR to do this.
    if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
      return SDValue();
    // SREM: (rotr (add (mul N, P), A), K)
    Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
    Created.push_back(Op0.getNode());
  }

  // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
  return DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                      (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
}

SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
                                         const DenormalMode &Mode,
                                         SDNodeFlags Flags) const {
  SDLoc DL(Op);
  EVT VT = Op.getValueType();
  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);

  // This is specifically a check for the handling of denormal inputs, not the
  // result.
  if (Mode.Input == DenormalMode::PreserveSign ||
      Mode.Input == DenormalMode::PositiveZero) {
    // Test = X == 0.0
    return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ, /*Chain=*/{},
                        /*Signaling=*/false, Flags);
  }

  // Testing it with denormal inputs to avoid wrong estimate.
  //
  // Test = fabs(X) < SmallestNormal
  const fltSemantics &FltSem = VT.getFltSemantics();
  APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
  SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
  SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op, Flags);
  return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT, /*Chain=*/{},
                      /*Signaling=*/false, Flags);
}

SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
                                             bool LegalOps, bool OptForSize,
                                             NegatibleCost &Cost,
                                             unsigned Depth) const {
  // fneg is removable even if it has multiple uses.
  if (Op.getOpcode() == ISD::FNEG) {
    Cost = NegatibleCost::Cheaper;
    return Op.getOperand(0);
  }

  // Don't recurse exponentially.
  if (Depth > SelectionDAG::MaxRecursionDepth)
    return SDValue();

  // Pre-increment recursion depth for use in recursive calls.
  ++Depth;
  const SDNodeFlags Flags = Op->getFlags();
  EVT VT = Op.getValueType();
  unsigned Opcode = Op.getOpcode();

  // Don't allow anything with multiple uses unless we know it is free.
  if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
    bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
                        isFPExtFree(VT, Op.getOperand(0).getValueType());
    if (!IsFreeExtend)
      return SDValue();
  }

  auto RemoveDeadNode = [&](SDValue N) {
    if (N && N.getNode()->use_empty())
      DAG.RemoveDeadNode(N.getNode());
  };

  SDLoc DL(Op);

  // Because getNegatedExpression can delete nodes we need a handle to keep
  // temporary nodes alive in case the recursion manages to create an identical
  // node.
  std::list<HandleSDNode> Handles;

  switch (Opcode) {
  case ISD::ConstantFP: {
    // Don't invert constant FP values after legalization unless the target says
    // the negated constant is legal.
    bool IsOpLegal =
        isOperationLegal(ISD::ConstantFP, VT) ||
        isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
                     OptForSize);

    if (LegalOps && !IsOpLegal)
      break;

    APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
    V.changeSign();
    SDValue CFP = DAG.getConstantFP(V, DL, VT);

    // If we already have the use of the negated floating constant, it is free
    // to negate it even it has multiple uses.
    if (!Op.hasOneUse() && CFP.use_empty())
      break;
    Cost = NegatibleCost::Neutral;
    return CFP;
  }
  case ISD::SPLAT_VECTOR: {
    // fold splat_vector(fneg(X)) -> splat_vector(-X)
    SDValue X = Op.getOperand(0);
    if (!isOperationLegal(ISD::SPLAT_VECTOR, VT))
      break;

    SDValue NegX = getCheaperNegatedExpression(X, DAG, LegalOps, OptForSize);
    if (!NegX)
      break;
    Cost = NegatibleCost::Cheaper;
    return DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, NegX);
  }
  case ISD::BUILD_VECTOR: {
    // Only permit BUILD_VECTOR of constants.
    if (llvm::any_of(Op->op_values(), [&](SDValue N) {
          return !N.isUndef() && !isa<ConstantFPSDNode>(N);
        }))
      break;

    bool IsOpLegal =
        (isOperationLegal(ISD::ConstantFP, VT) &&
         isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
        llvm::all_of(Op->op_values(), [&](SDValue N) {
          return N.isUndef() ||
                 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
                              OptForSize);
        });

    if (LegalOps && !IsOpLegal)
      break;

    SmallVector<SDValue, 4> Ops;
    for (SDValue C : Op->op_values()) {
      if (C.isUndef()) {
        Ops.push_back(C);
        continue;
      }
      APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
      V.changeSign();
      Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
    }
    Cost = NegatibleCost::Neutral;
    return DAG.getBuildVector(VT, DL, Ops);
  }
  case ISD::FADD: {
    if (!Flags.hasNoSignedZeros())
      break;

    // After operation legalization, it might not be legal to create new FSUBs.
    if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
      break;
    SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

    // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
    NegatibleCost CostX = NegatibleCost::Expensive;
    SDValue NegX =
        getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
    // Prevent this node from being deleted by the next call.
    if (NegX)
      Handles.emplace_back(NegX);

    // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
    NegatibleCost CostY = NegatibleCost::Expensive;
    SDValue NegY =
        getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

    // We're done with the handles.
    Handles.clear();

    // Negate the X if its cost is less or equal than Y.
    if (NegX && (CostX <= CostY)) {
      Cost = CostX;
      SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
      if (NegY != N)
        RemoveDeadNode(NegY);
      return N;
    }

    // Negate the Y if it is not expensive.
    if (NegY) {
      Cost = CostY;
      SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
      if (NegX != N)
        RemoveDeadNode(NegX);
      return N;
    }
    break;
  }
  case ISD::FSUB: {
    // We can't turn -(A-B) into B-A when we honor signed zeros.
    if (!Flags.hasNoSignedZeros())
      break;

    SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
    // fold (fneg (fsub 0, Y)) -> Y
    if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
      if (C->isZero()) {
        Cost = NegatibleCost::Cheaper;
        return Y;
      }

    // fold (fneg (fsub X, Y)) -> (fsub Y, X)
    Cost = NegatibleCost::Neutral;
    return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
  }
  case ISD::FMUL:
  case ISD::FDIV: {
    SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
    NegatibleCost CostX = NegatibleCost::Expensive;
    SDValue NegX =
        getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
    // Prevent this node from being deleted by the next call.
    if (NegX)
      Handles.emplace_back(NegX);

    // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
    NegatibleCost CostY = NegatibleCost::Expensive;
    SDValue NegY =
        getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

    // We're done with the handles.
    Handles.clear();

    // Negate the X if its cost is less or equal than Y.
    if (NegX && (CostX <= CostY)) {
      Cost = CostX;
      SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
      if (NegY != N)
        RemoveDeadNode(NegY);
      return N;
    }

    // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
    if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
      if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
        break;

    // Negate the Y if it is not expensive.
    if (NegY) {
      Cost = CostY;
      SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
      if (NegX != N)
        RemoveDeadNode(NegX);
      return N;
    }
    break;
  }
  case ISD::FMA:
  case ISD::FMULADD:
  case ISD::FMAD: {
    if (!Flags.hasNoSignedZeros())
      break;

    SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
    NegatibleCost CostZ = NegatibleCost::Expensive;
    SDValue NegZ =
        getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
    // Give up if fail to negate the Z.
    if (!NegZ)
      break;

    // Prevent this node from being deleted by the next two calls.
    Handles.emplace_back(NegZ);

    // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
    NegatibleCost CostX = NegatibleCost::Expensive;
    SDValue NegX =
        getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
    // Prevent this node from being deleted by the next call.
    if (NegX)
      Handles.emplace_back(NegX);

    // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
    NegatibleCost CostY = NegatibleCost::Expensive;
    SDValue NegY =
        getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

    // We're done with the handles.
    Handles.clear();

    // Negate the X if its cost is less or equal than Y.
    if (NegX && (CostX <= CostY)) {
      Cost = std::min(CostX, CostZ);
      SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
      if (NegY != N)
        RemoveDeadNode(NegY);
      return N;
    }

    // Negate the Y if it is not expensive.
    if (NegY) {
      Cost = std::min(CostY, CostZ);
      SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
      if (NegX != N)
        RemoveDeadNode(NegX);
      return N;
    }
    break;
  }

  case ISD::FP_EXTEND:
  case ISD::FSIN:
    if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
                                            OptForSize, Cost, Depth))
      return DAG.getNode(Opcode, DL, VT, NegV);
    break;
  case ISD::FP_ROUND:
    if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
                                            OptForSize, Cost, Depth))
      return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
    break;
  case ISD::SELECT:
  case ISD::VSELECT: {
    // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
    // iff at least one cost is cheaper and the other is neutral/cheaper
    SDValue LHS = Op.getOperand(1);
    NegatibleCost CostLHS = NegatibleCost::Expensive;
    SDValue NegLHS =
        getNegatedExpression(LHS, DAG, LegalOps, OptForSize, CostLHS, Depth);
    if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
      RemoveDeadNode(NegLHS);
      break;
    }

    // Prevent this node from being deleted by the next call.
    Handles.emplace_back(NegLHS);

    SDValue RHS = Op.getOperand(2);
    NegatibleCost CostRHS = NegatibleCost::Expensive;
    SDValue NegRHS =
        getNegatedExpression(RHS, DAG, LegalOps, OptForSize, CostRHS, Depth);

    // We're done with the handles.
    Handles.clear();

    if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
        (CostLHS != NegatibleCost::Cheaper &&
         CostRHS != NegatibleCost::Cheaper)) {
      RemoveDeadNode(NegLHS);
      RemoveDeadNode(NegRHS);
      break;
    }

    Cost = std::min(CostLHS, CostRHS);
    return DAG.getSelect(DL, VT, Op.getOperand(0), NegLHS, NegRHS);
  }
  }

  return SDValue();
}

//===----------------------------------------------------------------------===//
// Legalization Utilities
//===----------------------------------------------------------------------===//

bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
                                    SDValue LHS, SDValue RHS,
                                    SmallVectorImpl<SDValue> &Result,
                                    EVT HiLoVT, SelectionDAG &DAG,
                                    MulExpansionKind Kind, SDValue LL,
                                    SDValue LH, SDValue RL, SDValue RH) const {
  assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
         Opcode == ISD::SMUL_LOHI);

  bool HasMULHS = (Kind == MulExpansionKind::Always) ||
                  isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
  bool HasMULHU = (Kind == MulExpansionKind::Always) ||
                  isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
  bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                      isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
  bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                      isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);

  if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
    return false;

  unsigned OuterBitSize = VT.getScalarSizeInBits();
  unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();

  // LL, LH, RL, and RH must be either all NULL or all set to a value.
  assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
         (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));

  auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
                          bool Signed) -> bool {
    if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
      SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
      Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
      Hi = Lo.getValue(1);
      return true;
    }
    if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
      Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
      Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
      return true;
    }
    return false;
  };

  SDValue Lo, Hi;

  if (!LL.getNode() && !RL.getNode() &&
      isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
    LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
    RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
  }

  if (!LL.getNode())
    return false;

  APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
  if (DAG.MaskedValueIsZero(LHS, HighMask) &&
      DAG.MaskedValueIsZero(RHS, HighMask)) {
    // The inputs are both zero-extended.
    if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
      Result.push_back(Lo);
      Result.push_back(Hi);
      if (Opcode != ISD::MUL) {
        SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
        Result.push_back(Zero);
        Result.push_back(Zero);
      }
      return true;
    }
  }

  if (!VT.isVector() && Opcode == ISD::MUL &&
      DAG.ComputeMaxSignificantBits(LHS) <= InnerBitSize &&
      DAG.ComputeMaxSignificantBits(RHS) <= InnerBitSize) {
    // The input values are both sign-extended.
    // TODO non-MUL case?
    if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
      Result.push_back(Lo);
      Result.push_back(Hi);
      return true;
    }
  }

  unsigned ShiftAmount = OuterBitSize - InnerBitSize;
  SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl);

  if (!LH.getNode() && !RH.getNode() &&
      isOperationLegalOrCustom(ISD::SRL, VT) &&
      isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
    LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
    LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
    RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
    RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
  }

  if (!LH.getNode())
    return false;

  if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
    return false;

  Result.push_back(Lo);

  if (Opcode == ISD::MUL) {
    RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
    LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
    Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
    Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
    Result.push_back(Hi);
    return true;
  }

  // Compute the full width result.
  auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
    Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
    Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
    Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
    return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
  };

  SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
  if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
    return false;

  // This is effectively the add part of a multiply-add of half-sized operands,
  // so it cannot overflow.
  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

  if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
    return false;

  SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
  EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

  bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
                  isOperationLegalOrCustom(ISD::ADDE, VT));
  if (UseGlue)
    Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
                       Merge(Lo, Hi));
  else
    Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next,
                       Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));

  SDValue Carry = Next.getValue(1);
  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

  if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
    return false;

  if (UseGlue)
    Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
                     Carry);
  else
    Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
                     Zero, Carry);

  Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

  if (Opcode == ISD::SMUL_LOHI) {
    SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
                                  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
    Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);

    NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
                          DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
    Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
  }

  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
  Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
  Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
  return true;
}

bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
                               SelectionDAG &DAG, MulExpansionKind Kind,
                               SDValue LL, SDValue LH, SDValue RL,
                               SDValue RH) const {
  SmallVector<SDValue, 2> Result;
  bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
                           N->getOperand(0), N->getOperand(1), Result, HiLoVT,
                           DAG, Kind, LL, LH, RL, RH);
  if (Ok) {
    assert(Result.size() == 2);
    Lo = Result[0];
    Hi = Result[1];
  }
  return Ok;
}

// Optimize unsigned division or remainder by constants for types twice as large
// as a legal VT.
//
// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
// can be computed
// as:
//   Sum = __builtin_uadd_overflow(Lo, High, &Sum);
//   Remainder = Sum % Constant;
//
// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
// High:Low into 3 chunks of W bits and compute remainder as
//   Sum = Chunk0 + Chunk1 + Chunk2;
//   Remainder = Sum % Constant;
//
// This is based on "Remainder by Summing Digits" from Hacker's Delight.
//
// For division, we can compute the remainder using the algorithm described
// above, subtract it from the dividend to get an exact multiple of Constant.
// Then multiply that exact multiply by the multiplicative inverse modulo
// (1 << (BitWidth / 2)) to get the quotient.

// If Constant is even, we can shift right the dividend and the divisor by the
// number of trailing zeros in Constant before applying the remainder algorithm.
// If we're after the quotient, we can subtract this value from the shifted
// dividend and multiply by the multiplicative inverse of the shifted divisor.
// If we want the remainder, we shift the value left by the number of trailing
// zeros and add the bits that were shifted out of the dividend.
bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
    SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
    SelectionDAG &DAG, SDValue LL, SDValue LH) const {
  unsigned Opcode = N->getOpcode();
  EVT VT = N->getValueType(0);

  unsigned BitWidth = Divisor.getBitWidth();
  unsigned HBitWidth = BitWidth / 2;
  assert(VT.getScalarSizeInBits() == BitWidth &&
         HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");

  // If the divisor is even, shift it until it becomes odd.
  unsigned TrailingZeros = 0;
  if (!Divisor[0]) {
    TrailingZeros = Divisor.countr_zero();
    Divisor.lshrInPlace(TrailingZeros);
  }

  // After removing trailing zeros, the divisor needs to be less than
  // (1 << HBitWidth).
  APInt HalfMaxPlus1 = APInt::getOneBitSet(BitWidth, HBitWidth);
  if (Divisor.uge(HalfMaxPlus1))
    return false;

  // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
  // (1 << W) % Divisor == -1.
  unsigned BestChunkWidth = 0, AltChunkWidth = 0;
  for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
    // Skip HBitWidth-1, it doesn't have enough bits for carries.
    if (I == HBitWidth - 1)
      continue;

    APInt Mod = APInt::getOneBitSet(Divisor.getBitWidth(), I).urem(Divisor);

    if (Mod.isOne()) {
      BestChunkWidth = I;
      break;
    }

    // We have an alternate strategy for Remainder == Divisor - 1.
    // FIXME: Support HBitWidth.
    if (I != HBitWidth && Mod == Divisor - 1)
      AltChunkWidth = I;
  }

  bool Alternate = false;
  if (!BestChunkWidth) {
    if (!AltChunkWidth)
      return false;
    Alternate = true;
    BestChunkWidth = AltChunkWidth;
  }

  SDLoc dl(N);

  assert(!LL == !LH && "Expected both input halves or no input halves!");
  if (!LL)
    std::tie(LL, LH) = DAG.SplitScalar(N->getOperand(0), dl, HiLoVT, HiLoVT);

  bool HasFSHR = isOperationLegal(ISD::FSHR, HiLoVT);

  auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
    assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
    if (HasFSHR)
      return DAG.getNode(ISD::FSHR, dl, HiLoVT, Hi, Lo,
                         DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl));
    return DAG.getNode(
        ISD::OR, dl, HiLoVT,
        DAG.getNode(ISD::SRL, dl, HiLoVT, Lo,
                    DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl)),
        DAG.getNode(
            ISD::SHL, dl, HiLoVT, Hi,
            DAG.getShiftAmountConstant(HBitWidth - ShiftAmt, HiLoVT, dl)));
  };

  // Helper to perform a right shift on a 128-bit value split into two halves.
  // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
  auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
    if (ShiftAmt == 0)
      return;
    if (ShiftAmt < HBitWidth) {
      Lo = GetFSHR(Lo, Hi, ShiftAmt);
      Hi = DAG.getNode(ISD::SRL, dl, HiLoVT, Hi,
                       DAG.getShiftAmountConstant(ShiftAmt, HiLoVT, dl));
    } else if (ShiftAmt == HBitWidth) {
      Lo = Hi;
      Hi = DAG.getConstant(0, dl, HiLoVT);
    } else {
      Lo = DAG.getNode(
          ISD::SRL, dl, HiLoVT, Hi,
          DAG.getShiftAmountConstant(ShiftAmt - HBitWidth, HiLoVT, dl));
      Hi = DAG.getConstant(0, dl, HiLoVT);
    }
  };

  // Shift the input by the number of TrailingZeros in the divisor. The
  // shifted out bits will be added to the remainder later.
  SDValue PartialRemL, PartialRemH;
  if (TrailingZeros && Opcode != ISD::UDIV) {
    // Save the shifted off bits if we need the remainder.
    if (TrailingZeros < HBitWidth) {
      APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros);
      PartialRemL = DAG.getNode(ISD::AND, dl, HiLoVT, LL,
                                DAG.getConstant(Mask, dl, HiLoVT));
    } else if (TrailingZeros == HBitWidth) {
      // All of LL is part of the remainder.
      PartialRemL = LL;
    } else {
      // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
      PartialRemL = LL;
      APInt Mask = APInt::getLowBitsSet(HBitWidth, TrailingZeros - HBitWidth);
      PartialRemH = DAG.getNode(ISD::AND, dl, HiLoVT, LH,
                                DAG.getConstant(Mask, dl, HiLoVT));
    }
  }

  SDValue Sum;
  // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
  // out, add that to the final sum.
  if (BestChunkWidth == HBitWidth) {
    assert(!Alternate);
    // Shift LH:LL right if there were trailing zeros in the divisor.
    ShiftRight(LL, LH, TrailingZeros);

    // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
    EVT SetCCType =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), HiLoVT);
    if (isOperationLegalOrCustom(ISD::UADDO_CARRY, HiLoVT)) {
      SDVTList VTList = DAG.getVTList(HiLoVT, SetCCType);
      Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH);
      Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum,
                        DAG.getConstant(0, dl, HiLoVT), Sum.getValue(1));
    } else {
      Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH);
      SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT);
      // If the boolean for the target is 0 or 1, we can add the setcc result
      // directly.
      if (getBooleanContents(HiLoVT) ==
          TargetLoweringBase::ZeroOrOneBooleanContent)
        Carry = DAG.getZExtOrTrunc(Carry, dl, HiLoVT);
      else
        Carry = DAG.getSelect(dl, HiLoVT, Carry, DAG.getConstant(1, dl, HiLoVT),
                              DAG.getConstant(0, dl, HiLoVT));
      Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry);
    }
  } else {
    // Otherwise split into multple chunks and add them together. We chose
    // BestChunkWidth so that the sum will not overflow.
    SDValue Mask = DAG.getConstant(
        APInt::getLowBitsSet(HBitWidth, BestChunkWidth), dl, HiLoVT);

    for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
      // If there were trailing zeros in the divisor, increase the shift amount.
      unsigned Shift = I + TrailingZeros;
      SDValue Chunk;
      if (Shift == 0)
        Chunk = LL;
      else if (Shift >= HBitWidth)
        Chunk = DAG.getNode(
            ISD::SRL, dl, HiLoVT, LH,
            DAG.getShiftAmountConstant(Shift - HBitWidth, HiLoVT, dl));
      else
        Chunk = GetFSHR(LL, LH, Shift);
      // If we're on the last chunk, we don't need an AND.
      if (I + BestChunkWidth < BitWidth - TrailingZeros)
        Chunk = DAG.getNode(ISD::AND, dl, HiLoVT, Chunk, Mask);
      if (!Sum) {
        Sum = Chunk;
      } else {
        // For Alternate, we need to subtract odd chunks.
        unsigned ChunkNum = I / BestChunkWidth;
        unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
        Sum = DAG.getNode(Opc, dl, HiLoVT, Sum, Chunk);
      }
    }

    // For Alternate, the sum may be negative, but we need a positive sum. We
    // can increase it by a multiple of the divisor to make it positive. For 3
    // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
    // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
    // is a multiple of the divisor. Add that 1 or 2 times to make the sum
    // positive.
    if (Alternate) {
      unsigned NumChunks = divideCeil(BitWidth - TrailingZeros, BestChunkWidth);
      assert(NumChunks <= 4);

      APInt Adjust = APInt::getOneBitSet(HBitWidth, BestChunkWidth);
      Adjust.setBit(0);
      // If there are 4 chunks, we need to adjust twice.
      if (NumChunks == 4)
        Adjust <<= 1;
      Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum,
                        DAG.getConstant(Adjust, dl, HiLoVT));
    }
  }

  // Perform a HiLoVT urem on the Sum using truncated divisor.
  SDValue RemL =
      DAG.getNode(ISD::UREM, dl, HiLoVT, Sum,
                  DAG.getConstant(Divisor.trunc(HBitWidth), dl, HiLoVT));
  SDValue RemH = DAG.getConstant(0, dl, HiLoVT);

  if (Opcode != ISD::UREM) {
    // If we didn't shift LH/LR earlier, do it now.
    if (BestChunkWidth != HBitWidth)
      ShiftRight(LL, LH, TrailingZeros);

    // Subtract the remainder from the shifted dividend.
    SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH);
    SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH);

    Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem);

    // Multiply by the multiplicative inverse of the divisor modulo
    // (1 << BitWidth).
    APInt MulFactor = Divisor.multiplicativeInverse();

    SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend,
                                   DAG.getConstant(MulFactor, dl, VT));

    // Split the quotient into low and high parts.
    SDValue QuotL, QuotH;
    std::tie(QuotL, QuotH) = DAG.SplitScalar(Quotient, dl, HiLoVT, HiLoVT);
    Result.push_back(QuotL);
    Result.push_back(QuotH);
  }

  if (Opcode != ISD::UDIV) {
    // If we shifted the input, shift the remainder left and add the bits we
    // shifted off the input.
    if (TrailingZeros) {
      if (TrailingZeros < HBitWidth) {
        // Shift RemH:RemL left by TrailingZeros.
        // RemH gets the high bits shifted out of RemL.
        RemH = DAG.getNode(
            ISD::SRL, dl, HiLoVT, RemL,
            DAG.getShiftAmountConstant(HBitWidth - TrailingZeros, HiLoVT, dl));
        RemL =
            DAG.getNode(ISD::SHL, dl, HiLoVT, RemL,
                        DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl));
        // OR in the partial remainder.
        RemL = DAG.getNode(ISD::OR, dl, HiLoVT, RemL, PartialRemL,
                           SDNodeFlags::Disjoint);
      } else if (TrailingZeros == HBitWidth) {
        // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
        // PartialRemL.
        RemH = RemL;
        RemL = PartialRemL;
      } else {
        // Shift left by more than HBitWidth.
        RemH = DAG.getNode(
            ISD::SHL, dl, HiLoVT, RemL,
            DAG.getShiftAmountConstant(TrailingZeros - HBitWidth, HiLoVT, dl));
        RemH = DAG.getNode(ISD::OR, dl, HiLoVT, RemH, PartialRemH,
                           SDNodeFlags::Disjoint);
        RemL = PartialRemL;
      }
    }
    Result.push_back(RemL);
    Result.push_back(RemH);
  }

  return true;
}

bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
    SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
    EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {

  SDValue N0 = N->getOperand(0);
  EVT VT = N0->getValueType(0);
  SDLoc DL{N};

  assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");

  // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
  auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
                                 const APInt &Const,
                                 SmallVectorImpl<SDValue> &Result) {
    SDValue LHS = DAG.getNode(ISD::BUILD_PAIR, DL, VT, LL, LH);
    SDValue RHS = DAG.getConstant(Const, DL, VT);
    auto [RL, RH] = DAG.SplitScalar(RHS, DL, HiLoVT, HiLoVT);
    return expandMUL_LOHI(Opc, VT, DL, LHS, RHS, Result, HiLoVT, DAG,
                          TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
                          LL, LH, RL, RH);
  };

  // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
  auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
                            SDValue RH) {
    SDValue AddSubNode =
        DAG.getNode(Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
                    DAG.getVTList(HiLoVT, MVT::i1), LL, RL);
    SDValue OutL = AddSubNode.getValue(0);
    SDValue Overflow = AddSubNode.getValue(1);
    SDValue AddSubWithOverflow =
        DAG.getNode(Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
                    DAG.getVTList(HiLoVT, MVT::i1), LH, RH, Overflow);
    SDValue OutH = AddSubWithOverflow.getValue(0);
    return std::make_pair(OutL, OutH);
  };

  // This helper creates a SRL of the pair (LL, LH) by Shift.
  auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
    unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
    if (Shift < HBitWidth) {
      SDValue ShAmt = DAG.getShiftAmountConstant(Shift, HiLoVT, DL);
      SDValue ResL = DAG.getNode(ISD::FSHR, DL, HiLoVT, LH, LL, ShAmt);
      SDValue ResH = DAG.getNode(ISD::SRL, DL, HiLoVT, LH, ShAmt);
      return std::make_pair(ResL, ResH);
    }
    SDValue Zero = DAG.getConstant(0, DL, HiLoVT);
    if (Shift == HBitWidth)
      return std::make_pair(LH, Zero);
    assert(Shift - HBitWidth < HBitWidth &&
           "We shouldn't generate an undefined shift");
    SDValue ShAmt = DAG.getShiftAmountConstant(Shift - HBitWidth, HiLoVT, DL);
    return std::make_pair(DAG.getNode(ISD::SRL, DL, HiLoVT, LH, ShAmt), Zero);
  };

  // Knowledge of leading zeros may help to reduce the multiplier.
  unsigned KnownLeadingZeros = DAG.computeKnownBits(N0).countMinLeadingZeros();

  UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
      Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()));

  assert(!LL == !LH && "Expected both input halves or no input halves!");
  if (!LL)
    std::tie(LL, LH) = DAG.SplitScalar(N0, DL, HiLoVT, HiLoVT);
  SDValue QL = LL;
  SDValue QH = LH;
  if (Magics.PreShift != 0)
    std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.PreShift);

  SmallVector<SDValue, 4> UMulResult;
  if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
    return false;

  QL = UMulResult[2];
  QH = UMulResult[3];

  if (Magics.IsAdd) {
    auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
    std::tie(NPQL, NPQH) = MakeSRLLong(NPQL, NPQH, 1);
    std::tie(QL, QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
  }

  if (Magics.PostShift != 0)
    std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.PostShift);

  unsigned Opcode = N->getOpcode();
  if (Opcode != ISD::UREM) {
    Result.push_back(QL);
    Result.push_back(QH);
  }

  if (Opcode != ISD::UDIV) {
    SmallVector<SDValue, 2> MulResult;
    if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
      return false;

    assert(MulResult.size() == 2);

    auto [RemL, RemH] =
        MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);

    Result.push_back(RemL);
    Result.push_back(RemH);
  }

  return true;
}

bool TargetLowering::expandDIVREMByConstant(SDNode *N,
                                            SmallVectorImpl<SDValue> &Result,
                                            EVT HiLoVT, SelectionDAG &DAG,
                                            SDValue LL, SDValue LH) const {
  unsigned Opcode = N->getOpcode();

  // TODO: Support signed division/remainder.
  if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
    return false;
  assert(
      (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
      "Unexpected opcode");

  auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!CN)
    return false;

  APInt Divisor = CN->getAPIntValue();

  // We depend on the UREM by constant optimization in DAGCombiner that requires
  // high multiply.
  if (!isOperationLegalOrCustom(ISD::MULHU, HiLoVT) &&
      !isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT))
    return false;

  // Don't expand if optimizing for size.
  if (DAG.shouldOptForSize())
    return false;

  // Early out for 0 or 1 divisors.
  if (Divisor.ule(1))
    return false;

  if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
                                                  DAG, LL, LH))
    return true;

  if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
                                            LH))
    return true;

  return false;
}

// Check that (every element of) Z is undef or not an exact multiple of BW.
static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
  return ISD::matchUnaryPredicate(
      Z,
      [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
      /*AllowUndefs=*/true, /*AllowTruncation=*/true);
}

static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) {
  EVT VT = Node->getValueType(0);
  SDValue ShX, ShY;
  SDValue ShAmt, InvShAmt;
  SDValue X = Node->getOperand(0);
  SDValue Y = Node->getOperand(1);
  SDValue Z = Node->getOperand(2);
  SDValue Mask = Node->getOperand(3);
  SDValue VL = Node->getOperand(4);

  unsigned BW = VT.getScalarSizeInBits();
  bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL;
  SDLoc DL(SDValue(Node, 0));

  EVT ShVT = Z.getValueType();
  if (isNonZeroModBitWidthOrUndef(Z, BW)) {
    // fshl: X << C | Y >> (BW - C)
    // fshr: X << (BW - C) | Y >> C
    // where C = Z % BW is not zero
    SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
    ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
    InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitWidthC, ShAmt, Mask, VL);
    ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask,
                      VL);
    ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask,
                      VL);
  } else {
    // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
    // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
    SDValue BitMask = DAG.getConstant(BW - 1, DL, ShVT);
    if (isPowerOf2_32(BW)) {
      // Z % BW -> Z & (BW - 1)
      ShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, Z, BitMask, Mask, VL);
      // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
      SDValue NotZ = DAG.getNode(ISD::VP_XOR, DL, ShVT, Z,
                                 DAG.getAllOnesConstant(DL, ShVT), Mask, VL);
      InvShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, NotZ, BitMask, Mask, VL);
    } else {
      SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
      ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL);
      InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitMask, ShAmt, Mask, VL);
    }

    SDValue One = DAG.getConstant(1, DL, ShVT);
    if (IsFSHL) {
      ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL);
      SDValue ShY1 = DAG.getNode(ISD::VP_SRL, DL, VT, Y, One, Mask, VL);
      ShY = DAG.getNode(ISD::VP_SRL, DL, VT, ShY1, InvShAmt, Mask, VL);
    } else {
      SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL);
      ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL);
      ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, ShAmt, Mask, VL);
    }
  }
  return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL);
}

SDValue TargetLowering::expandFunnelShift(SDNode *Node,
                                          SelectionDAG &DAG) const {
  if (Node->isVPOpcode())
    return expandVPFunnelShift(Node, DAG);

  EVT VT = Node->getValueType(0);

  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
                        !isOperationLegalOrCustom(ISD::SRL, VT) ||
                        !isOperationLegalOrCustom(ISD::SUB, VT) ||
                        !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
    return SDValue();

  SDValue X = Node->getOperand(0);
  SDValue Y = Node->getOperand(1);
  SDValue Z = Node->getOperand(2);

  unsigned BW = VT.getScalarSizeInBits();
  bool IsFSHL = Node->getOpcode() == ISD::FSHL;
  SDLoc DL(SDValue(Node, 0));

  EVT ShVT = Z.getValueType();

  // If a funnel shift in the other direction is more supported, use it.
  unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
  if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
      isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
    if (isNonZeroModBitWidthOrUndef(Z, BW)) {
      // fshl X, Y, Z -> fshr X, Y, -Z
      // fshr X, Y, Z -> fshl X, Y, -Z
      Z = DAG.getNegative(Z, DL, ShVT);
    } else {
      // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
      // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
      SDValue One = DAG.getConstant(1, DL, ShVT);
      if (IsFSHL) {
        Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
        X = DAG.getNode(ISD::SRL, DL, VT, X, One);
      } else {
        X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
        Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
      }
      Z = DAG.getNOT(DL, Z, ShVT);
    }
    return DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
  }

  SDValue ShX, ShY;
  SDValue ShAmt, InvShAmt;
  if (isNonZeroModBitWidthOrUndef(Z, BW)) {
    // fshl: X << C | Y >> (BW - C)
    // fshr: X << (BW - C) | Y >> C
    // where C = Z % BW is not zero
    SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
    ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
    InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
    ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
    ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
  } else {
    // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
    // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
    SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
    if (isPowerOf2_32(BW)) {
      // Z % BW -> Z & (BW - 1)
      ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
      // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
      InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
    } else {
      SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
      ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
      InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
    }

    SDValue One = DAG.getConstant(1, DL, ShVT);
    if (IsFSHL) {
      ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
      SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
      ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
    } else {
      SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
      ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
      ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
    }
  }
  return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
}

// TODO: Merge with expandFunnelShift.
SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
                                  SelectionDAG &DAG) const {
  EVT VT = Node->getValueType(0);
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  bool IsLeft = Node->getOpcode() == ISD::ROTL;
  SDValue Op0 = Node->getOperand(0);
  SDValue Op1 = Node->getOperand(1);
  SDLoc DL(SDValue(Node, 0));

  EVT ShVT = Op1.getValueType();
  SDValue Zero = DAG.getConstant(0, DL, ShVT);

  // If a rotate in the other direction is more supported, use it.
  unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
  if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
      isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
    SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
    return DAG.getNode(RevRot, DL, VT, Op0, Sub);
  }

  if (!AllowVectorOps && VT.isVector() &&
      (!isOperationLegalOrCustom(ISD::SHL, VT) ||
       !isOperationLegalOrCustom(ISD::SRL, VT) ||
       !isOperationLegalOrCustom(ISD::SUB, VT) ||
       !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
       !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
    return SDValue();

  unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
  unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
  SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
  SDValue ShVal;
  SDValue HsVal;
  if (isPowerOf2_32(EltSizeInBits)) {
    // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
    // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
    SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
    SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
    ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
    SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
    HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
  } else {
    // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
    // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
    SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
    SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
    ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
    SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
    SDValue One = DAG.getConstant(1, DL, ShVT);
    HsVal =
        DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
  }
  return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
}

/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
/// a chain of halving decompositions (halving element width) and/or vector
/// widening (doubling element count). This guides expansion strategy selection:
/// if true, the halving/widening path produces better code than bit-by-bit.
///
/// HalveDepth tracks halving steps only (each creates ~4x more operations).
/// Widening steps are cheap (O(1) pad/extract) and don't count.
/// Limiting halvings to 2 prevents exponential blowup:
///   1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
///   2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
///   3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
                                  EVT VT, unsigned HalveDepth = 0,
                                  unsigned TotalDepth = 0) {
  if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
    return false;
  if (TLI.isOperationLegalOrCustom(ISD::CLMUL, VT))
    return true;
  if (!TLI.isTypeLegal(VT))
    return false;

  unsigned BW = VT.getScalarSizeInBits();

  // Halve: halve element width, same element count.
  // This is the expensive step -- each halving creates ~4x more operations.
  if (BW % 2 == 0) {
    EVT HalfEltVT = EVT::getIntegerVT(Ctx, BW / 2);
    EVT HalfVT = VT.changeVectorElementType(Ctx, HalfEltVT);
    if (TLI.isTypeLegal(HalfVT) &&
        canNarrowCLMULToLegal(TLI, Ctx, HalfVT, HalveDepth + 1, TotalDepth + 1))
      return true;
  }

  // Widen: double element count (fixed-width vectors only).
  // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
  EVT WideVT = VT.getDoubleNumVectorElementsVT(Ctx);
  if (TLI.isTypeLegal(WideVT) &&
      canNarrowCLMULToLegal(TLI, Ctx, WideVT, HalveDepth, TotalDepth + 1))
    return true;

  return false;
}

SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc DL(Node);
  EVT VT = Node->getValueType(0);
  SDValue X = Node->getOperand(0);
  SDValue Y = Node->getOperand(1);
  unsigned BW = VT.getScalarSizeInBits();
  unsigned Opcode = Node->getOpcode();
  LLVMContext &Ctx = *DAG.getContext();

  switch (Opcode) {
  case ISD::CLMUL: {
    // For vector types, try decomposition strategies that leverage legal
    // CLMUL on narrower or wider element types, avoiding the expensive
    // bit-by-bit expansion.
    if (VT.isVector()) {
      // Strategy 1: Halving decomposition to half-element-width CLMUL.
      // Applies ExpandIntRes_CLMUL's identity element-wise:
      //   CLMUL(X, Y) = (Hi << HalfBW) | Lo
      // where:
      //   Lo = CLMUL(XLo, YLo)
      //   Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
      unsigned HalfBW = BW / 2;
      if (BW % 2 == 0) {
        EVT HalfEltVT = EVT::getIntegerVT(Ctx, HalfBW);
        EVT HalfVT =
            EVT::getVectorVT(Ctx, HalfEltVT, VT.getVectorElementCount());
        if (isTypeLegal(HalfVT) && canNarrowCLMULToLegal(*this, Ctx, HalfVT,
                                                         /*HalveDepth=*/1)) {
          SDValue ShAmt = DAG.getShiftAmountConstant(HalfBW, VT, DL);

          // Extract low and high halves of each element.
          SDValue XLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, X);
          SDValue XHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT,
                                    DAG.getNode(ISD::SRL, DL, VT, X, ShAmt));
          SDValue YLo = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, Y);
          SDValue YHi = DAG.getNode(ISD::TRUNCATE, DL, HalfVT,
                                    DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt));

          // Lo = CLMUL(XLo, YLo)
          SDValue Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, XLo, YLo);

          // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
          SDValue LoH = DAG.getNode(ISD::CLMULH, DL, HalfVT, XLo, YLo);
          SDValue Cross1 = DAG.getNode(ISD::CLMUL, DL, HalfVT, XLo, YHi);
          SDValue Cross2 = DAG.getNode(ISD::CLMUL, DL, HalfVT, XHi, YLo);
          SDValue Cross = DAG.getNode(ISD::XOR, DL, HalfVT, Cross1, Cross2);
          SDValue Hi = DAG.getNode(ISD::XOR, DL, HalfVT, LoH, Cross);

          // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
          SDValue LoExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo);
          SDValue HiExt = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Hi);
          SDValue HiShifted = DAG.getNode(ISD::SHL, DL, VT, HiExt, ShAmt);
          return DAG.getNode(ISD::OR, DL, VT, LoExt, HiShifted);
        }
      }

      // Strategy 2: Promote to double-element-width CLMUL.
      // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
      {
        EVT ExtVT = VT.widenIntegerElementType(Ctx);
        if (isTypeLegal(ExtVT) && isOperationLegalOrCustom(ISD::CLMUL, ExtVT)) {
          // If CLMUL on ExtVT is Custom (not Legal), the target may
          // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
          // fallback costs O(BW) vectorized iterations. Only widen when
          // element count is small enough that scalarization is cheaper.
          unsigned NumElts = VT.getVectorMinNumElements();
          if (isOperationLegal(ISD::CLMUL, ExtVT) || NumElts < BW) {
            SDValue XExt = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, X);
            SDValue YExt = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, Y);
            SDValue Mul = DAG.getNode(ISD::CLMUL, DL, ExtVT, XExt, YExt);
            return DAG.getNode(ISD::TRUNCATE, DL, VT, Mul);
          }
        }
      }

      // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
      // vector, extract lower result). CLMUL is element-wise, so upper
      // (undef) lanes don't affect the lower results.
      // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
      if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
        EVT WideVT = EVT::getVectorVT(Ctx, VT.getVectorElementType(), EC * 2);
        if (isTypeLegal(WideVT) && canNarrowCLMULToLegal(*this, Ctx, WideVT)) {
          SDValue Undef = DAG.getUNDEF(WideVT);
          SDValue XWide = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, Undef,
                                      X, DAG.getVectorIdxConstant(0, DL));
          SDValue YWide = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, Undef,
                                      Y, DAG.getVectorIdxConstant(0, DL));
          SDValue WideRes = DAG.getNode(ISD::CLMUL, DL, WideVT, XWide, YWide);
          return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, WideRes,
                             DAG.getVectorIdxConstant(0, DL));
        }
      }
    }

    // Special case: clmul(X, ~0) is equivalent to a "parallel prefix XOR" or
    // "bitwise parity" operation.
    if (isAllOnesOrAllOnesSplat(Y)) {
      SDValue R = X;
      for (unsigned I = 1; I < BW; I <<= 1) {
        SDValue ShAmt = DAG.getShiftAmountConstant(I, VT, DL);
        SDValue Shifted = DAG.getNode(ISD::SHL, DL, VT, R, ShAmt);
        R = DAG.getNode(ISD::XOR, DL, VT, R, Shifted);
      }
      return R;
    }

    // NOTE: If you change this expansion, please update the cost model
    // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
    // Intrinsic::clmul.

    // Strategy 4: multiplication with holes.
    //
    // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
    // do occur, they wind up in a "hole" and are subsequently masked out of the
    // result.
    //
    // A hole of 3 bits is optimal for 32-bit and 64-bit inputs. 128-bit
    // integers need a larger hole, and for smaller integers the fallback below
    // is more efficient.
    //
    // Based on bmul64 in bearssl and bmul in the rust polyval crate.
    if (BW >= 32 && BW <= 64 &&
        isOperationLegalOrCustom(ISD::MUL, getTypeToTransformTo(Ctx, VT))) {

      // Set every fourth bit of each nibble, equivalent to 0b00010001...0001.
      APInt MaskVal = APInt::getSplat(BW, APInt(4, 0b0001));

      // Create versions of X and Y that keep only the I-th bit of
      // each nibble.
      SDValue M[4], Xp[4], Yp[4];
      for (unsigned I = 0; I < 4; ++I) {
        M[I] = DAG.getConstant(MaskVal.shl(I), DL, VT);
        Xp[I] = DAG.getNode(ISD::AND, DL, VT, X, M[I]);
        Yp[I] = DAG.getNode(ISD::AND, DL, VT, Y, M[I]);
      }

      // Codegens these expressions (16 multiplications):
      //
      // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
      // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
      // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
      // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
      SDValue Res = DAG.getConstant(0, DL, VT);
      for (unsigned I = 0; I < 4; ++I) {
        SDValue Zi = DAG.getConstant(0, DL, VT);
        for (unsigned J = 0; J < 4; ++J) {
          unsigned K = (I + 4 - J) % 4;
          SDValue P = DAG.getNode(ISD::MUL, DL, VT, Xp[J], Yp[K]);
          Zi = DAG.getNode(ISD::XOR, DL, VT, Zi, P);
        }

        // Keep only the bits belonging to this iteration, and bitwise or it all
        // together.
        Zi = DAG.getNode(ISD::AND, DL, VT, Zi, M[I]);
        Res = DAG.getNode(ISD::OR, DL, VT, Res, Zi, SDNodeFlags::Disjoint);
      }
      return Res;
    }

    // Strategy 5: the naive fallback.
    EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), Ctx, VT);

    SDValue Res = DAG.getConstant(0, DL, VT);
    for (unsigned I = 0; I < BW; ++I) {
      SDValue ShiftAmt = DAG.getShiftAmountConstant(I, VT, DL);
      SDValue Mask = DAG.getConstant(APInt::getOneBitSet(BW, I), DL, VT);
      SDValue YMasked = DAG.getNode(ISD::AND, DL, VT, Y, Mask);

      // For targets with a fast bit test instruction (e.g., x86 BT) or without
      // multiply, use a shift-based expansion to avoid expensive MUL
      // instructions.
      SDValue Part;
      if (!hasBitTest(Y, ShiftAmt) &&
          isOperationLegalOrCustom(
              ISD::MUL, getTypeToTransformTo(*DAG.getContext(), VT))) {
        Part = DAG.getNode(ISD::MUL, DL, VT, X, YMasked);
      } else {
        // Canonical bit test: (Y & (1 << I)) != 0
        SDValue Zero = DAG.getConstant(0, DL, VT);
        SDValue Cond = DAG.getSetCC(DL, SetCCVT, YMasked, Zero, ISD::SETEQ);
        SDValue XShifted = DAG.getNode(ISD::SHL, DL, VT, X, ShiftAmt);
        Part = DAG.getSelect(DL, VT, Cond, Zero, XShifted);
      }
      Res = DAG.getNode(ISD::XOR, DL, VT, Res, Part);
    }
    return Res;
  }
  case ISD::CLMULR:
    // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
    if (isOperationLegalOrCustom(ISD::CLMUL, VT) &&
        isOperationLegalOrCustom(ISD::CLMULH, VT)) {
      SDValue Lo = DAG.getNode(ISD::CLMUL, DL, VT, X, Y);
      SDValue Hi = DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
      Lo = DAG.getNode(ISD::SRL, DL, VT, Lo,
                       DAG.getShiftAmountConstant(BW - 1, VT, DL));
      Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
                       DAG.getShiftAmountConstant(1, VT, DL));
      return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
    }
    [[fallthrough]];
  case ISD::CLMULH: {
    EVT ExtVT = VT.widenIntegerElementType(Ctx);
    // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
    // when any of these hold:
    // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
    // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
    // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
    //     expanded via halving/widening to reach legal CLMUL. The bitreverse
    //     path creates CLMUL(VT) which will be expanded efficiently. The
    //     promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
    //     causing a cycle.
    // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
    // => trunc path is preferred over the bitreverse path, as it avoids the
    // cost of 3 bitreverse operations.
    if (!isOperationLegalOrCustom(ISD::ZERO_EXTEND, ExtVT) ||
        !isOperationLegalOrCustom(ISD::SRL, ExtVT) ||
        (!isOperationLegalOrCustom(ISD::CLMUL, ExtVT) &&
         (isOperationLegalOrCustom(ISD::CLMUL, VT) ||
          canNarrowCLMULToLegal(*this, Ctx, VT)))) {
      SDValue XRev = DAG.getNode(ISD::BITREVERSE, DL, VT, X);
      SDValue YRev = DAG.getNode(ISD::BITREVERSE, DL, VT, Y);
      SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, VT, XRev, YRev);
      SDValue Res = DAG.getNode(ISD::BITREVERSE, DL, VT, ClMul);
      if (Opcode == ISD::CLMULH)
        Res = DAG.getNode(ISD::SRL, DL, VT, Res,
                          DAG.getShiftAmountConstant(1, VT, DL));
      return Res;
    }
    SDValue XExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, X);
    SDValue YExt = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Y);
    SDValue ClMul = DAG.getNode(ISD::CLMUL, DL, ExtVT, XExt, YExt);
    unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
    SDValue HiBits = DAG.getNode(ISD::SRL, DL, ExtVT, ClMul,
                                 DAG.getShiftAmountConstant(ShAmt, ExtVT, DL));
    return DAG.getNode(ISD::TRUNCATE, DL, VT, HiBits);
  }
  }
  llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
}

SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc DL(Node);
  EVT VT = Node->getValueType(0);
  SDValue Val = Node->getOperand(0);
  SDValue Msk = Node->getOperand(1);
  unsigned BW = VT.getScalarSizeInBits();

  // Hacker's Delight §7-4: Compress, or Generalized Extract
  SDValue X = DAG.getNode(ISD::AND, DL, VT, Val, Msk);
  SDValue M = Msk;
  SDValue One = DAG.getShiftAmountConstant(1, VT, DL);
  SDValue Mk = DAG.getNode(ISD::SHL, DL, VT, DAG.getNOT(DL, M, VT), One);

  // Repeatedly compute which bits would shift to the right by an odd amount,
  // shift all such bits in parallel using a mask, and double the shift amount.
  for (unsigned I = 1; I < BW; I *= 2) {
    // This expands the "parallel prefix" operation to clmul(Mk, ~0).
    SDValue Mp =
        DAG.getNode(ISD::CLMUL, DL, VT, Mk, DAG.getAllOnesConstant(DL, VT));
    SDValue Mv = DAG.getNode(ISD::AND, DL, VT, Mp, M);
    SDValue ShiftI = DAG.getShiftAmountConstant(I, VT, DL);
    SDValue MvS = DAG.getNode(ISD::SRL, DL, VT, Mv, ShiftI);
    M = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ISD::XOR, DL, VT, M, Mv), MvS,
                    SDNodeFlags::Disjoint);
    SDValue T = DAG.getNode(ISD::AND, DL, VT, X, Mv);
    SDValue TS = DAG.getNode(ISD::SRL, DL, VT, T, ShiftI);
    X = DAG.getNode(ISD::OR, DL, VT, DAG.getNode(ISD::XOR, DL, VT, X, T), TS,
                    SDNodeFlags::Disjoint);
    if (I * 2 < BW)
      Mk = DAG.getNode(ISD::AND, DL, VT, Mk, DAG.getNOT(DL, Mp, VT));
  }

  return X;
}

SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc DL(Node);
  EVT VT = Node->getValueType(0);
  SDValue Val = Node->getOperand(0);
  SDValue Msk = Node->getOperand(1);
  unsigned BW = VT.getScalarSizeInBits();

  // Hacker's Delight §7-5: Expand, or Generalized Insert.
  unsigned LogBW = Log2_32_Ceil(BW);
  SmallVector<SDValue, 8> MvArray(LogBW);
  SDValue One = DAG.getShiftAmountConstant(1, VT, DL);
  SDValue Mc = Msk;
  SDValue Mk = DAG.getNode(ISD::SHL, DL, VT, DAG.getNOT(DL, Msk, VT), One);

  // First pass: compute move masks for each power of two that a bit moves by.
  for (unsigned S = 0; S < LogBW; ++S) {
    unsigned ShiftS = 1u << S;
    // This expands the "parallel prefix" operation to clmul(Mk, ~0).
    SDValue Mp =
        DAG.getNode(ISD::CLMUL, DL, VT, Mk, DAG.getAllOnesConstant(DL, VT));
    SDValue Mv = DAG.getNode(ISD::AND, DL, VT, Mp, Mc);
    MvArray[S] = Mv;
    if (S + 1 < LogBW) {
      SDValue McXorMv = DAG.getNode(ISD::XOR, DL, VT, Mc, Mv);
      SDValue MvShifted = DAG.getNode(
          ISD::SRL, DL, VT, Mv, DAG.getShiftAmountConstant(ShiftS, VT, DL));
      Mc = DAG.getNode(ISD::OR, DL, VT, McXorMv, MvShifted,
                       SDNodeFlags::Disjoint);
      Mk = DAG.getNode(ISD::AND, DL, VT, Mk, DAG.getNOT(DL, Mp, VT));
    }
  }

  // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
  // Each pass handles half the shift amount of the previous pass.
  SDValue X = Val;
  for (int S = (int)LogBW - 1; S >= 0; --S) {
    SDValue ShiftSv = DAG.getShiftAmountConstant(1ull << S, VT, DL);
    SDValue T = DAG.getNode(ISD::SHL, DL, VT, X, ShiftSv);
    SDValue UnshiftedBits =
        DAG.getNode(ISD::AND, DL, VT, X, DAG.getNOT(DL, MvArray[S], VT));
    SDValue ShiftedBits = DAG.getNode(ISD::AND, DL, VT, T, MvArray[S]);
    X = DAG.getNode(ISD::OR, DL, VT, UnshiftedBits, ShiftedBits,
                    SDNodeFlags::Disjoint);
  }

  return DAG.getNode(ISD::AND, DL, VT, X, Msk);
}

void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
                                      SelectionDAG &DAG) const {
  assert(Node->getNumOperands() == 3 && "Not a double-shift!");
  EVT VT = Node->getValueType(0);
  unsigned VTBits = VT.getScalarSizeInBits();
  assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");

  bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
  bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
  SDValue ShOpLo = Node->getOperand(0);
  SDValue ShOpHi = Node->getOperand(1);
  SDValue ShAmt = Node->getOperand(2);
  EVT ShAmtVT = ShAmt.getValueType();
  EVT ShAmtCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
  SDLoc dl(Node);

  // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
  // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
  // away during isel.
  SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
                                  DAG.getConstant(VTBits - 1, dl, ShAmtVT));
  SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
                                     DAG.getConstant(VTBits - 1, dl, ShAmtVT))
                       : DAG.getConstant(0, dl, VT);

  SDValue Tmp2, Tmp3;
  if (IsSHL) {
    Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
  } else {
    Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
    Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
  }

  // If the shift amount is larger or equal than the width of a part we don't
  // use the result from the FSHL/FSHR. Insert a test and select the appropriate
  // values for large shift amounts.
  SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
                                DAG.getConstant(VTBits, dl, ShAmtVT));
  SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
                              DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);

  if (IsSHL) {
    Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
    Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
  } else {
    Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
    Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
  }
}

SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
                                            SelectionDAG &DAG) const {
  // This implements llvm.canonicalize.f* by multiplication with 1.0, as
  // suggested in
  // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
  // It uses strict_fp operations even outside a strict_fp context in order
  // to guarantee that the canonicalization is not optimized away by later
  // passes. The result chain introduced by that is intentionally ignored
  // since no ordering requirement is intended here.
  EVT VT = Node->getValueType(0);
  SDLoc DL(Node);
  SDNodeFlags Flags = Node->getFlags();
  Flags.setNoFPExcept(true);
  SDValue One = DAG.getConstantFP(1.0, DL, VT);
  SDValue Mul =
      DAG.getNode(ISD::STRICT_FMUL, DL, {VT, MVT::Other},
                  {DAG.getEntryNode(), Node->getOperand(0), One}, Flags);
  return Mul;
}

SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
                                                      SelectionDAG &DAG) const {
  // Expand conversion from a native IEEE float type to an arbitrary FP format
  // returning the result as an integer using bit manipulation.
  EVT ResVT = Node->getValueType(0);
  SDLoc dl(Node);

  SDValue FloatVal = Node->getOperand(0);
  const uint64_t SemEnum = Node->getConstantOperandVal(1);
  const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
  const auto RoundMode =
      static_cast<RoundingMode>(Node->getConstantOperandVal(2));
  const bool Saturate = Node->getConstantOperandVal(3) != 0;

  // Supported destination formats.
  switch (Sem) {
  case APFloatBase::S_Float8E5M2:
  case APFloatBase::S_Float8E4M3FN:
  case APFloatBase::S_Float6E3M2FN:
  case APFloatBase::S_Float6E2M3FN:
  case APFloatBase::S_Float4E2M1FN:
    break;
  default:
    DAG.getContext()->emitError("CONVERT_TO_ARBITRARY_FP: not implemented "
                                "destination format (semantics enum " +
                                Twine(SemEnum) + ")");
    return SDValue();
  }

  // Supported rounding modes.
  switch (RoundMode) {
  case RoundingMode::NearestTiesToEven:
  case RoundingMode::TowardZero:
  case RoundingMode::TowardPositive:
  case RoundingMode::TowardNegative:
  case RoundingMode::NearestTiesToAway:
    break;
  default:
    DAG.getContext()->emitError(
        "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
        Twine(static_cast<int>(RoundMode)) + ")");
    return SDValue();
  }

  // Destination format parameters.
  const fltSemantics &DstSem = APFloatBase::EnumToSemantics(Sem);
  const unsigned DstBits = APFloat::getSizeInBits(DstSem);
  const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
  const unsigned DstMant = DstPrecision - 1;
  const unsigned DstExpBits = DstBits - DstMant - 1;
  const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
  const unsigned DstExpMax = (1U << DstExpBits) - 1;
  const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
  const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
  const fltNanEncoding DstNanEnc = DstSem.nanEncoding;

  // Compute the maximum normal exponent for the destination format.
  const unsigned DstExpMaxNormal =
      DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
                                                     : DstExpMax;

  // For NanOnly formats the max exponent field for finite values
  // is DstExpMax, but the encoding with exp = DstExpMax and
  // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
  // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
  // avoid the NaN encoding.
  uint64_t DstMaxMantAtMaxExp = DstMantMask;
  if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
      DstNanEnc == fltNanEncoding::AllOnes)
    DstMaxMantAtMaxExp = DstMantMask - 1;

  // Source format parameters.
  EVT SrcVT = FloatVal.getValueType();
  const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
  const unsigned SrcBits = APFloat::getSizeInBits(SrcSem);
  const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
  const unsigned SrcMant = SrcPrecision - 1;
  const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;

  // Work in the source integer type. Match the destination shape so the
  // expansion stays vector when ResVT is a vector.
  EVT IntScalarVT = EVT::getIntegerVT(*DAG.getContext(), SrcBits);
  EVT IntVT = ResVT.changeElementType(*DAG.getContext(), IntScalarVT);
  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IntVT);
  EVT FPSetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);

  SDValue Zero = DAG.getConstant(0, dl, IntVT);
  SDValue One = DAG.getConstant(1, dl, IntVT);

  // Bitcast source float to integer to extract the sign bit.
  SDValue Src = DAG.getNode(ISD::BITCAST, dl, IntVT, FloatVal);
  SDValue SignBit =
      DAG.getNode(ISD::SRL, dl, IntVT, Src,
                  DAG.getShiftAmountConstant(SrcBits - 1, IntVT, dl));

  // Classify the input.
  SDValue FPZero = DAG.getConstantFP(0.0, dl, SrcVT);
  SDValue FPInf = DAG.getConstantFP(APFloat::getInf(SrcSem), dl, SrcVT);
  SDValue AbsVal = DAG.getNode(ISD::FABS, dl, SrcVT, FloatVal);
  SDValue IsNaN = DAG.getSetCC(dl, FPSetCCVT, FloatVal, FPZero, ISD::SETUO);
  SDValue IsInf = DAG.getSetCC(dl, FPSetCCVT, AbsVal, FPInf, ISD::SETOEQ);
  SDValue IsZero = DAG.getSetCC(dl, FPSetCCVT, FloatVal, FPZero, ISD::SETOEQ);

  // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
  // source denormals automatically. The result is unspecified for Inf/NaN, but
  // those inputs are detected above and override the final result.
  EVT FrexpExpScalarVT =
      getValueType(DAG.getDataLayout(), Type::getInt32Ty(*DAG.getContext()));
  EVT FrexpExpVT = SrcVT.changeElementType(*DAG.getContext(), FrexpExpScalarVT);
  SDValue Frexp =
      DAG.getNode(ISD::FFREXP, dl, DAG.getVTList(SrcVT, FrexpExpVT), FloatVal);
  SDValue FrexpFrac = Frexp.getValue(0);
  SDValue FrexpExp = Frexp.getValue(1);

  SDValue FrexpFracInt = DAG.getNode(ISD::BITCAST, dl, IntVT, FrexpFrac);
  SDValue EffSrcMant = DAG.getNode(ISD::AND, dl, IntVT, FrexpFracInt,
                                   DAG.getConstant(SrcMantMask, dl, IntVT));

  SDValue FrexpExpExt = DAG.getSExtOrTrunc(FrexpExp, dl, IntVT);
  SDValue NewExp = DAG.getNode(ISD::ADD, dl, IntVT, FrexpExpExt,
                               DAG.getConstant(DstBias - 1, dl, IntVT));

  // Compute rounding increment given the round bit, sticky bits, and LSB
  // of the truncated mantissa.
  auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
                            SDValue LSB) -> SDValue {
    switch (RoundMode) {
    case RoundingMode::NearestTiesToEven: {
      // Round up if round_bit && (sticky || lsb)
      SDValue StickyOrLSB = DAG.getNode(ISD::OR, dl, IntVT, StickyBits, LSB);
      return DAG.getNode(ISD::AND, dl, IntVT, RoundBit, StickyOrLSB);
    }
    case RoundingMode::TowardZero:
      return Zero;
    case RoundingMode::TowardPositive: {
      // Round up if positive and any truncated bits are set.
      SDValue AnyTruncBits =
          DAG.getNode(ISD::OR, dl, IntVT, RoundBit, StickyBits);
      SDValue HasTruncBits =
          DAG.getSetCC(dl, SetCCVT, AnyTruncBits, Zero, ISD::SETNE);
      SDValue IsPositive = DAG.getSetCC(dl, SetCCVT, SignBit, Zero, ISD::SETEQ);
      SDValue DoRound =
          DAG.getNode(ISD::AND, dl, SetCCVT, HasTruncBits, IsPositive);
      return DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, DoRound);
    }
    case RoundingMode::TowardNegative: {
      // Round up if negative and any truncated bits are set (to -Inf).
      SDValue AnyTruncBits =
          DAG.getNode(ISD::OR, dl, IntVT, RoundBit, StickyBits);
      SDValue HasTruncBits =
          DAG.getSetCC(dl, SetCCVT, AnyTruncBits, Zero, ISD::SETNE);
      SDValue IsNegative = DAG.getSetCC(dl, SetCCVT, SignBit, Zero, ISD::SETNE);
      SDValue DoRound =
          DAG.getNode(ISD::AND, dl, SetCCVT, HasTruncBits, IsNegative);
      return DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, DoRound);
    }
    case RoundingMode::NearestTiesToAway:
      return RoundBit;
    default:
      llvm_unreachable("unsupported rounding mode");
    }
  };

  // Round mantissa from SrcMant bits to DstMant bits.
  SDValue TruncMant;
  SDValue RoundUp;
  if (SrcMant > DstMant) {
    const unsigned Shift = SrcMant - DstMant;
    SDValue ShiftConst = DAG.getShiftAmountConstant(Shift, IntVT, dl);
    TruncMant = DAG.getNode(ISD::SRL, dl, IntVT, EffSrcMant, ShiftConst);

    // Check bit at position Shift - 1 aka the round bit.
    SDValue RoundBit;
    if (Shift >= 1) {
      SDValue RoundBitShift = DAG.getShiftAmountConstant(Shift - 1, IntVT, dl);
      SDValue ShiftedMant =
          DAG.getNode(ISD::SRL, dl, IntVT, EffSrcMant, RoundBitShift);
      RoundBit = DAG.getNode(ISD::AND, dl, IntVT, ShiftedMant, One);
    } else {
      RoundBit = Zero;
    }

    // OR of all bits below the round bit to get sticky bits.
    SDValue StickyBits;
    if (Shift >= 2) {
      uint64_t StickyMask = maskTrailingOnes<uint64_t>(Shift - 1);
      StickyBits = DAG.getNode(ISD::AND, dl, IntVT, EffSrcMant,
                               DAG.getConstant(StickyMask, dl, IntVT));
      StickyBits = DAG.getSetCC(dl, SetCCVT, StickyBits, Zero, ISD::SETNE);
      StickyBits = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, StickyBits);
    } else {
      StickyBits = Zero;
    }

    // LSB of truncated mantissa.
    SDValue LSB = DAG.getNode(ISD::AND, dl, IntVT, TruncMant, One);

    RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
  } else {
    // If DstMant >= SrcMant, then no rounding needed, just shift left.
    SDValue MantShift =
        DAG.getShiftAmountConstant(DstMant - SrcMant, IntVT, dl);
    TruncMant = DAG.getNode(ISD::SHL, dl, IntVT, EffSrcMant, MantShift);
    RoundUp = Zero;
  }

  // Apply rounding.
  SDValue RoundedMant = DAG.getNode(ISD::ADD, dl, IntVT, TruncMant, RoundUp);

  // Handle mantissa overflow from rounding.
  // If rounded_mant > DstMantMask, carry into exponent.
  SDValue MantOverflow =
      DAG.getSetCC(dl, SetCCVT, RoundedMant,
                   DAG.getConstant(DstMantMask, dl, IntVT), ISD::SETGT);
  // On overflow: mant = 0, exp += 1.
  SDValue AdjMant = DAG.getSelect(dl, IntVT, MantOverflow, Zero, RoundedMant);
  SDValue AdjExp =
      DAG.getNode(ISD::ADD, dl, IntVT, NewExp,
                  DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, MantOverflow));

  // Precompute sign shifted to MSB of destination.
  SDValue SignShifted =
      DAG.getNode(ISD::SHL, dl, IntVT, SignBit,
                  DAG.getShiftAmountConstant(DstBits - 1, IntVT, dl));

  // Destination denormal conversion (when new_exp <= 0).
  // Shift the mantissa right by 1 - new_exp additional bits and set the
  // exponent field to 0.
  SDValue ExpIsNeg = DAG.getSetCC(dl, SetCCVT, AdjExp,
                                  DAG.getConstant(1, dl, IntVT), ISD::SETLT);

  SDValue DenormResult;
  {
    // denorm_shift = 1 - NewExp.
    SDValue DenormShift = DAG.getNode(ISD::SUB, dl, IntVT, One, NewExp);

    // full_src_mant = (1 << SrcMant) | EffSrcMant.
    SDValue ImplicitOne =
        DAG.getNode(ISD::SHL, dl, IntVT, One,
                    DAG.getShiftAmountConstant(SrcMant, IntVT, dl));
    SDValue FullSrcMant =
        DAG.getNode(ISD::OR, dl, IntVT, EffSrcMant, ImplicitOne);

    // Total right shift = DenormShift + (SrcMant - DstMant).
    int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
    SDValue TotalShift =
        DAG.getNode(ISD::ADD, dl, IntVT, DenormShift,
                    DAG.getSignedConstant(MantDelta, dl, IntVT));

    // Clamp total shift to avoid UB, then truncate denorm mantissa.
    EVT ShiftVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
    SDValue MaxShift = DAG.getConstant(SrcBits - 1, dl, IntVT);
    SDValue ClampedShift =
        DAG.getNode(ISD::UMIN, dl, IntVT, TotalShift, MaxShift);
    SDValue DenormTruncMant =
        DAG.getNode(ISD::SRL, dl, IntVT, FullSrcMant,
                    DAG.getZExtOrTrunc(ClampedShift, dl, ShiftVT));

    // Rounding for denorm path.
    SDValue DenormRoundUp;
    {
      // Round bit is at position TotalShift - 1 of FullSrcMant.
      // Clamp to at least 1 so the subtraction doesn't underflow and create
      // shift nodes with invalid shift amounts.
      SDValue SafeShift = DAG.getNode(ISD::UMAX, dl, IntVT, ClampedShift, One);
      SDValue RoundBitPos = DAG.getNode(ISD::SUB, dl, IntVT, SafeShift, One);
      SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(RoundBitPos, dl, ShiftVT);
      SDValue DenormRoundBit = DAG.getNode(
          ISD::AND, dl, IntVT,
          DAG.getNode(ISD::SRL, dl, IntVT, FullSrcMant, RoundBitPosAmt), One);

      // Sticky: all bits below round bit.
      // sticky_mask = (1 << RoundBitPos) - 1
      SDValue StickyMask = DAG.getNode(
          ISD::SUB, dl, IntVT,
          DAG.getNode(ISD::SHL, dl, IntVT, One, RoundBitPosAmt), One);
      SDValue DenormStickyBits =
          DAG.getNode(ISD::AND, dl, IntVT, FullSrcMant, StickyMask);
      SDValue HasSticky = DAG.getNode(
          ISD::ZERO_EXTEND, dl, IntVT,
          DAG.getSetCC(dl, SetCCVT, DenormStickyBits, Zero, ISD::SETNE));

      SDValue DenormLSB =
          DAG.getNode(ISD::AND, dl, IntVT, DenormTruncMant, One);

      DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);

      // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
      SDValue ShiftGEOne =
          DAG.getSetCC(dl, SetCCVT, ClampedShift, One, ISD::SETUGE);
      DenormRoundUp = DAG.getSelect(dl, IntVT, ShiftGEOne, DenormRoundUp, Zero);
    }

    SDValue DenormRoundedMant =
        DAG.getNode(ISD::ADD, dl, IntVT, DenormTruncMant, DenormRoundUp);

    // If rounding caused overflow into the normal range, then we get the
    // smallest normal number.
    SDValue DenormMantOF =
        DAG.getSetCC(dl, SetCCVT, DenormRoundedMant,
                     DAG.getConstant(DstMantMask, dl, IntVT), ISD::SETGT);
    SDValue DenormFinalMant =
        DAG.getSelect(dl, IntVT, DenormMantOF, Zero, DenormRoundedMant);
    SDValue DenormFinalExp = DAG.getSelect(dl, IntVT, DenormMantOF, One, Zero);

    // Assemble: sign | (exp << DstMant) | mant
    SDValue DenormExpShifted =
        DAG.getNode(ISD::SHL, dl, IntVT, DenormFinalExp,
                    DAG.getShiftAmountConstant(DstMant, IntVT, dl));
    DenormResult = DAG.getNode(
        ISD::OR, dl, IntVT,
        DAG.getNode(ISD::OR, dl, IntVT, SignShifted, DenormExpShifted),
        DenormFinalMant);
  }

  // Exponent overflow detection.
  SDValue ExpOF =
      DAG.getSetCC(dl, SetCCVT, AdjExp,
                   DAG.getConstant(DstExpMaxNormal, dl, IntVT), ISD::SETGT);

  // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
  // a value that exceeds the max allowed mantissa at that exponent.
  SDValue ExpAtMax =
      DAG.getSetCC(dl, SetCCVT, AdjExp,
                   DAG.getConstant(DstExpMaxNormal, dl, IntVT), ISD::SETEQ);
  SDValue MantExceedsMax =
      DAG.getSetCC(dl, SetCCVT, AdjMant,
                   DAG.getConstant(DstMaxMantAtMaxExp, dl, IntVT), ISD::SETGT);
  SDValue ExpMantOF =
      DAG.getNode(ISD::AND, dl, SetCCVT, ExpAtMax, MantExceedsMax);
  SDValue IsOverflow = DAG.getNode(ISD::OR, dl, SetCCVT, ExpOF, ExpMantOF);

  // Build overflow result.
  SDValue OverflowResult;

  if (Saturate) {
    // Clamp to max finite value:
    // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
    uint64_t MaxFinite =
        ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
    OverflowResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
                                 DAG.getConstant(MaxFinite, dl, IntVT));
  } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
    // Produce infinity.
    uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
    OverflowResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
                                 DAG.getConstant(InfBits, dl, IntVT));
  } else {
    // Emit poison if no Inf in format and not saturating.
    OverflowResult = DAG.getPOISON(IntVT);
  }

  // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
  SDValue NormExpShifted =
      DAG.getNode(ISD::SHL, dl, IntVT, AdjExp,
                  DAG.getShiftAmountConstant(DstMant, IntVT, dl));
  SDValue NormResult = DAG.getNode(
      ISD::OR, dl, IntVT,
      DAG.getNode(ISD::OR, dl, IntVT, SignShifted, NormExpShifted), AdjMant);

  // Build special-value results.
  SDValue NaNResult;
  if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
    // Produce canonical NaN.
    const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
    NaNResult =
        DAG.getConstant(((uint64_t)DstExpMax << DstMant) | QNaNBit, dl, IntVT);
  } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
             DstNanEnc == fltNanEncoding::AllOnes) {
    // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
    NaNResult = DAG.getConstant(((uint64_t)DstExpMax << DstMant) | DstMantMask,
                                dl, IntVT);
  } else {
    // NaN -> poison for finite only values.
    NaNResult = DAG.getPOISON(IntVT);
  }

  // Inf handling.
  SDValue InfResult;
  if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
    // Produce signed infinity.
    uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
    InfResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
                            DAG.getConstant(InfBits, dl, IntVT));
  } else if (Saturate) {
    // Inf saturates to max finite.
    uint64_t MaxFinite =
        ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
    InfResult = DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
                            DAG.getConstant(MaxFinite, dl, IntVT));
  } else {
    // No Inf and not saturating -> poison.
    InfResult = DAG.getPOISON(IntVT);
  }

  SDValue ZeroResult = SignShifted;

  // Final selection in an order: NaN takes priority, then Inf, then Zero.
  SDValue FiniteResult =
      DAG.getSelect(dl, IntVT, ExpIsNeg, DenormResult, NormResult);
  FiniteResult =
      DAG.getSelect(dl, IntVT, IsOverflow, OverflowResult, FiniteResult);

  SDValue Result = FiniteResult;
  Result = DAG.getSelect(dl, IntVT, IsZero, ZeroResult, Result);
  Result = DAG.getSelect(dl, IntVT, IsInf, InfResult, Result);
  Result = DAG.getSelect(dl, IntVT, IsNaN, NaNResult, Result);

  // Truncate to destination integer type.
  return DAG.getZExtOrTrunc(Result, dl, ResVT);
}

SDValue
TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
                                                SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT DstVT = Node->getValueType(0);
  EVT DstScalarVT = DstVT.getScalarType();

  SDValue IntVal = Node->getOperand(0);
  const uint64_t SemEnum = Node->getConstantOperandVal(1);
  const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);

  // Supported source formats.
  switch (Sem) {
  case APFloatBase::S_Float8E5M2:
  case APFloatBase::S_Float8E4M3FN:
  case APFloatBase::S_Float6E3M2FN:
  case APFloatBase::S_Float6E2M3FN:
  case APFloatBase::S_Float4E2M1FN:
    break;
  default:
    DAG.getContext()->emitError("CONVERT_FROM_ARBITRARY_FP: not implemented "
                                "source format (semantics enum " +
                                Twine(SemEnum) + ")");
    return SDValue();
  }

  const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(Sem);
  const unsigned SrcBits = APFloat::getSizeInBits(SrcSem);
  const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
  const unsigned SrcMant = SrcPrecision - 1;
  const unsigned SrcExp = SrcBits - SrcMant - 1;
  const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
  const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;

  // Destination format parameters.
  const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
  const unsigned DstBits = APFloat::getSizeInBits(DstSem);
  const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
  const unsigned DstExpBits = DstBits - DstMant - 1;
  const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
  const int DstBias = 1 - DstMinExp;
  const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;

  // Work in an integer type matching the destination float width.
  EVT IntScalarVT = EVT::getIntegerVT(*DAG.getContext(), DstBits);
  EVT IntVT = DstVT.isVector()
                  ? EVT::getVectorVT(*DAG.getContext(), IntScalarVT,
                                     DstVT.getVectorElementCount())
                  : IntScalarVT;

  SDValue Src = DAG.getZExtOrTrunc(IntVal, dl, IntVT);

  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), IntVT);

  SDValue Zero = DAG.getConstant(0, dl, IntVT);
  SDValue One = DAG.getConstant(1, dl, IntVT);

  // Extract bit fields.
  const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
  const uint64_t ExpMask = (1ULL << SrcExp) - 1;

  SDValue MantField = DAG.getNode(ISD::AND, dl, IntVT, Src,
                                  DAG.getConstant(MantMask, dl, IntVT));

  SDValue ExpField =
      DAG.getNode(ISD::AND, dl, IntVT,
                  DAG.getNode(ISD::SRL, dl, IntVT, Src,
                              DAG.getShiftAmountConstant(SrcMant, IntVT, dl)),
                  DAG.getConstant(ExpMask, dl, IntVT));

  SDValue SignBit =
      DAG.getNode(ISD::SRL, dl, IntVT, Src,
                  DAG.getShiftAmountConstant(SrcBits - 1, IntVT, dl));

  SDValue SignShifted =
      DAG.getNode(ISD::SHL, dl, IntVT, SignBit,
                  DAG.getShiftAmountConstant(DstBits - 1, IntVT, dl));

  // Classify the input.
  SDValue ExpAllOnes = DAG.getConstant(ExpMask, dl, IntVT);
  SDValue IsExpAllOnes =
      DAG.getSetCC(dl, SetCCVT, ExpField, ExpAllOnes, ISD::SETEQ);
  SDValue IsExpZero = DAG.getSetCC(dl, SetCCVT, ExpField, Zero, ISD::SETEQ);
  SDValue IsMantZero = DAG.getSetCC(dl, SetCCVT, MantField, Zero, ISD::SETEQ);
  SDValue IsMantNonZero =
      DAG.getSetCC(dl, SetCCVT, MantField, Zero, ISD::SETNE);

  SDValue IsNaN;
  if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
    IsNaN = DAG.getBoolConstant(false, dl, SetCCVT, IntVT);
  } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
    IsNaN = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantNonZero);
  } else {
    assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
    SDValue MantAllOnes = DAG.getConstant(MantMask, dl, IntVT);
    SDValue IsMantAllOnes =
        DAG.getSetCC(dl, SetCCVT, MantField, MantAllOnes, ISD::SETEQ);
    IsNaN = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantAllOnes);
  }

  SDValue IsInf;
  if (NFBehavior == fltNonfiniteBehavior::IEEE754)
    IsInf = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantZero);
  else
    IsInf = DAG.getBoolConstant(false, dl, SetCCVT, IntVT);

  SDValue IsZero = DAG.getNode(ISD::AND, dl, SetCCVT, IsExpZero, IsMantZero);
  SDValue IsDenorm =
      DAG.getNode(ISD::AND, dl, SetCCVT, IsExpZero, IsMantNonZero);

  // Normal value conversion.
  const int BiasAdjust = DstBias - SrcBias;
  SDValue NormDstExp =
      DAG.getNode(ISD::ADD, dl, IntVT, ExpField,
                  DAG.getConstant(APInt(DstBits, BiasAdjust, true), dl, IntVT));

  SDValue NormDstMant;
  if (DstMant > SrcMant) {
    SDValue NormDstMantShift =
        DAG.getShiftAmountConstant(DstMant - SrcMant, IntVT, dl);
    NormDstMant = DAG.getNode(ISD::SHL, dl, IntVT, MantField, NormDstMantShift);
  } else {
    NormDstMant = MantField;
  }

  SDValue DstMantShift = DAG.getShiftAmountConstant(DstMant, IntVT, dl);
  SDValue NormExpShifted =
      DAG.getNode(ISD::SHL, dl, IntVT, NormDstExp, DstMantShift);
  SDValue NormResult =
      DAG.getNode(ISD::OR, dl, IntVT,
                  DAG.getNode(ISD::OR, dl, IntVT, SignShifted, NormExpShifted),
                  NormDstMant);

  // Denormal value conversion.
  SDValue DenormResult;
  {
    const unsigned IntVTBits = DstBits;
    SDValue LeadingZeros =
        DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, IntVT, MantField);

    const int DenormExpConst =
        (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
    SDValue DenormDstExp = DAG.getNode(
        ISD::SUB, dl, IntVT,
        DAG.getConstant(APInt(DstBits, DenormExpConst, true), dl, IntVT),
        LeadingZeros);

    SDValue MantMSB =
        DAG.getNode(ISD::SUB, dl, IntVT,
                    DAG.getConstant(IntVTBits - 1, dl, IntVT), LeadingZeros);

    SDValue LeadingOne = DAG.getNode(ISD::SHL, dl, IntVT, One, MantMSB);
    SDValue Frac = DAG.getNode(ISD::XOR, dl, IntVT, MantField, LeadingOne);

    const unsigned ShiftSub = IntVTBits - 1 - DstMant;
    SDValue ShiftAmount = DAG.getNode(ISD::SUB, dl, IntVT, LeadingZeros,
                                      DAG.getConstant(ShiftSub, dl, IntVT));

    SDValue DenormDstMant = DAG.getNode(ISD::SHL, dl, IntVT, Frac, ShiftAmount);

    SDValue DenormExpShifted =
        DAG.getNode(ISD::SHL, dl, IntVT, DenormDstExp, DstMantShift);
    DenormResult = DAG.getNode(
        ISD::OR, dl, IntVT,
        DAG.getNode(ISD::OR, dl, IntVT, SignShifted, DenormExpShifted),
        DenormDstMant);
  }

  SDValue FiniteResult =
      DAG.getSelect(dl, IntVT, IsDenorm, DenormResult, NormResult);

  const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
  SDValue NaNResult =
      DAG.getConstant((DstExpAllOnes << DstMant) | QNaNBit, dl, IntVT);

  SDValue InfResult =
      DAG.getNode(ISD::OR, dl, IntVT, SignShifted,
                  DAG.getConstant(DstExpAllOnes << DstMant, dl, IntVT));

  SDValue ZeroResult = SignShifted;

  SDValue Result = FiniteResult;
  Result = DAG.getSelect(dl, IntVT, IsZero, ZeroResult, Result);
  Result = DAG.getSelect(dl, IntVT, IsInf, InfResult, Result);
  Result = DAG.getSelect(dl, IntVT, IsNaN, NaNResult, Result);

  return DAG.getNode(ISD::BITCAST, dl, DstVT, Result);
}

bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
                                      SelectionDAG &DAG) const {
  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
  SDValue Src = Node->getOperand(OpNo);
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Node->getValueType(0);
  SDLoc dl(SDValue(Node, 0));

  // FIXME: Only f32 to i64 conversions are supported.
  if (SrcVT != MVT::f32 || DstVT != MVT::i64)
    return false;

  if (Node->isStrictFPOpcode())
    // When a NaN is converted to an integer a trap is allowed. We can't
    // use this expansion here because it would eliminate that trap. Other
    // traps are also allowed and cannot be eliminated. See
    // IEEE 754-2008 sec 5.8.
    return false;

  // Expand f32 -> i64 conversion
  // This algorithm comes from compiler-rt's implementation of fixsfdi:
  // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
  unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
  EVT IntVT = SrcVT.changeTypeToInteger();
  EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());

  SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
  SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
  SDValue Bias = DAG.getConstant(127, dl, IntVT);
  SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
  SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
  SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);

  SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);

  SDValue ExponentBits = DAG.getNode(
      ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
      DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
  SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);

  SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
                             DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
                             DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
  Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);

  SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
                          DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
                          DAG.getConstant(0x00800000, dl, IntVT));

  R = DAG.getZExtOrTrunc(R, dl, DstVT);

  R = DAG.getSelectCC(
      dl, Exponent, ExponentLoBit,
      DAG.getNode(ISD::SHL, dl, DstVT, R,
                  DAG.getZExtOrTrunc(
                      DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
                      dl, IntShVT)),
      DAG.getNode(ISD::SRL, dl, DstVT, R,
                  DAG.getZExtOrTrunc(
                      DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
                      dl, IntShVT)),
      ISD::SETGT);

  SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
                            DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);

  Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
                           DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
  return true;
}

bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
                                      SDValue &Chain,
                                      SelectionDAG &DAG) const {
  SDLoc dl(SDValue(Node, 0));
  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
  SDValue Src = Node->getOperand(OpNo);

  EVT SrcVT = Src.getValueType();
  EVT DstVT = Node->getValueType(0);
  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
  EVT DstSetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);

  // Only expand vector types if we have the appropriate vector bit operations.
  unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
                                                   ISD::FP_TO_SINT;
  if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
                           !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
    return false;

  // If the maximum float value is smaller then the signed integer range,
  // the destination signmask can't be represented by the float, so we can
  // just use FP_TO_SINT directly.
  const fltSemantics &APFSem = SrcVT.getFltSemantics();
  APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));
  APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
  if (APFloat::opOverflow &
      APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
    if (Node->isStrictFPOpcode()) {
      Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
                           { Node->getOperand(0), Src });
      Chain = Result.getValue(1);
    } else
      Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
    return true;
  }

  // Don't expand it if there isn't cheap fsub instruction.
  if (!isOperationLegalOrCustom(
          Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
    return false;

  SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
  SDValue Sel;

  if (Node->isStrictFPOpcode()) {
    Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
                       Node->getOperand(0), /*IsSignaling*/ true);
    Chain = Sel.getValue(1);
  } else {
    Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
  }

  bool Strict = Node->isStrictFPOpcode() ||
                shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);

  if (Strict) {
    // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
    // signmask then offset (the result of which should be fully representable).
    // Sel = Src < 0x8000000000000000
    // FltOfs = select Sel, 0, 0x8000000000000000
    // IntOfs = select Sel, 0, 0x8000000000000000
    // Result = fp_to_sint(Src - FltOfs) ^ IntOfs

    // TODO: Should any fast-math-flags be set for the FSUB?
    SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
                                   DAG.getConstantFP(0.0, dl, SrcVT), Cst);
    Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
    SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
                                   DAG.getConstant(0, dl, DstVT),
                                   DAG.getConstant(SignMask, dl, DstVT));
    SDValue SInt;
    if (Node->isStrictFPOpcode()) {
      SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
                                { Chain, Src, FltOfs });
      SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
                         { Val.getValue(1), Val });
      Chain = SInt.getValue(1);
    } else {
      SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
      SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
    }
    Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
  } else {
    // Expand based on maximum range of FP_TO_SINT:
    // True = fp_to_sint(Src)
    // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
    // Result = select (Src < 0x8000000000000000), True, False

    SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
    // TODO: Should any fast-math-flags be set for the FSUB?
    SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
                                DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
    False = DAG.getNode(ISD::XOR, dl, DstVT, False,
                        DAG.getConstant(SignMask, dl, DstVT));
    Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
    Result = DAG.getSelect(dl, DstVT, Sel, True, False);
  }
  return true;
}

bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
                                      SDValue &Chain, SelectionDAG &DAG) const {
  // This transform is not correct for converting 0 when rounding mode is set
  // to round toward negative infinity which will produce -0.0. So disable
  // under strictfp.
  if (Node->isStrictFPOpcode())
    return false;

  SDValue Src = Node->getOperand(0);
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Node->getValueType(0);

  // If the input is known to be non-negative and SINT_TO_FP is legal then use
  // it.
  if (Node->getFlags().hasNonNeg() &&
      isOperationLegalOrCustom(ISD::SINT_TO_FP, SrcVT)) {
    Result =
        DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), DstVT, Node->getOperand(0));
    return true;
  }

  if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
    return false;

  // Only expand vector types if we have the appropriate vector bit
  // operations.
  if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
                           !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
                           !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
                           !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
                           !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
    return false;

  SDLoc dl(SDValue(Node, 0));

  // Implementation of unsigned i64 to f64 following the algorithm in
  // __floatundidf in compiler_rt.  This implementation performs rounding
  // correctly in all rounding modes with the exception of converting 0
  // when rounding toward negative infinity. In that case the fsub will
  // produce -0.0. This will be added to +0.0 and produce -0.0 which is
  // incorrect.
  SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
  SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
      llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT);
  SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
  SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
  SDValue HiShift = DAG.getShiftAmountConstant(32, SrcVT, dl);

  SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
  SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
  SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
  SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
  SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
  SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
  Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
  return true;
}

SDValue
TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
                                               SelectionDAG &DAG) const {
  unsigned Opcode = Node->getOpcode();
  assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
          Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
         "Wrong opcode");

  if (Node->getFlags().hasNoNaNs()) {
    ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
    EVT VT = Node->getValueType(0);
    if ((!isCondCodeLegal(Pred, VT.getSimpleVT()) ||
         !isOperationLegalOrCustom(ISD::VSELECT, VT)) &&
        VT.isVector())
      return SDValue();
    SDValue Op1 = Node->getOperand(0);
    SDValue Op2 = Node->getOperand(1);
    return DAG.getSelectCC(SDLoc(Node), Op1, Op2, Op1, Op2, Pred,
                           Node->getFlags());
  }

  return SDValue();
}

SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
                                              SelectionDAG &DAG) const {
  if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
    return Expanded;

  EVT VT = Node->getValueType(0);
  if (VT.isScalableVector())
    report_fatal_error(
        "Expanding fminnum/fmaxnum for scalable vectors is undefined.");

  SDLoc dl(Node);
  unsigned NewOp =
      Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;

  if (isOperationLegalOrCustom(NewOp, VT)) {
    SDValue Quiet0 = Node->getOperand(0);
    SDValue Quiet1 = Node->getOperand(1);

    if (!Node->getFlags().hasNoNaNs()) {
      // Insert canonicalizes if it's possible we need to quiet to get correct
      // sNaN behavior.
      if (!DAG.isKnownNeverSNaN(Quiet0)) {
        Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
                             Node->getFlags());
      }
      if (!DAG.isKnownNeverSNaN(Quiet1)) {
        Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
                             Node->getFlags());
      }
    }

    return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
  }

  // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
  // instead if there are no NaNs.
  if (Node->getFlags().hasNoNaNs() ||
      (DAG.isKnownNeverNaN(Node->getOperand(0)) &&
       DAG.isKnownNeverNaN(Node->getOperand(1)))) {
    unsigned IEEE2018Op =
        Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
    if (isOperationLegalOrCustom(IEEE2018Op, VT))
      return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
                         Node->getOperand(1), Node->getFlags());
  }

  if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
    return SelCC;

  return SDValue();
}

SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
                                                SelectionDAG &DAG) const {
  if (SDValue Expanded = expandVectorNaryOpBySplitting(N, DAG))
    return Expanded;

  SDLoc DL(N);
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  unsigned Opc = N->getOpcode();
  EVT VT = N->getValueType(0);
  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  bool IsMax = Opc == ISD::FMAXIMUM;
  SDNodeFlags Flags = N->getFlags();

  // First, implement comparison not propagating NaN. If no native fmin or fmax
  // available, use plain select with setcc instead.
  SDValue MinMax;
  unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
  unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;

  // FIXME: We should probably define fminnum/fmaxnum variants with correct
  // signed zero behavior.
  bool MinMaxMustRespectOrderedZero = false;

  if (isOperationLegalOrCustom(CompOpcIeee, VT)) {
    MinMax = DAG.getNode(CompOpcIeee, DL, VT, LHS, RHS, Flags);
    MinMaxMustRespectOrderedZero = true;
  } else if (isOperationLegalOrCustom(CompOpc, VT)) {
    MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS, Flags);
  } else {
    if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
      return DAG.UnrollVectorOp(N);

    // NaN (if exists) will be propagated later, so orderness doesn't matter.
    SDValue Compare =
        DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETOGT : ISD::SETOLT);
    MinMax = DAG.getSelect(DL, VT, Compare, LHS, RHS, Flags);
  }

  // Propagate any NaN of both operands
  if (!N->getFlags().hasNoNaNs() &&
      (!DAG.isKnownNeverNaN(RHS) || !DAG.isKnownNeverNaN(LHS))) {
    ConstantFP *FPNaN = ConstantFP::get(*DAG.getContext(),
                                        APFloat::getNaN(VT.getFltSemantics()));
    MinMax = DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETUO),
                           DAG.getConstantFP(*FPNaN, DL, VT), MinMax, Flags);
  }

  // fminimum/fmaximum requires -0.0 less than +0.0
  if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
      !DAG.isKnownNeverLogicalZero(RHS) && !DAG.isKnownNeverLogicalZero(LHS)) {
    SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
                                  DAG.getConstantFP(0.0, DL, VT), ISD::SETOEQ);
    SDValue TestZero =
        DAG.getTargetConstant(IsMax ? fcPosZero : fcNegZero, DL, MVT::i32);
    SDValue LCmp = DAG.getSelect(
        DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, LHS, TestZero), LHS,
        MinMax, Flags);
    SDValue RCmp = DAG.getSelect(
        DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, RHS, TestZero), RHS,
        LCmp, Flags);
    MinMax = DAG.getSelect(DL, VT, IsZero, RCmp, MinMax, Flags);
  }

  return MinMax;
}

SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
                                                      SelectionDAG &DAG) const {
  SDLoc DL(Node);
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  unsigned Opc = Node->getOpcode();
  EVT VT = Node->getValueType(0);
  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  bool IsMax = Opc == ISD::FMAXIMUMNUM;
  SDNodeFlags Flags = Node->getFlags();

  unsigned NewOp =
      Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;

  if (isOperationLegalOrCustom(NewOp, VT)) {
    if (!Flags.hasNoNaNs()) {
      // Insert canonicalizes if it's possible we need to quiet to get correct
      // sNaN behavior.
      if (!DAG.isKnownNeverSNaN(LHS)) {
        LHS = DAG.getNode(ISD::FCANONICALIZE, DL, VT, LHS, Flags);
      }
      if (!DAG.isKnownNeverSNaN(RHS)) {
        RHS = DAG.getNode(ISD::FCANONICALIZE, DL, VT, RHS, Flags);
      }
    }

    return DAG.getNode(NewOp, DL, VT, LHS, RHS, Flags);
  }

  // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
  // same behaviors for all of other cases: +0.0 vs -0.0 included.
  if (Flags.hasNoNaNs() ||
      (DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS))) {
    unsigned IEEE2019Op =
        Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
    if (isOperationLegalOrCustom(IEEE2019Op, VT))
      return DAG.getNode(IEEE2019Op, DL, VT, LHS, RHS, Flags);
  }

  // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
  // either one for +0.0 vs -0.0.
  if ((Flags.hasNoNaNs() ||
       (DAG.isKnownNeverSNaN(LHS) && DAG.isKnownNeverSNaN(RHS))) &&
      (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(LHS) ||
       DAG.isKnownNeverLogicalZero(RHS))) {
    unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
    if (isOperationLegalOrCustom(IEEE2008Op, VT))
      return DAG.getNode(IEEE2008Op, DL, VT, LHS, RHS, Flags);
  }

  if (VT.isVector() &&
      (isOperationLegalOrCustomOrPromote(Opc, VT.getVectorElementType()) ||
       !isOperationLegalOrCustom(ISD::VSELECT, VT)))
    return DAG.UnrollVectorOp(Node);

  // If only one operand is NaN, override it with another operand.
  if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(LHS)) {
    LHS = DAG.getSelectCC(DL, LHS, LHS, RHS, LHS, ISD::SETUO);
  }
  if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(RHS)) {
    RHS = DAG.getSelectCC(DL, RHS, RHS, LHS, RHS, ISD::SETUO);
  }

  // Always prefer RHS if equal.
  SDValue MinMax =
      DAG.getSelectCC(DL, LHS, RHS, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);

  // TODO: We need quiet sNaN if strictfp.

  // Fixup signed zero behavior.
  if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(LHS) ||
      DAG.isKnownNeverLogicalZero(RHS)) {
    return MinMax;
  }
  SDValue TestZero =
      DAG.getTargetConstant(IsMax ? fcPosZero : fcNegZero, DL, MVT::i32);
  SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
                                DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ);
  EVT IntVT = VT.changeTypeToInteger();
  EVT FloatVT = VT.changeElementType(*DAG.getContext(), MVT::f32);
  SDValue LHSTrunc = LHS;
  if (!isTypeLegal(IntVT) && !isOperationLegalOrCustom(ISD::IS_FPCLASS, VT)) {
    LHSTrunc = DAG.getNode(ISD::FP_ROUND, DL, FloatVT, LHS,
                           DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  }
  // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
  // we preferred RHS when generate MinMax, if the operands are equal.
  SDValue RetZero = DAG.getSelect(
      DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, LHSTrunc, TestZero), LHS,
      MinMax, Flags);
  return DAG.getSelect(DL, VT, IsZero, RetZero, MinMax, Flags);
}

/// Returns a true value if if this FPClassTest can be performed with an ordered
/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
/// std::nullopt if it cannot be performed as a compare with 0.
static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
                                           const fltSemantics &Semantics,
                                           const MachineFunction &MF) {
  FPClassTest OrderedMask = Test & ~fcNan;
  FPClassTest NanTest = Test & fcNan;
  bool IsOrdered = NanTest == fcNone;
  bool IsUnordered = NanTest == fcNan;

  // Skip cases that are testing for only a qnan or snan.
  if (!IsOrdered && !IsUnordered)
    return std::nullopt;

  if (OrderedMask == fcZero &&
      MF.getDenormalMode(Semantics).Input == DenormalMode::IEEE)
    return IsOrdered;
  if (OrderedMask == (fcZero | fcSubnormal) &&
      MF.getDenormalMode(Semantics).inputsAreZero())
    return IsOrdered;
  return std::nullopt;
}

SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
                                         const FPClassTest OrigTestMask,
                                         SDNodeFlags Flags, const SDLoc &DL,
                                         SelectionDAG &DAG) const {
  EVT OperandVT = Op.getValueType();
  assert(OperandVT.isFloatingPoint());
  FPClassTest Test = OrigTestMask;

  // Degenerated cases.
  if (Test == fcNone)
    return DAG.getBoolConstant(false, DL, ResultVT, OperandVT);
  if (Test == fcAllFlags)
    return DAG.getBoolConstant(true, DL, ResultVT, OperandVT);

  // PPC double double is a pair of doubles, of which the higher part determines
  // the value class.
  if (OperandVT == MVT::ppcf128) {
    Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op,
                     DAG.getConstant(1, DL, MVT::i32));
    OperandVT = MVT::f64;
  }

  // Floating-point type properties.
  EVT ScalarFloatVT = OperandVT.getScalarType();
  const Type *FloatTy = ScalarFloatVT.getTypeForEVT(*DAG.getContext());
  const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
  bool IsF80 = (ScalarFloatVT == MVT::f80);

  // Some checks can be implemented using float comparisons, if floating point
  // exceptions are ignored.
  if (Flags.hasNoFPExcept() &&
      isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) {
    FPClassTest FPTestMask = Test;
    bool IsInvertedFP = false;

    if (FPClassTest InvertedFPCheck =
            invertFPClassTestIfSimpler(FPTestMask, true)) {
      FPTestMask = InvertedFPCheck;
      IsInvertedFP = true;
    }

    ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
    ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;

    // See if we can fold an | fcNan into an unordered compare.
    FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;

    // Can't fold the ordered check if we're only testing for snan or qnan
    // individually.
    if ((FPTestMask & fcNan) != fcNan)
      OrderedFPTestMask = FPTestMask;

    const bool IsOrdered = FPTestMask == OrderedFPTestMask;

    if (std::optional<bool> IsCmp0 =
            isFCmpEqualZero(FPTestMask, Semantics, DAG.getMachineFunction());
        IsCmp0 && (isCondCodeLegalOrCustom(
                      *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
                      OperandVT.getScalarType().getSimpleVT()))) {

      // If denormals could be implicitly treated as 0, this is not equivalent
      // to a compare with 0 since it will also be true for denormals.
      return DAG.getSetCC(DL, ResultVT, Op,
                          DAG.getConstantFP(0.0, DL, OperandVT),
                          *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
    }

    if (FPTestMask == fcNan &&
        isCondCodeLegalOrCustom(IsInvertedFP ? ISD::SETO : ISD::SETUO,
                                OperandVT.getScalarType().getSimpleVT()))
      return DAG.getSetCC(DL, ResultVT, Op, Op,
                          IsInvertedFP ? ISD::SETO : ISD::SETUO);

    bool IsOrderedInf = FPTestMask == fcInf;
    if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
        isCondCodeLegalOrCustom(IsOrderedInf ? OrderedCmpOpcode
                                             : UnorderedCmpOpcode,
                                OperandVT.getScalarType().getSimpleVT()) &&
        isOperationLegalOrCustom(ISD::FABS, OperandVT.getScalarType()) &&
        (isOperationLegal(ISD::ConstantFP, OperandVT.getScalarType()) ||
         (OperandVT.isVector() &&
          isOperationLegalOrCustom(ISD::BUILD_VECTOR, OperandVT)))) {
      // isinf(x) --> fabs(x) == inf
      SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
      SDValue Inf =
          DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT);
      return DAG.getSetCC(DL, ResultVT, Abs, Inf,
                          IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
    }

    if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
        isCondCodeLegalOrCustom(IsOrdered ? OrderedCmpOpcode
                                          : UnorderedCmpOpcode,
                                OperandVT.getSimpleVT())) {
      // isposinf(x) --> x == inf
      // isneginf(x) --> x == -inf
      // isposinf(x) || nan --> x u== inf
      // isneginf(x) || nan --> x u== -inf

      SDValue Inf = DAG.getConstantFP(
          APFloat::getInf(Semantics, OrderedFPTestMask == fcNegInf), DL,
          OperandVT);
      return DAG.getSetCC(DL, ResultVT, Op, Inf,
                          IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
    }

    if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
      // TODO: Could handle ordered case, but it produces worse code for
      // x86. Maybe handle ordered if fabs is free?

      ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
      ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;

      if (isCondCodeLegalOrCustom(IsOrdered ? OrderedOp : UnorderedOp,
                                  OperandVT.getScalarType().getSimpleVT())) {
        // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal

        // TODO: Maybe only makes sense if fabs is free. Integer test of
        // exponent bits seems better for x86.
        SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
        SDValue SmallestNormal = DAG.getConstantFP(
            APFloat::getSmallestNormalized(Semantics), DL, OperandVT);
        return DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal,
                            IsOrdered ? OrderedOp : UnorderedOp);
      }
    }

    if (FPTestMask == fcNormal) {
      // TODO: Handle unordered
      ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
      ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;

      if (isCondCodeLegalOrCustom(IsFiniteOp,
                                  OperandVT.getScalarType().getSimpleVT()) &&
          isCondCodeLegalOrCustom(IsNormalOp,
                                  OperandVT.getScalarType().getSimpleVT()) &&
          isFAbsFree(OperandVT)) {
        // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
        SDValue Inf =
            DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT);
        SDValue SmallestNormal = DAG.getConstantFP(
            APFloat::getSmallestNormalized(Semantics), DL, OperandVT);

        SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
        SDValue IsFinite = DAG.getSetCC(DL, ResultVT, Abs, Inf, IsFiniteOp);
        SDValue IsNormal =
            DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal, IsNormalOp);
        unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
        return DAG.getNode(LogicOp, DL, ResultVT, IsFinite, IsNormal);
      }
    }
  }

  // Some checks may be represented as inversion of simpler check, for example
  // "inf|normal|subnormal|zero" => !"nan".
  bool IsInverted = false;

  if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, false)) {
    Test = InvertedCheck;
    IsInverted = true;
  }

  // In the general case use integer operations.
  unsigned BitSize = OperandVT.getScalarSizeInBits();
  EVT IntVT = OperandVT.changeElementType(
      *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), BitSize));
  SDValue OpAsInt = DAG.getBitcast(IntVT, Op);

  // Various masks.
  APInt SignBit = APInt::getSignMask(BitSize);
  APInt ValueMask = APInt::getSignedMaxValue(BitSize);     // All bits but sign.
  APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.
  const unsigned ExplicitIntBitInF80 = 63;
  APInt ExpMask = Inf;
  if (IsF80)
    ExpMask.clearBit(ExplicitIntBitInF80);
  APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;
  APInt QNaNBitMask =
      APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
  APInt InversionMask = APInt::getAllOnes(ResultVT.getScalarSizeInBits());

  SDValue ValueMaskV = DAG.getConstant(ValueMask, DL, IntVT);
  SDValue SignBitV = DAG.getConstant(SignBit, DL, IntVT);
  SDValue ExpMaskV = DAG.getConstant(ExpMask, DL, IntVT);
  SDValue ZeroV = DAG.getConstant(0, DL, IntVT);
  SDValue InfV = DAG.getConstant(Inf, DL, IntVT);
  SDValue ResultInversionMask = DAG.getConstant(InversionMask, DL, ResultVT);

  SDValue Res;
  const auto appendResult = [&](SDValue PartialRes) {
    if (PartialRes) {
      if (Res)
        Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes);
      else
        Res = PartialRes;
    }
  };

  SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
  const auto getIntBitIsSet = [&]() -> SDValue {
    if (!IntBitIsSetV) {
      APInt IntBitMask(BitSize, 0);
      IntBitMask.setBit(ExplicitIntBitInF80);
      SDValue IntBitMaskV = DAG.getConstant(IntBitMask, DL, IntVT);
      SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV);
      IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE);
    }
    return IntBitIsSetV;
  };

  // Split the value into sign bit and absolute value.
  SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV);
  SDValue SignV = DAG.getSetCC(DL, ResultVT, OpAsInt,
                               DAG.getConstant(0, DL, IntVT), ISD::SETLT);

  // Tests that involve more than one class should be processed first.
  SDValue PartialRes;

  if (IsF80)
    ; // Detect finite numbers of f80 by checking individual classes because
      // they have different settings of the explicit integer bit.
  else if ((Test & fcFinite) == fcFinite) {
    // finite(V) ==> (a << 1) < (inf << 1)
    //
    // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
    // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.

    assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
           "finite check requires IEEE-like FP");

    SDValue One = DAG.getShiftAmountConstant(1, IntVT, DL);
    SDValue TwiceOp = DAG.getNode(ISD::SHL, DL, IntVT, OpAsInt, One);
    SDValue TwiceInf = DAG.getNode(ISD::SHL, DL, IntVT, ExpMaskV, One);

    PartialRes = DAG.getSetCC(DL, ResultVT, TwiceOp, TwiceInf, ISD::SETULT);
    Test &= ~fcFinite;
  } else if ((Test & fcFinite) == fcPosFinite) {
    // finite(V) && V > 0 ==> V < exp_mask
    PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT);
    Test &= ~fcPosFinite;
  } else if ((Test & fcFinite) == fcNegFinite) {
    // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
    PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT);
    PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
    Test &= ~fcNegFinite;
  }
  appendResult(PartialRes);

  if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
    // fcZero | fcSubnormal => test all exponent bits are 0
    // TODO: Handle sign bit specific cases
    if (PartialCheck == (fcZero | fcSubnormal)) {
      SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ExpMaskV);
      SDValue ExpIsZero =
          DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
      appendResult(ExpIsZero);
      Test &= ~PartialCheck & fcAllFlags;
    }
  }

  // Check for individual classes.

  if (unsigned PartialCheck = Test & fcZero) {
    if (PartialCheck == fcPosZero)
      PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ);
    else if (PartialCheck == fcZero)
      PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ);
    else // ISD::fcNegZero
      PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ);
    appendResult(PartialRes);
  }

  if (unsigned PartialCheck = Test & fcSubnormal) {
    // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
    // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
    SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
    SDValue MantissaV = DAG.getConstant(AllOneMantissa, DL, IntVT);
    SDValue VMinusOneV =
        DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT));
    PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT);
    if (PartialCheck == fcNegSubnormal)
      PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
    appendResult(PartialRes);
  }

  if (unsigned PartialCheck = Test & fcInf) {
    if (PartialCheck == fcPosInf)
      PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ);
    else if (PartialCheck == fcInf)
      PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ);
    else { // ISD::fcNegInf
      APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
      SDValue NegInfV = DAG.getConstant(NegInf, DL, IntVT);
      PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ);
    }
    appendResult(PartialRes);
  }

  if (unsigned PartialCheck = Test & fcNan) {
    APInt InfWithQnanBit = Inf | QNaNBitMask;
    SDValue InfWithQnanBitV = DAG.getConstant(InfWithQnanBit, DL, IntVT);
    if (PartialCheck == fcNan) {
      // isnan(V) ==> abs(V) > int(inf)
      PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
      if (IsF80) {
        // Recognize unsupported values as NaNs for compatibility with glibc.
        // In them (exp(V)==0) == int_bit.
        SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV);
        SDValue ExpIsZero =
            DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ);
        SDValue IsPseudo =
            DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ);
        PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo);
      }
    } else if (PartialCheck == fcQNan) {
      // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
      PartialRes =
          DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE);
    } else { // ISD::fcSNan
      // issignaling(V) ==> abs(V) > unsigned(Inf) &&
      //                    abs(V) < (unsigned(Inf) | quiet_bit)
      SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT);
      SDValue IsNotQnan =
          DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT);
      PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan);
    }
    appendResult(PartialRes);
  }

  if (unsigned PartialCheck = Test & fcNormal) {
    // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
    APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
    SDValue ExpLSBV = DAG.getConstant(ExpLSB, DL, IntVT);
    SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV);
    APInt ExpLimit = ExpMask - ExpLSB;
    SDValue ExpLimitV = DAG.getConstant(ExpLimit, DL, IntVT);
    PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT);
    if (PartialCheck == fcNegNormal)
      PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV);
    else if (PartialCheck == fcPosNormal) {
      SDValue PosSignV =
          DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInversionMask);
      PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV);
    }
    if (IsF80)
      PartialRes =
          DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet());
    appendResult(PartialRes);
  }

  if (!Res)
    return DAG.getConstant(IsInverted, DL, ResultVT);
  if (IsInverted)
    Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInversionMask);
  return Res;
}

// Only expand vector types if we have the appropriate vector bit operations.
static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
  assert(VT.isVector() && "Expected vector type");
  unsigned Len = VT.getScalarSizeInBits();
  return TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
         TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
         TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
         (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) &&
         TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT);
}

SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Op = Node->getOperand(0);
  unsigned Len = VT.getScalarSizeInBits();
  assert(VT.isInteger() && "CTPOP not implemented for this type.");

  // TODO: Add support for irregular type lengths.
  if (!(Len <= 128 && Len % 8 == 0))
    return SDValue();

  // Only expand vector types if we have the appropriate vector bit operations.
  if (VT.isVector() && !canExpandVectorCTPOP(*this, VT))
    return SDValue();

  // This is the "best" algorithm from
  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
  SDValue Mask55 =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
  SDValue Mask33 =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
  SDValue Mask0F =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);

  // v = v - ((v >> 1) & 0x55555555...)
  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
                   DAG.getNode(ISD::AND, dl, VT,
                               DAG.getNode(ISD::SRL, dl, VT, Op,
                                           DAG.getConstant(1, dl, ShVT)),
                               Mask55));
  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
  Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
                   DAG.getNode(ISD::AND, dl, VT,
                               DAG.getNode(ISD::SRL, dl, VT, Op,
                                           DAG.getConstant(2, dl, ShVT)),
                               Mask33));
  // v = (v + (v >> 4)) & 0x0F0F0F0F...
  Op = DAG.getNode(ISD::AND, dl, VT,
                   DAG.getNode(ISD::ADD, dl, VT, Op,
                               DAG.getNode(ISD::SRL, dl, VT, Op,
                                           DAG.getConstant(4, dl, ShVT))),
                   Mask0F);

  if (Len <= 8)
    return Op;

  // Avoid the multiply if we only have 2 bytes to add.
  // TODO: Only doing this for scalars because vectors weren't as obviously
  // improved.
  if (Len == 16 && !VT.isVector()) {
    // v = (v + (v >> 8)) & 0x00FF;
    return DAG.getNode(ISD::AND, dl, VT,
                     DAG.getNode(ISD::ADD, dl, VT, Op,
                                 DAG.getNode(ISD::SRL, dl, VT, Op,
                                             DAG.getConstant(8, dl, ShVT))),
                     DAG.getConstant(0xFF, dl, VT));
  }

  // v = (v * 0x01010101...) >> (Len - 8)
  SDValue V;
  if (isOperationLegalOrCustomOrPromote(
          ISD::MUL, getTypeToTransformTo(*DAG.getContext(), VT))) {
    SDValue Mask01 =
        DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
    V = DAG.getNode(ISD::MUL, dl, VT, Op, Mask01);
  } else {
    V = Op;
    for (unsigned Shift = 8; Shift < Len; Shift *= 2) {
      SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl);
      V = DAG.getNode(ISD::ADD, dl, VT, V,
                      DAG.getNode(ISD::SHL, dl, VT, V, ShiftC));
    }
  }
  return DAG.getNode(ISD::SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT));
}

SDValue TargetLowering::expandVPCTPOP(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Op = Node->getOperand(0);
  SDValue Mask = Node->getOperand(1);
  SDValue VL = Node->getOperand(2);
  unsigned Len = VT.getScalarSizeInBits();
  assert(VT.isInteger() && "VP_CTPOP not implemented for this type.");

  // TODO: Add support for irregular type lengths.
  if (!(Len <= 128 && Len % 8 == 0))
    return SDValue();

  // This is same algorithm of expandCTPOP from
  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
  SDValue Mask55 =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
  SDValue Mask33 =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
  SDValue Mask0F =
      DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);

  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5;

  // v = v - ((v >> 1) & 0x55555555...)
  Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT,
                     DAG.getNode(ISD::VP_SRL, dl, VT, Op,
                                 DAG.getConstant(1, dl, ShVT), Mask, VL),
                     Mask55, Mask, VL);
  Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL);

  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
  Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL);
  Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT,
                     DAG.getNode(ISD::VP_SRL, dl, VT, Op,
                                 DAG.getConstant(2, dl, ShVT), Mask, VL),
                     Mask33, Mask, VL);
  Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL);

  // v = (v + (v >> 4)) & 0x0F0F0F0F...
  Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(4, dl, ShVT),
                     Mask, VL),
  Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL);
  Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL);

  if (Len <= 8)
    return Op;

  // v = (v * 0x01010101...) >> (Len - 8)
  SDValue V;
  if (isOperationLegalOrCustomOrPromote(
          ISD::VP_MUL, getTypeToTransformTo(*DAG.getContext(), VT))) {
    SDValue Mask01 =
        DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
    V = DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL);
  } else {
    V = Op;
    for (unsigned Shift = 8; Shift < Len; Shift *= 2) {
      SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl);
      V = DAG.getNode(ISD::VP_ADD, dl, VT, V,
                      DAG.getNode(ISD::VP_SHL, dl, VT, V, ShiftC, Mask, VL),
                      Mask, VL);
    }
  }
  return DAG.getNode(ISD::VP_SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT),
                     Mask, VL);
}

SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Op = Node->getOperand(0);
  unsigned NumBitsPerElt = VT.getScalarSizeInBits();

  // If the non-ZERO_POISON version is supported we can use that instead.
  if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
      isOperationLegalOrCustom(ISD::CTLZ, VT))
    return DAG.getNode(ISD::CTLZ, dl, VT, Op);

  // If the ZERO_POISON version is supported use that and handle the zero case.
  if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_POISON, VT)) {
    EVT SetCCVT =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
    SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, VT, Op);
    SDValue Zero = DAG.getConstant(0, dl, VT);
    SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    return DAG.getSelect(dl, VT, SrcIsZero,
                         DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
  }

  // Only expand vector types if we have the appropriate vector bit operations.
  // This includes the operations needed to expand CTPOP if it isn't supported.
  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
                        (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
                         !canExpandVectorCTPOP(*this, VT)) ||
                        !isOperationLegalOrCustom(ISD::SRL, VT) ||
                        !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
    return SDValue();

  // for now, we do this:
  // x = x | (x >> 1);
  // x = x | (x >> 2);
  // ...
  // x = x | (x >>16);
  // x = x | (x >>32); // for 64-bit input
  // return popcount(~x);
  //
  // Ref: "Hacker's Delight" by Henry Warren
  for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
    SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
    Op = DAG.getNode(ISD::OR, dl, VT, Op,
                     DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
  }
  Op = DAG.getNOT(dl, Op, VT);
  return DAG.getNode(ISD::CTPOP, dl, VT, Op);
}

SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Op = Node->getOperand(0);
  SDValue Mask = Node->getOperand(1);
  SDValue VL = Node->getOperand(2);
  unsigned NumBitsPerElt = VT.getScalarSizeInBits();

  // do this:
  // x = x | (x >> 1);
  // x = x | (x >> 2);
  // ...
  // x = x | (x >>16);
  // x = x | (x >>32); // for 64-bit input
  // return popcount(~x);
  for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
    SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
    Op = DAG.getNode(ISD::VP_OR, dl, VT, Op,
                     DAG.getNode(ISD::VP_SRL, dl, VT, Op, Tmp, Mask, VL), Mask,
                     VL);
  }
  Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getAllOnesConstant(dl, VT),
                   Mask, VL);
  return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL);
}

SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  SDValue Op = DAG.getFreeze(Node->getOperand(0));
  unsigned NumBitsPerElt = VT.getScalarSizeInBits();

  // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
  // This transforms the sign bits into leading zeros that can be counted.
  SDValue ShiftAmt = DAG.getShiftAmountConstant(NumBitsPerElt - 1, VT, dl);
  SDValue SignBit = DAG.getNode(ISD::SRA, dl, VT, Op, ShiftAmt);
  SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, SignBit);
  SDValue Shl =
      DAG.getNode(ISD::SHL, dl, VT, Xor, DAG.getShiftAmountConstant(1, VT, dl));
  SDValue Or = DAG.getNode(ISD::OR, dl, VT, Shl, DAG.getConstant(1, dl, VT));
  return DAG.getNode(ISD::CTLZ_ZERO_POISON, dl, VT, Or);
}

SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
                                        const SDLoc &DL, EVT VT, SDValue Op,
                                        unsigned BitWidth) const {
  if (BitWidth != 32 && BitWidth != 64)
    return SDValue();

  const DataLayout &TD = DAG.getDataLayout();
  if (!isOperationCustom(ISD::ConstantPool, getPointerTy(TD)))
    return SDValue();

  APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
                                  : APInt(64, 0x0218A392CD3D5DBFULL);
  MachinePointerInfo PtrInfo =
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction());
  unsigned ShiftAmt = BitWidth - Log2_32(BitWidth);
  SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
  SDValue Lookup = DAG.getNode(
      ISD::SRL, DL, VT,
      DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg),
                  DAG.getConstant(DeBruijn, DL, VT)),
      DAG.getShiftAmountConstant(ShiftAmt, VT, DL));
  Lookup = DAG.getSExtOrTrunc(Lookup, DL, getPointerTy(TD));

  SmallVector<uint8_t> Table(BitWidth, 0);
  for (unsigned i = 0; i < BitWidth; i++) {
    APInt Shl = DeBruijn.shl(i);
    APInt Lshr = Shl.lshr(ShiftAmt);
    Table[Lshr.getZExtValue()] = i;
  }

  // Create a ConstantArray in Constant Pool
  auto *CA = ConstantDataArray::get(*DAG.getContext(), Table);
  SDValue CPIdx = DAG.getConstantPool(CA, getPointerTy(TD),
                                      TD.getPrefTypeAlign(CA->getType()));
  SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(),
                                   DAG.getMemBasePlusOffset(CPIdx, Lookup, DL),
                                   PtrInfo, MVT::i8);
  if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
    return ExtLoad;

  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ);
  return DAG.getSelect(DL, VT, SrcIsZero,
                       DAG.getConstant(BitWidth, DL, VT), ExtLoad);
}

SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  SDValue Op = Node->getOperand(0);
  unsigned NumBitsPerElt = VT.getScalarSizeInBits();

  // If the non-ZERO_POISON version is supported we can use that instead.
  if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
      isOperationLegalOrCustom(ISD::CTTZ, VT))
    return DAG.getNode(ISD::CTTZ, dl, VT, Op);

  // If the ZERO_POISON version is supported use that and handle the zero case.
  if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_POISON, VT)) {
    EVT SetCCVT =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
    SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_POISON, dl, VT, Op);
    SDValue Zero = DAG.getConstant(0, dl, VT);
    SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
    return DAG.getSelect(dl, VT, SrcIsZero,
                         DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
  }

  // Only expand vector types if we have the appropriate vector bit operations.
  // This includes the operations needed to expand CTPOP if it isn't supported.
  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
                        (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
                         !isOperationLegalOrCustom(ISD::CTLZ, VT) &&
                         !canExpandVectorCTPOP(*this, VT)) ||
                        !isOperationLegalOrCustom(ISD::SUB, VT) ||
                        !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
                        !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
    return SDValue();

  // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
  // to be expanded or converted to a libcall.
  if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(ISD::CTPOP, VT) &&
      !isOperationLegal(ISD::CTLZ, VT))
    if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt))
      return V;

  // for now, we use: { return popcount(~x & (x - 1)); }
  // unless the target has ctlz but not ctpop, in which case we use:
  // { return 32 - nlz(~x & (x-1)); }
  // Ref: "Hacker's Delight" by Henry Warren
  SDValue Tmp = DAG.getNode(
      ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
      DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));

  // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
  if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
    return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
                       DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
  }

  return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
}

SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
  SDValue Op = Node->getOperand(0);
  SDValue Mask = Node->getOperand(1);
  SDValue VL = Node->getOperand(2);
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);

  // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
  SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op,
                            DAG.getAllOnesConstant(dl, VT), Mask, VL);
  SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op,
                                 DAG.getConstant(1, dl, VT), Mask, VL);
  SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);
  return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL);
}

SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
                                             SelectionDAG &DAG) const {
  // %cond = to_bool_vec %source
  // %splat = splat /*val=*/VL
  // %tz = step_vector
  // %v = vp.select %cond, /*true=*/tz, /*false=*/%splat
  // %r = vp.reduce.umin %v
  SDLoc DL(N);
  SDValue Source = N->getOperand(0);
  SDValue Mask = N->getOperand(1);
  SDValue EVL = N->getOperand(2);
  EVT SrcVT = Source.getValueType();
  EVT ResVT = N->getValueType(0);
  EVT ResVecVT =
      EVT::getVectorVT(*DAG.getContext(), ResVT, SrcVT.getVectorElementCount());

  // Convert to boolean vector.
  if (SrcVT.getScalarType() != MVT::i1) {
    SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
    SrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
                             SrcVT.getVectorElementCount());
    Source = DAG.getNode(ISD::VP_SETCC, DL, SrcVT, Source, AllZero,
                         DAG.getCondCode(ISD::SETNE), Mask, EVL);
  }

  SDValue ExtEVL = DAG.getZExtOrTrunc(EVL, DL, ResVT);
  SDValue Splat = DAG.getSplat(ResVecVT, DL, ExtEVL);
  SDValue StepVec = DAG.getStepVector(DL, ResVecVT);
  SDValue Select =
      DAG.getNode(ISD::VP_SELECT, DL, ResVecVT, Source, StepVec, Splat, EVL);
  return DAG.getNode(ISD::VP_REDUCE_UMIN, DL, ResVT, ExtEVL, Select, Mask, EVL);
}

/// Returns a type-legalized version of \p Mask as the first item in the
/// pair. The second item contains a type-legalized step vector that's
/// guaranteed to fit the number of elements in \p Mask.
/// If the stepvector would require splitting, returns an empty SDValue
/// as the second item to signal that the operation should be split instead.
static std::pair<SDValue, SDValue>
getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
                          SelectionDAG &DAG) {
  EVT MaskVT = Mask.getValueType();
  EVT BoolVT = MaskVT.getScalarType();

  // Find a suitable type for a stepvector.
  // If zero is poison, we can assume the upper limit of the result is VF-1.
  ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
  if (MaskVT.isScalableVector())
    VScaleRange = getVScaleRange(&DAG.getMachineFunction().getFunction(), 64);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  uint64_t EltWidth = TLI.getBitWidthForCttzElements(
      EVT(TLI.getVectorIdxTy(DAG.getDataLayout())),
      MaskVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
  // If the step vector element type is smaller than the mask element type,
  // use the mask type directly to avoid widening issues.
  EltWidth = std::max(EltWidth, BoolVT.getFixedSizeInBits());
  EVT StepVT = MVT::getIntegerVT(EltWidth);
  EVT StepVecVT = MaskVT.changeVectorElementType(*DAG.getContext(), StepVT);

  // If promotion or widening is required to make the type legal, do it here.
  // Promotion of integers within LegalizeVectorOps is looking for types of
  // the same size but with a smaller number of larger elements, not the usual
  // larger size with the same number of larger elements.
  TargetLowering::LegalizeTypeAction TypeAction =
      TLI.getTypeAction(*DAG.getContext(), StepVecVT);
  SDValue StepVec;
  if (TypeAction == TargetLowering::TypePromoteInteger) {
    StepVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), StepVecVT);
    StepVec = DAG.getStepVector(DL, StepVecVT);
  } else if (TypeAction == TargetLowering::TypeWidenVector) {
    // For widening, the element count changes. Create a step vector with only
    // the original elements valid and zeros for padding. Also widen the mask.
    EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), StepVecVT);
    unsigned WideNumElts = WideVecVT.getVectorNumElements();

    // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
    SDValue OrigStepVec = DAG.getStepVector(DL, StepVecVT);
    SDValue UndefStep = DAG.getPOISON(WideVecVT);
    StepVec = DAG.getInsertSubvector(DL, UndefStep, OrigStepVec, 0);

    // Widen mask: pad with zeros.
    EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), BoolVT, WideNumElts);
    SDValue ZeroMask = DAG.getConstant(0, DL, WideMaskVT);
    Mask = DAG.getInsertSubvector(DL, ZeroMask, Mask, 0);
  } else if (TypeAction == TargetLowering::TypeSplitVector) {
    // The stepvector type would require splitting. Signal to the caller
    // that the operation should be split instead of expanded.
    return {Mask, SDValue()};
  } else {
    StepVec = DAG.getStepVector(DL, StepVecVT);
  }

  return {Mask, StepVec};
}

SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
                                                   SelectionDAG &DAG) const {
  SDLoc DL(N);
  auto [Mask, StepVec] = getLegalMaskAndStepVector(
      N->getOperand(0), /*ZeroIsPoison=*/true, DL, DAG);

  // If StepVec is empty, the stepvector would require splitting.
  // Split the operation instead and let it be recursively legalized.
  if (!StepVec) {
    EVT MaskVT = N->getOperand(0).getValueType();
    EVT ResVT = N->getValueType(0);

    // Split the mask
    auto [LoVT, HiVT] = DAG.GetSplitDestVTs(MaskVT);
    auto [MaskLo, MaskHi] = DAG.SplitVector(N->getOperand(0), DL);

    // Create split VECTOR_FIND_LAST_ACTIVE operations
    SDValue LoResult =
        DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, ResVT, MaskLo);
    SDValue HiResult =
        DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, DL, ResVT, MaskHi);

    // Check if any lane is active in the high mask.
    SDValue AnyHiActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, MaskHi);
    SDValue Cond = DAG.getBoolExtOrTrunc(
        AnyHiActive, DL,
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i1),
        MVT::i1);

    // Adjust HiResult by adding the number of elements in Lo
    SDValue LoNumElts =
        DAG.getElementCount(DL, ResVT, LoVT.getVectorElementCount());
    SDValue AdjustedHiResult =
        DAG.getNode(ISD::ADD, DL, ResVT, HiResult, LoNumElts);

    // Return: AnyHiActive ? AdjustedHiResult : LoResult;
    return DAG.getNode(ISD::SELECT, DL, ResVT, Cond, AdjustedHiResult,
                       LoResult);
  }

  EVT StepVecVT = StepVec.getValueType();
  EVT StepVT = StepVec.getValueType().getVectorElementType();

  // Zero out lanes with inactive elements, then find the highest remaining
  // value from the stepvector.
  SDValue Zeroes = DAG.getConstant(0, DL, StepVecVT);
  SDValue ActiveElts = DAG.getSelect(DL, StepVecVT, Mask, StepVec, Zeroes);
  SDValue HighestIdx = DAG.getNode(ISD::VECREDUCE_UMAX, DL, StepVT, ActiveElts);
  return DAG.getZExtOrTrunc(HighestIdx, DL, N->getValueType(0));
}

SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  SDValue SourceValue = N->getOperand(0);
  SDValue SinkValue = N->getOperand(1);
  SDValue EltSizeInBytes = N->getOperand(2);

  // Note: The lane offset is scalable if the mask is scalable.
  ElementCount LaneOffsetEC =
      ElementCount::get(N->getConstantOperandVal(3), VT.isScalableVT());

  EVT AddrVT = SourceValue->getValueType(0);
  bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;

  // Take the difference between the pointers and divided by the element size,
  // to see how many lanes separate them.
  SDValue Diff = DAG.getNode(ISD::SUB, DL, AddrVT, SinkValue, SourceValue);
  if (IsReadAfterWrite)
    Diff = DAG.getNode(ISD::ABS, DL, AddrVT, Diff);
  Diff = DAG.getNode(ISD::SDIV, DL, AddrVT, Diff, EltSizeInBytes);

  // The pointers do not alias if:
  //  * Diff <= 0 (WAR_MASK)
  //  * Diff == 0 (RAW_MASK)
  EVT CmpVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), AddrVT);
  SDValue Zero = DAG.getConstant(0, DL, AddrVT);
  SDValue Cmp = DAG.getSetCC(DL, CmpVT, Diff, Zero,
                             IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);

  // The pointers do not alias if:
  // Lane + LaneOffset < Diff (WAR/RAW_MASK)
  SDValue LaneOffset = DAG.getElementCount(DL, AddrVT, LaneOffsetEC);
  SDValue MaskN = DAG.getSelect(
      DL, AddrVT, Cmp,
      DAG.getConstant(APInt::getMaxValue(AddrVT.getScalarSizeInBits()), DL,
                      AddrVT),
      Diff);

  return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, VT, LaneOffset, MaskN);
}

SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
                                  bool IsNegative) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue Op = N->getOperand(0);

  // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
  if (N->getOpcode() == ISD::ABS_MIN_POISON &&
      isOperationLegalOrCustom(ISD::ABS, VT)) {
    SDValue AbsVal = DAG.getNode(ISD::ABS, dl, VT, Op);
    if (IsNegative)
      return DAG.getNegative(AbsVal, dl, VT);
    return AbsVal;
  }

  // abs(x) -> smax(x,sub(0,x))
  if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
      isOperationLegal(ISD::SMAX, VT)) {
    SDValue Zero = DAG.getConstant(0, dl, VT);
    Op = DAG.getFreeze(Op);
    return DAG.getNode(ISD::SMAX, dl, VT, Op,
                       DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
  }

  // abs(x) -> umin(x,sub(0,x))
  if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
      isOperationLegal(ISD::UMIN, VT)) {
    SDValue Zero = DAG.getConstant(0, dl, VT);
    Op = DAG.getFreeze(Op);
    return DAG.getNode(ISD::UMIN, dl, VT, Op,
                       DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
  }

  // 0 - abs(x) -> smin(x, sub(0,x))
  if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
      isOperationLegal(ISD::SMIN, VT)) {
    SDValue Zero = DAG.getConstant(0, dl, VT);
    Op = DAG.getFreeze(Op);
    return DAG.getNode(ISD::SMIN, dl, VT, Op,
                       DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
  }

  // Only expand vector types if we have the appropriate vector operations.
  if (VT.isVector() &&
      (!isOperationLegalOrCustom(ISD::SRA, VT) ||
       (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
       (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
       !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
    return SDValue();

  Op = DAG.getFreeze(Op);
  SDValue Shift = DAG.getNode(
      ISD::SRA, dl, VT, Op,
      DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));
  SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);

  // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
  if (!IsNegative)
    return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift);

  // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
  return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
}

SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  bool IsSigned = N->getOpcode() == ISD::ABDS;

  // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
  // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
  unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
  unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
  if (isOperationLegal(MaxOpc, VT) && isOperationLegal(MinOpc, VT)) {
    LHS = DAG.getFreeze(LHS);
    RHS = DAG.getFreeze(RHS);
    SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
    SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
    return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
  }

  // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
  if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) {
    LHS = DAG.getFreeze(LHS);
    RHS = DAG.getFreeze(RHS);
    return DAG.getNode(ISD::OR, dl, VT,
                       DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS),
                       DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS));
  }

  // If the subtract doesn't overflow then just use abs(sub())
  bool IsNonNegative = DAG.SignBitIsZero(LHS) && DAG.SignBitIsZero(RHS);

  if (DAG.willNotOverflowSub(IsSigned || IsNonNegative, LHS, RHS))
    return DAG.getNode(ISD::ABS, dl, VT,
                       DAG.getNode(ISD::SUB, dl, VT, LHS, RHS));

  if (DAG.willNotOverflowSub(IsSigned || IsNonNegative, RHS, LHS))
    return DAG.getNode(ISD::ABS, dl, VT,
                       DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));

  EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
  LHS = DAG.getFreeze(LHS);
  RHS = DAG.getFreeze(RHS);
  SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC);

  // Branchless expansion iff cmp result is allbits:
  // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
  // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
  if (CCVT == VT && getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
    SDValue Diff = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
    SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Diff, Cmp);
    return DAG.getNode(ISD::SUB, dl, VT, Cmp, Xor);
  }

  // Similar to the branchless expansion, if we don't prefer selects, use the
  // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
  // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
  // rhs), uof(lhs, rhs)), uof(lhs, rhs))
  if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
      !preferSelectsOverBooleanArithmetic(VT)) {
    SDValue USubO =
        DAG.getNode(ISD::USUBO, dl, DAG.getVTList(VT, MVT::i1), {LHS, RHS});
    SDValue Cmp = DAG.getNode(ISD::SIGN_EXTEND, dl, VT, USubO.getValue(1));
    SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, USubO.getValue(0), Cmp);
    return DAG.getNode(ISD::SUB, dl, VT, Xor, Cmp);
  }

  // FIXME: Should really try to split the vector in case it's legal on a
  // subvector.
  if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
    return DAG.UnrollVectorOp(N);

  // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
  // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
  return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS),
                       DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
}

SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);

  unsigned Opc = N->getOpcode();
  bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
  bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
  unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
  unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
  unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
  unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
  assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
          Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
         "Unknown AVG node");

  // If the operands are already extended, we can add+shift.
  bool IsExt =
      (IsSigned && DAG.ComputeNumSignBits(LHS) >= 2 &&
       DAG.ComputeNumSignBits(RHS) >= 2) ||
      (!IsSigned && DAG.computeKnownBits(LHS).countMinLeadingZeros() >= 1 &&
       DAG.computeKnownBits(RHS).countMinLeadingZeros() >= 1);
  if (IsExt) {
    SDValue Sum = DAG.getNode(ISD::ADD, dl, VT, LHS, RHS);
    if (!IsFloor)
      Sum = DAG.getNode(ISD::ADD, dl, VT, Sum, DAG.getConstant(1, dl, VT));
    return DAG.getNode(ShiftOpc, dl, VT, Sum,
                       DAG.getShiftAmountConstant(1, VT, dl));
  }

  // For scalars, see if we can efficiently extend/truncate to use add+shift.
  if (VT.isScalarInteger()) {
    EVT ExtVT = VT.widenIntegerElementType(*DAG.getContext());
    if (isTypeLegal(ExtVT) && isTruncateFree(ExtVT, VT)) {
      LHS = DAG.getNode(ExtOpc, dl, ExtVT, LHS);
      RHS = DAG.getNode(ExtOpc, dl, ExtVT, RHS);
      SDValue Avg = DAG.getNode(ISD::ADD, dl, ExtVT, LHS, RHS);
      if (!IsFloor)
        Avg = DAG.getNode(ISD::ADD, dl, ExtVT, Avg,
                          DAG.getConstant(1, dl, ExtVT));
      // Just use SRL as we will be truncating away the extended sign bits.
      Avg = DAG.getNode(ISD::SRL, dl, ExtVT, Avg,
                        DAG.getShiftAmountConstant(1, ExtVT, dl));
      return DAG.getNode(ISD::TRUNCATE, dl, VT, Avg);
    }
  }

  // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
  if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
      isOperationLegalOrCustom(
          ISD::UADDO, getLegalTypeToTransformTo(*DAG.getContext(), VT))) {
    SDValue UAddWithOverflow =
        DAG.getNode(ISD::UADDO, dl, DAG.getVTList(VT, MVT::i1), {RHS, LHS});

    SDValue Sum = UAddWithOverflow.getValue(0);
    SDValue Overflow = UAddWithOverflow.getValue(1);

    // Right shift the sum by 1
    SDValue LShrVal = DAG.getNode(ISD::SRL, dl, VT, Sum,
                                  DAG.getShiftAmountConstant(1, VT, dl));

    SDValue ZeroExtOverflow = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Overflow);
    SDValue OverflowShl = DAG.getNode(
        ISD::SHL, dl, VT, ZeroExtOverflow,
        DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl));

    return DAG.getNode(ISD::OR, dl, VT, LShrVal, OverflowShl);
  }

  // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
  // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
  // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
  // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
  LHS = DAG.getFreeze(LHS);
  RHS = DAG.getFreeze(RHS);
  SDValue Sign = DAG.getNode(SignOpc, dl, VT, LHS, RHS);
  SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
  SDValue Shift =
      DAG.getNode(ShiftOpc, dl, VT, Xor, DAG.getShiftAmountConstant(1, VT, dl));
  return DAG.getNode(SumOpc, dl, VT, Sign, Shift);
}

SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue Op = N->getOperand(0);

  if (!VT.isSimple())
    return SDValue();

  EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
  default:
    return SDValue();
  case MVT::i16:
    // Use a rotate by 8. This can be further expanded if necessary.
    return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
  case MVT::i32:
    // This is meant for ARM specifically, which has ROTR but no ROTL.
    //   t = x ^ rotr(x, 16)
    //   t = bic(t, 0x00ff0000)
    //   t = lshr(t, 8)
    //   x = t ^ rotr(x, 8)
    if (isOperationLegalOrCustom(ISD::ROTR, VT)) {
      SDValue Rotr16 =
          DAG.getNode(ISD::ROTR, dl, VT, Op, DAG.getConstant(16, dl, SHVT));
      SDValue Tmp = DAG.getNode(ISD::XOR, dl, VT, Op, Rotr16);
      Tmp = DAG.getNode(ISD::AND, dl, VT, Tmp,
                        DAG.getConstant(0xFF00FFFF, dl, VT));
      Tmp = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(8, dl, SHVT));
      SDValue Rotr8 =
          DAG.getNode(ISD::ROTR, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
      return DAG.getNode(ISD::XOR, dl, VT, Tmp, Rotr8);
    }
    Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op,
                       DAG.getConstant(0xFF00, dl, VT));
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
  case MVT::i64:
    Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op,
                       DAG.getConstant(255ULL<<8, dl, VT));
    Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT));
    Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op,
                       DAG.getConstant(255ULL<<16, dl, VT));
    Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT));
    Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op,
                       DAG.getConstant(255ULL<<24, dl, VT));
    Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT));
    Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
    Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
                       DAG.getConstant(255ULL<<24, dl, VT));
    Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
                       DAG.getConstant(255ULL<<16, dl, VT));
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
                       DAG.getConstant(255ULL<<8, dl, VT));
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
    Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
    return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
  }
}

SDValue TargetLowering::expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue Op = N->getOperand(0);
  SDValue Mask = N->getOperand(1);
  SDValue EVL = N->getOperand(2);

  if (!VT.isSimple())
    return SDValue();

  EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
  default:
    return SDValue();
  case MVT::i16:
    Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL);
  case MVT::i32:
    Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
                       DAG.getConstant(0xFF00, dl, VT), Mask, EVL);
    Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
                       Mask, EVL);
    Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
    return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
  case MVT::i64:
    Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT),
                       Mask, EVL);
    Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
                       DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL);
    Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT),
                       Mask, EVL);
    Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
                       DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL);
    Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT),
                       Mask, EVL);
    Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op,
                       DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL);
    Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT),
                       Mask, EVL);
    Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4,
                       DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3,
                       DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
                       DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL);
    Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT),
                       Mask, EVL);
    Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL);
    Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL);
    Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL);
    Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL);
    Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL);
    return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL);
  }
}

SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue Op = N->getOperand(0);
  EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
  unsigned Sz = VT.getScalarSizeInBits();

  SDValue Tmp, Tmp2, Tmp3;

  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
  // and finally the i1 pairs.
  // TODO: We can easily support i4/i2 legal types if any target ever does.
  if (Sz >= 8 && isPowerOf2_32(Sz)) {
    // Create the masks - repeating the pattern every byte.
    APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
    APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
    APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));

    // BSWAP if the type is wider than a single byte.
    Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);

    // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

    // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

    // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
    return Tmp;
  }

  Tmp = DAG.getConstant(0, dl, VT);
  for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
    if (I < J)
      Tmp2 =
          DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
    else
      Tmp2 =
          DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));

    APInt Shift = APInt::getOneBitSet(Sz, J);
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
  }

  return Tmp;
}

SDValue TargetLowering::expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
  assert(N->getOpcode() == ISD::VP_BITREVERSE);

  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  SDValue Op = N->getOperand(0);
  SDValue Mask = N->getOperand(1);
  SDValue EVL = N->getOperand(2);
  EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
  unsigned Sz = VT.getScalarSizeInBits();

  SDValue Tmp, Tmp2, Tmp3;

  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
  // and finally the i1 pairs.
  // TODO: We can easily support i4/i2 legal types if any target ever does.
  if (Sz >= 8 && isPowerOf2_32(Sz)) {
    // Create the masks - repeating the pattern every byte.
    APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
    APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
    APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));

    // BSWAP if the type is wider than a single byte.
    Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op);

    // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
                       DAG.getConstant(Mask4, dl, VT), Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT),
                       Mask, EVL);
    Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);

    // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
                       DAG.getConstant(Mask2, dl, VT), Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT),
                       Mask, EVL);
    Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);

    // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
    Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT),
                       Mask, EVL);
    Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2,
                       DAG.getConstant(Mask1, dl, VT), Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT),
                       Mask, EVL);
    Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT),
                       Mask, EVL);
    Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL);
    return Tmp;
  }
  return SDValue();
}

std::pair<SDValue, SDValue>
TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
                                    SelectionDAG &DAG) const {
  SDLoc SL(LD);
  SDValue Chain = LD->getChain();
  SDValue BasePTR = LD->getBasePtr();
  EVT SrcVT = LD->getMemoryVT();
  EVT DstVT = LD->getValueType(0);
  ISD::LoadExtType ExtType = LD->getExtensionType();

  if (SrcVT.isScalableVector())
    report_fatal_error("Cannot scalarize scalable vector loads");

  unsigned NumElem = SrcVT.getVectorNumElements();

  EVT SrcEltVT = SrcVT.getScalarType();
  EVT DstEltVT = DstVT.getScalarType();

  // A vector must always be stored in memory as-is, i.e. without any padding
  // between the elements, since various code depend on it, e.g. in the
  // handling of a bitcast of a vector type to int, which may be done with a
  // vector store followed by an integer load. A vector that does not have
  // elements that are byte-sized must therefore be stored as an integer
  // built out of the extracted vector elements.
  if (!SrcEltVT.isByteSized()) {
    unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
    EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);

    unsigned NumSrcBits = SrcVT.getSizeInBits();
    EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);

    unsigned SrcEltBits = SrcEltVT.getSizeInBits();
    SDValue SrcEltBitMask = DAG.getConstant(
        APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);

    // Load the whole vector and avoid masking off the top bits as it makes
    // the codegen worse.
    SDValue Load =
        DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
                       LD->getPointerInfo(), SrcIntVT, LD->getBaseAlign(),
                       LD->getMemOperand()->getFlags(), LD->getAAInfo());

    SmallVector<SDValue, 8> Vals;
    for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
      unsigned ShiftIntoIdx =
          (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
      SDValue ShiftAmount = DAG.getShiftAmountConstant(
          ShiftIntoIdx * SrcEltVT.getSizeInBits(), LoadVT, SL);
      SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
      SDValue Elt =
          DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
      SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);

      if (ExtType != ISD::NON_EXTLOAD) {
        unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
        Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
      }

      Vals.push_back(Scalar);
    }

    SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
    return std::make_pair(Value, Load.getValue(1));
  }

  unsigned Stride = SrcEltVT.getSizeInBits() / 8;
  assert(SrcEltVT.isByteSized());

  SmallVector<SDValue, 8> Vals;
  SmallVector<SDValue, 8> LoadChains;

  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
    SDValue ScalarLoad = DAG.getExtLoad(
        ExtType, SL, DstEltVT, Chain, BasePTR,
        LD->getPointerInfo().getWithOffset(Idx * Stride), SrcEltVT,
        LD->getBaseAlign(), LD->getMemOperand()->getFlags(), LD->getAAInfo());

    BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::getFixed(Stride));

    Vals.push_back(ScalarLoad.getValue(0));
    LoadChains.push_back(ScalarLoad.getValue(1));
  }

  SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
  SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

  return std::make_pair(Value, NewChain);
}

SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
                                             SelectionDAG &DAG) const {
  SDLoc SL(ST);

  SDValue Chain = ST->getChain();
  SDValue BasePtr = ST->getBasePtr();
  SDValue Value = ST->getValue();
  EVT StVT = ST->getMemoryVT();

  if (StVT.isScalableVector())
    report_fatal_error("Cannot scalarize scalable vector stores");

  // The type of the data we want to save
  EVT RegVT = Value.getValueType();
  EVT RegSclVT = RegVT.getScalarType();

  // The type of data as saved in memory.
  EVT MemSclVT = StVT.getScalarType();

  unsigned NumElem = StVT.getVectorNumElements();

  // A vector must always be stored in memory as-is, i.e. without any padding
  // between the elements, since various code depend on it, e.g. in the
  // handling of a bitcast of a vector type to int, which may be done with a
  // vector store followed by an integer load. A vector that does not have
  // elements that are byte-sized must therefore be stored as an integer
  // built out of the extracted vector elements.
  if (!MemSclVT.isByteSized()) {
    unsigned NumBits = StVT.getSizeInBits();
    EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);

    SDValue CurrVal = DAG.getConstant(0, SL, IntVT);

    for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
      SDValue Elt = DAG.getExtractVectorElt(SL, RegSclVT, Value, Idx);
      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
      SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
      unsigned ShiftIntoIdx =
          (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
      SDValue ShiftAmount =
          DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
      SDValue ShiftedElt =
          DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
      CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
    }

    return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
                        ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
                        ST->getAAInfo());
  }

  // Store Stride in bytes
  unsigned Stride = MemSclVT.getSizeInBits() / 8;
  assert(Stride && "Zero stride!");
  // Extract each of the elements from the original vector and save them into
  // memory individually.
  SmallVector<SDValue, 8> Stores;
  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
    SDValue Elt = DAG.getExtractVectorElt(SL, RegSclVT, Value, Idx);

    SDValue Ptr =
        DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Idx * Stride));

    // This scalar TruncStore may be illegal, but we legalize it later.
    SDValue Store = DAG.getTruncStore(
        Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
        MemSclVT, ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
        ST->getAAInfo());

    Stores.push_back(Store);
  }

  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
}

std::pair<SDValue, SDValue>
TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
  assert(LD->getAddressingMode() == ISD::UNINDEXED &&
         "unaligned indexed loads not implemented!");
  SDValue Chain = LD->getChain();
  SDValue Ptr = LD->getBasePtr();
  EVT VT = LD->getValueType(0);
  EVT LoadedVT = LD->getMemoryVT();
  SDLoc dl(LD);
  auto &MF = DAG.getMachineFunction();

  if (VT.isFloatingPoint() || VT.isVector()) {
    EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
    if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
      if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
          LoadedVT.isVector()) {
        // Scalarize the load and let the individual components be handled.
        return scalarizeVectorLoad(LD, DAG);
      }

      // Expand to a (misaligned) integer load of the same size,
      // then bitconvert to floating point or vector.
      SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
                                    LD->getMemOperand());
      SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
      if (LoadedVT != VT)
        Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
                             ISD::ANY_EXTEND, dl, VT, Result);

      return std::make_pair(Result, newLoad.getValue(1));
    }

    // Copy the value to a (aligned) stack slot using (unaligned) integer
    // loads and stores, then do a (aligned) load from the stack slot.
    MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
    unsigned LoadedBytes = LoadedVT.getStoreSize();
    unsigned RegBytes = RegVT.getSizeInBits() / 8;
    unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;

    // Make sure the stack slot is also aligned for the register type.
    SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
    auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
    SmallVector<SDValue, 8> Stores;
    SDValue StackPtr = StackBase;
    unsigned Offset = 0;

    EVT PtrVT = Ptr.getValueType();
    EVT StackPtrVT = StackPtr.getValueType();

    SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
    SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

    // Do all but one copies using the full register width.
    for (unsigned i = 1; i < NumRegs; i++) {
      // Load one integer register's worth from the original location.
      SDValue Load = DAG.getLoad(
          RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
          LD->getBaseAlign(), LD->getMemOperand()->getFlags(), LD->getAAInfo());
      // Follow the load with a store to the stack slot.  Remember the store.
      Stores.push_back(DAG.getStore(
          Load.getValue(1), dl, Load, StackPtr,
          MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
      // Increment the pointers.
      Offset += RegBytes;

      Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
      StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
    }

    // The last copy may be partial.  Do an extending load.
    EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
                                  8 * (LoadedBytes - Offset));
    SDValue Load = DAG.getExtLoad(
        ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
        LD->getPointerInfo().getWithOffset(Offset), MemVT, LD->getBaseAlign(),
        LD->getMemOperand()->getFlags(), LD->getAAInfo());
    // Follow the load with a store to the stack slot.  Remember the store.
    // On big-endian machines this requires a truncating store to ensure
    // that the bits end up in the right place.
    Stores.push_back(DAG.getTruncStore(
        Load.getValue(1), dl, Load, StackPtr,
        MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));

    // The order of the stores doesn't matter - say it with a TokenFactor.
    SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

    // Finally, perform the original load only redirected to the stack slot.
    Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
                          MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
                          LoadedVT);

    // Callers expect a MERGE_VALUES node.
    return std::make_pair(Load, TF);
  }

  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
         "Unaligned load of unsupported type.");

  // Compute the new VT that is half the size of the old one.  This is an
  // integer MVT.
  unsigned NumBits = LoadedVT.getSizeInBits();
  EVT NewLoadedVT;
  NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
  NumBits >>= 1;

  Align Alignment = LD->getBaseAlign();
  unsigned IncrementSize = NumBits / 8;
  ISD::LoadExtType HiExtType = LD->getExtensionType();

  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
  if (HiExtType == ISD::NON_EXTLOAD)
    HiExtType = ISD::ZEXTLOAD;

  // Load the value in two parts
  SDValue Lo, Hi;
  if (DAG.getDataLayout().isLittleEndian()) {
    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
                        NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                        LD->getAAInfo());

    Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
    Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
                        LD->getPointerInfo().getWithOffset(IncrementSize),
                        NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                        LD->getAAInfo());
  } else {
    Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
                        NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                        LD->getAAInfo());

    Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
                        LD->getPointerInfo().getWithOffset(IncrementSize),
                        NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                        LD->getAAInfo());
  }

  // aggregate the two parts
  SDValue ShiftAmount = DAG.getShiftAmountConstant(NumBits, VT, dl);
  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
  Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);

  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
                             Hi.getValue(1));

  return std::make_pair(Result, TF);
}

SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
                                             SelectionDAG &DAG) const {
  assert(ST->getAddressingMode() == ISD::UNINDEXED &&
         "unaligned indexed stores not implemented!");
  SDValue Chain = ST->getChain();
  SDValue Ptr = ST->getBasePtr();
  SDValue Val = ST->getValue();
  EVT VT = Val.getValueType();
  Align Alignment = ST->getBaseAlign();
  auto &MF = DAG.getMachineFunction();
  EVT StoreMemVT = ST->getMemoryVT();

  SDLoc dl(ST);
  if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
    EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
    if (isTypeLegal(intVT)) {
      if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
          StoreMemVT.isVector()) {
        // Scalarize the store and let the individual components be handled.
        SDValue Result = scalarizeVectorStore(ST, DAG);
        return Result;
      }
      // Expand to a bitconvert of the value to the integer type of the
      // same size, then a (misaligned) int store.
      // FIXME: Does not handle truncating floating point stores!
      SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
      Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
                            Alignment, ST->getMemOperand()->getFlags());
      return Result;
    }
    // Do a (aligned) store to a stack slot, then copy from the stack slot
    // to the final destination using (unaligned) integer loads and stores.
    MVT RegVT = getRegisterType(
        *DAG.getContext(),
        EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
    EVT PtrVT = Ptr.getValueType();
    unsigned StoredBytes = StoreMemVT.getStoreSize();
    unsigned RegBytes = RegVT.getSizeInBits() / 8;
    unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;

    // Make sure the stack slot is also aligned for the register type.
    SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
    auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

    // Perform the original store, only redirected to the stack slot.
    SDValue Store = DAG.getTruncStore(
        Chain, dl, Val, StackPtr,
        MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);

    EVT StackPtrVT = StackPtr.getValueType();

    SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
    SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
    SmallVector<SDValue, 8> Stores;
    unsigned Offset = 0;

    // Do all but one copies using the full register width.
    for (unsigned i = 1; i < NumRegs; i++) {
      // Load one integer register's worth from the stack slot.
      SDValue Load = DAG.getLoad(
          RegVT, dl, Store, StackPtr,
          MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
      // Store it to the final location.  Remember the store.
      Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
                                    ST->getPointerInfo().getWithOffset(Offset),
                                    ST->getBaseAlign(),
                                    ST->getMemOperand()->getFlags()));
      // Increment the pointers.
      Offset += RegBytes;
      StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
      Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
    }

    // The last store may be partial.  Do a truncating store.  On big-endian
    // machines this requires an extending load from the stack slot to ensure
    // that the bits are in the right place.
    EVT LoadMemVT =
        EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));

    // Load from the stack slot.
    SDValue Load = DAG.getExtLoad(
        ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
        MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);

    Stores.push_back(DAG.getTruncStore(
        Load.getValue(1), dl, Load, Ptr,
        ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
        ST->getBaseAlign(), ST->getMemOperand()->getFlags(), ST->getAAInfo()));
    // The order of the stores doesn't matter - say it with a TokenFactor.
    SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
    return Result;
  }

  assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
         "Unaligned store of unknown type.");
  // Get the half-size VT
  EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
  unsigned NumBits = NewStoredVT.getFixedSizeInBits();
  unsigned IncrementSize = NumBits / 8;

  // Divide the stored value in two parts.
  SDValue ShiftAmount =
      DAG.getShiftAmountConstant(NumBits, Val.getValueType(), dl);
  SDValue Lo = Val;
  // If Val is a constant, replace the upper bits with 0. The SRL will constant
  // fold and not use the upper bits. A smaller constant may be easier to
  // materialize.
  if (auto *C = dyn_cast<ConstantSDNode>(Lo); C && !C->isOpaque())
    Lo = DAG.getNode(
        ISD::AND, dl, VT, Lo,
        DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl,
                        VT));
  SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);

  // Store the two parts
  SDValue Store1, Store2;
  Store1 = DAG.getTruncStore(Chain, dl,
                             DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
                             Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
                             ST->getMemOperand()->getFlags());

  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));
  Store2 = DAG.getTruncStore(
      Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
      ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
      ST->getMemOperand()->getFlags(), ST->getAAInfo());

  SDValue Result =
      DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
  return Result;
}

SDValue
TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
                                       const SDLoc &DL, EVT DataVT,
                                       SelectionDAG &DAG,
                                       bool IsCompressedMemory) const {
  SDValue Increment;
  EVT AddrVT = Addr.getValueType();
  EVT MaskVT = Mask.getValueType();
  assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
         "Incompatible types of Data and Mask");
  if (IsCompressedMemory) {
    // Incrementing the pointer according to number of '1's in the mask.
    if (DataVT.isScalableVector()) {
      EVT MaskExtVT = MaskVT.changeElementType(*DAG.getContext(), MVT::i32);
      SDValue MaskExt = DAG.getNode(ISD::ZERO_EXTEND, DL, MaskExtVT, Mask);
      Increment = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, MaskExt);
    } else {
      EVT MaskIntVT =
          EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
      SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
      if (MaskIntVT.getSizeInBits() < 32) {
        MaskInIntReg =
            DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
        MaskIntVT = MVT::i32;
      }
      Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
    }
    // Scale is an element size in bytes.
    SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
                                    AddrVT);
    Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
    Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
  } else
    Increment = DAG.getTypeSize(DL, AddrVT, DataVT.getStoreSize());

  return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
}

static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
                                       EVT VecVT, const SDLoc &dl,
                                       ElementCount SubEC) {
  assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
         "Cannot index a scalable vector within a fixed-width vector");

  unsigned NElts = VecVT.getVectorMinNumElements();
  unsigned NumSubElts = SubEC.getKnownMinValue();
  EVT IdxVT = Idx.getValueType();

  if (VecVT.isScalableVector() && !SubEC.isScalable()) {
    // If this is a constant index and we know the value plus the number of the
    // elements in the subvector minus one is less than the minimum number of
    // elements then it's safe to return Idx.
    if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
      if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
        return Idx;
    SDValue VS =
        DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
    unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
    SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,
                              DAG.getConstant(NumSubElts, dl, IdxVT));
    return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
  }
  if (isPowerOf2_32(NElts) && NumSubElts == 1) {
    APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
    return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
                       DAG.getConstant(Imm, dl, IdxVT));
  }
  unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
                     DAG.getConstant(MaxIndex, dl, IdxVT));
}

SDValue
TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
                                        EVT VecVT, SDValue Index,
                                        const SDNodeFlags PtrArithFlags) const {
  return getVectorSubVecPointer(
      DAG, VecPtr, VecVT,
      EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),
      Index, PtrArithFlags);
}

SDValue
TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
                                       EVT VecVT, EVT SubVecVT, SDValue Index,
                                       const SDNodeFlags PtrArithFlags) const {
  SDLoc dl(Index);
  // Make sure the index type is big enough to compute in.
  Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());

  EVT EltVT = VecVT.getVectorElementType();

  // Calculate the element offset and add it to the pointer.
  unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
  assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
         "Converting bits to bytes lost precision");
  assert(SubVecVT.getVectorElementType() == EltVT &&
         "Sub-vector must be a vector with matching element type");
  Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
                                  SubVecVT.getVectorElementCount());

  EVT IdxVT = Index.getValueType();
  if (SubVecVT.isScalableVector())
    Index =
        DAG.getNode(ISD::MUL, dl, IdxVT, Index,
                    DAG.getVScale(dl, IdxVT, APInt(IdxVT.getSizeInBits(), 1)));

  Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
                      DAG.getConstant(EltSize, dl, IdxVT));
  return DAG.getMemBasePlusOffset(VecPtr, Index, dl, PtrArithFlags);
}

//===----------------------------------------------------------------------===//
// Implementation of Emulated TLS Model
//===----------------------------------------------------------------------===//

SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
                                                SelectionDAG &DAG) const {
  // Access to address of TLS varialbe xyz is lowered to a function call:
  //   __emutls_get_address( address of global variable named "__emutls_v.xyz" )
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  PointerType *VoidPtrType = PointerType::get(*DAG.getContext(), 0);
  SDLoc dl(GA);

  ArgListTy Args;
  const GlobalValue *GV =
      cast<GlobalValue>(GA->getGlobal()->stripPointerCastsAndAliases());
  SmallString<32> NameString("__emutls_v.");
  NameString += GV->getName();
  StringRef EmuTlsVarName(NameString);
  const GlobalVariable *EmuTlsVar =
      GV->getParent()->getNamedGlobal(EmuTlsVarName);
  assert(EmuTlsVar && "Cannot find EmuTlsVar ");
  Args.emplace_back(DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT), VoidPtrType);

  SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);

  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
  CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);

  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
  // At last for X86 targets, maybe good for other targets too?
  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
  MFI.setAdjustsStack(true); // Is this only for X86 target?
  MFI.setHasCalls(true);

  assert((GA->getOffset() == 0) &&
         "Emulated TLS must have zero offset in GlobalAddressSDNode");
  return CallResult.first;
}

SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
                                                SelectionDAG &DAG) const {
  assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
  if (!isCtlzFast())
    return SDValue();
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
  SDLoc dl(Op);
  if (isNullConstant(Op.getOperand(1)) && CC == ISD::SETEQ) {
    EVT VT = Op.getOperand(0).getValueType();
    SDValue Zext = Op.getOperand(0);
    if (VT.bitsLT(MVT::i32)) {
      VT = MVT::i32;
      Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
    }
    unsigned Log2b = Log2_32(VT.getSizeInBits());
    SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
    SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
                              DAG.getConstant(Log2b, dl, MVT::i32));
    return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
  }
  return SDValue();
}

SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
  SDValue Op0 = Node->getOperand(0);
  SDValue Op1 = Node->getOperand(1);
  EVT VT = Op0.getValueType();
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  unsigned Opcode = Node->getOpcode();
  SDLoc DL(Node);

  // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
  unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(Opcode);
  if (isOperationLegal(AltOpcode, VT) && DAG.SignBitIsZero(Op0) &&
      DAG.SignBitIsZero(Op1))
    return DAG.getNode(AltOpcode, DL, VT, Op0, Op1);

  // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
  if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT &&
      getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
    Op0 = DAG.getFreeze(Op0);
    SDValue Zero = DAG.getConstant(0, DL, VT);
    return DAG.getNode(ISD::SUB, DL, VT, Op0,
                       DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ));
  }

  // umin(x,y) -> sub(x,usubsat(x,y))
  // TODO: Missing freeze(Op0)?
  if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
      isOperationLegal(ISD::USUBSAT, VT)) {
    return DAG.getNode(ISD::SUB, DL, VT, Op0,
                       DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
  }

  // umax(x,y) -> add(x,usubsat(y,x))
  // TODO: Missing freeze(Op0)?
  if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
      isOperationLegal(ISD::USUBSAT, VT)) {
    return DAG.getNode(ISD::ADD, DL, VT, Op0,
                       DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
  }

  // FIXME: Should really try to split the vector in case it's legal on a
  // subvector.
  if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
    return DAG.UnrollVectorOp(Node);

  // Attempt to find an existing SETCC node that we can reuse.
  // TODO: Do we need a generic doesSETCCNodeExist?
  // TODO: Missing freeze(Op0)/freeze(Op1)?
  auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
                         ISD::CondCode PrefCommuteCC,
                         ISD::CondCode AltCommuteCC) {
    SDVTList BoolVTList = DAG.getVTList(BoolVT);
    for (ISD::CondCode CC : {PrefCC, AltCC}) {
      if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
                            {Op0, Op1, DAG.getCondCode(CC)})) {
        SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
        return DAG.getSelect(DL, VT, Cond, Op0, Op1);
      }
    }
    for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
      if (DAG.doesNodeExist(ISD::SETCC, BoolVTList,
                            {Op0, Op1, DAG.getCondCode(CC)})) {
        SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, CC);
        return DAG.getSelect(DL, VT, Cond, Op1, Op0);
      }
    }
    SDValue Cond = DAG.getSetCC(DL, BoolVT, Op0, Op1, PrefCC);
    return DAG.getSelect(DL, VT, Cond, Op0, Op1);
  };

  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
  //                      -> Y = (A < B) ? B : A
  //                      -> Y = (A >= B) ? A : B
  //                      -> Y = (A <= B) ? B : A
  switch (Opcode) {
  case ISD::SMAX:
    return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
  case ISD::SMIN:
    return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
  case ISD::UMAX:
    return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
  case ISD::UMIN:
    return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
  }

  llvm_unreachable("How did we get here?");
}

SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
  unsigned Opcode = Node->getOpcode();
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  EVT VT = LHS.getValueType();
  SDLoc dl(Node);

  assert(VT == RHS.getValueType() && "Expected operands to be the same type");
  assert(VT.isInteger() && "Expected operands to be integers");

  // usub.sat(a, b) -> umax(a, b) - b
  if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
    SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
    return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
  }

  // usub.sat(a, 1) -> sub(a, zext(a != 0))
  // Prefer this on targets without legal/cost-effective overflow-carry nodes.
  if (Opcode == ISD::USUBSAT && isOneOrOneSplat(RHS) &&
      !isOperationLegalOrCustom(ISD::USUBO_CARRY, VT)) {
    LHS = DAG.getFreeze(LHS);
    SDValue Zero = DAG.getConstant(0, dl, VT);
    EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
    SDValue IsNonZero = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETNE);
    SDValue Subtrahend = DAG.getBoolExtOrTrunc(IsNonZero, dl, VT, BoolVT);
    Subtrahend =
        DAG.getNode(ISD::AND, dl, VT, Subtrahend, DAG.getConstant(1, dl, VT));
    return DAG.getNode(ISD::SUB, dl, VT, LHS, Subtrahend);
  }

  // uadd.sat(a, b) -> umin(a, ~b) + b
  if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
    SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
    SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
    return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
  }

  unsigned OverflowOp;
  switch (Opcode) {
  case ISD::SADDSAT:
    OverflowOp = ISD::SADDO;
    break;
  case ISD::UADDSAT:
    OverflowOp = ISD::UADDO;
    break;
  case ISD::SSUBSAT:
    OverflowOp = ISD::SSUBO;
    break;
  case ISD::USUBSAT:
    OverflowOp = ISD::USUBO;
    break;
  default:
    llvm_unreachable("Expected method to receive signed or unsigned saturation "
                     "addition or subtraction node.");
  }

  // FIXME: Should really try to split the vector in case it's legal on a
  // subvector.
  if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
    return DAG.UnrollVectorOp(Node);

  unsigned BitWidth = LHS.getScalarValueSizeInBits();
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
  SDValue SumDiff = Result.getValue(0);
  SDValue Overflow = Result.getValue(1);
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);

  if (Opcode == ISD::UADDSAT) {
    if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
      // (LHS + RHS) | OverflowMask
      SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
      return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
    }
    // Overflow ? 0xffff.... : (LHS + RHS)
    return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
  }

  if (Opcode == ISD::USUBSAT) {
    if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
      // (LHS - RHS) & ~OverflowMask
      SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
      SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
      return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
    }
    // Overflow ? 0 : (LHS - RHS)
    return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
  }

  assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
         "Expected signed saturating add/sub opcode");

  const APInt MinVal = APInt::getSignedMinValue(BitWidth);
  const APInt MaxVal = APInt::getSignedMaxValue(BitWidth);

  KnownBits KnownLHS = DAG.computeKnownBits(LHS);
  KnownBits KnownRHS = DAG.computeKnownBits(RHS);

  // If either of the operand signs are known, then they are guaranteed to
  // only saturate in one direction. If non-negative they will saturate
  // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
  //
  // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
  // sign of 'y' has to be flipped.

  bool LHSIsNonNegative = KnownLHS.isNonNegative();
  bool RHSIsNonNegative =
      Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
  if (LHSIsNonNegative || RHSIsNonNegative) {
    SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
    return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff);
  }

  bool LHSIsNegative = KnownLHS.isNegative();
  bool RHSIsNegative =
      Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
  if (LHSIsNegative || RHSIsNegative) {
    SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
    return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff);
  }

  // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
  SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,
                              DAG.getConstant(BitWidth - 1, dl, VT));
  Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);
  return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
}

SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
  unsigned Opcode = Node->getOpcode();
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  EVT VT = LHS.getValueType();
  EVT ResVT = Node->getValueType(0);
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDLoc dl(Node);

  auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
  auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
  SDValue IsLT = DAG.getSetCC(dl, BoolVT, LHS, RHS, LTPredicate);
  SDValue IsGT = DAG.getSetCC(dl, BoolVT, LHS, RHS, GTPredicate);

  // We can't perform arithmetic on i1 values. Extending them would
  // probably result in worse codegen, so let's just use two selects instead.
  // Some targets are also just better off using selects rather than subtraction
  // because one of the conditions can be merged with one of the selects.
  // And finally, if we don't know the contents of high bits of a boolean value
  // we can't perform any arithmetic either.
  if (preferSelectsOverBooleanArithmetic(VT) ||
      BoolVT.getScalarSizeInBits() == 1 ||
      getBooleanContents(BoolVT) == UndefinedBooleanContent) {
    SDValue SelectZeroOrOne =
        DAG.getSelect(dl, ResVT, IsGT, DAG.getConstant(1, dl, ResVT),
                      DAG.getConstant(0, dl, ResVT));
    return DAG.getSelect(dl, ResVT, IsLT, DAG.getAllOnesConstant(dl, ResVT),
                         SelectZeroOrOne);
  }

  if (getBooleanContents(BoolVT) == ZeroOrNegativeOneBooleanContent)
    std::swap(IsGT, IsLT);
  return DAG.getSExtOrTrunc(DAG.getNode(ISD::SUB, dl, BoolVT, IsGT, IsLT), dl,
                            ResVT);
}

SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
  unsigned Opcode = Node->getOpcode();
  bool IsSigned = Opcode == ISD::SSHLSAT;
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  EVT VT = LHS.getValueType();
  SDLoc dl(Node);

  assert((Node->getOpcode() == ISD::SSHLSAT ||
          Node->getOpcode() == ISD::USHLSAT) &&
         "Expected a SHLSAT opcode");
  assert(VT.isInteger() && "Expected operands to be integers");

  if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
    return DAG.UnrollVectorOp(Node);

  // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.

  unsigned BW = VT.getScalarSizeInBits();
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
  SDValue Orig =
      DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);

  SDValue SatVal;
  if (IsSigned) {
    SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
    SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
    SDValue Cond =
        DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT);
    SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax);
  } else {
    SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
  }
  SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE);
  return DAG.getSelect(dl, VT, Cond, SatVal, Result);
}

void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
                                         bool Signed, SDValue &Lo, SDValue &Hi,
                                         SDValue LHS, SDValue RHS,
                                         SDValue HiLHS, SDValue HiRHS) const {
  EVT VT = LHS.getValueType();
  assert(RHS.getValueType() == VT && "Mismatching operand types");

  assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
  assert((!Signed || !HiLHS) &&
         "Signed flag should only be set when HiLHS and RiRHS are null");

  // We'll expand the multiplication by brute force because we have no other
  // options. This is a trivially-generalized version of the code from
  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
  // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
  // sign bits while calculating the Hi half.
  unsigned Bits = VT.getSizeInBits();
  unsigned HalfBits = Bits / 2;
  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT);
  SDValue LL = DAG.getNode(ISD::AND, dl, VT, LHS, Mask);
  SDValue RL = DAG.getNode(ISD::AND, dl, VT, RHS, Mask);

  SDValue T = DAG.getNode(ISD::MUL, dl, VT, LL, RL);
  SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask);

  SDValue Shift = DAG.getShiftAmountConstant(HalfBits, VT, dl);
  // This is always an unsigned shift.
  SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift);

  unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
  SDValue LH = DAG.getNode(ShiftOpc, dl, VT, LHS, Shift);
  SDValue RH = DAG.getNode(ShiftOpc, dl, VT, RHS, Shift);

  SDValue U =
      DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LH, RL), TH);
  SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask);
  SDValue UH = DAG.getNode(ShiftOpc, dl, VT, U, Shift);

  SDValue V =
      DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LL, RH), UL);
  SDValue VH = DAG.getNode(ShiftOpc, dl, VT, V, Shift);

  Lo = DAG.getNode(ISD::ADD, dl, VT, TL,
                   DAG.getNode(ISD::SHL, dl, VT, V, Shift));

  Hi = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LH, RH),
                   DAG.getNode(ISD::ADD, dl, VT, UH, VH));

  // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
  // the products to Hi.
  if (HiLHS) {
    SDValue RHLL = DAG.getNode(ISD::MUL, dl, VT, HiRHS, LHS);
    SDValue RLLH = DAG.getNode(ISD::MUL, dl, VT, RHS, HiLHS);
    Hi = DAG.getNode(ISD::ADD, dl, VT, Hi,
                     DAG.getNode(ISD::ADD, dl, VT, RHLL, RLLH));
  }
}

void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
                                        bool Signed, const SDValue LHS,
                                        const SDValue RHS, SDValue &Lo,
                                        SDValue &Hi) const {
  EVT VT = LHS.getValueType();
  assert(RHS.getValueType() == VT && "Mismatching operand types");
  EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
  // We can fall back to a libcall with an illegal type for the MUL if we
  // have a libcall big enough.
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
  if (WideVT == MVT::i16)
    LC = RTLIB::MUL_I16;
  else if (WideVT == MVT::i32)
    LC = RTLIB::MUL_I32;
  else if (WideVT == MVT::i64)
    LC = RTLIB::MUL_I64;
  else if (WideVT == MVT::i128)
    LC = RTLIB::MUL_I128;

  RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(LC);
  if (LibcallImpl == RTLIB::Unsupported) {
    forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
    return;
  }

  SDValue HiLHS, HiRHS;
  if (Signed) {
    // The high part is obtained by SRA'ing all but one of the bits of low
    // part.
    unsigned LoSize = VT.getFixedSizeInBits();
    SDValue Shift = DAG.getShiftAmountConstant(LoSize - 1, VT, dl);
    HiLHS = DAG.getNode(ISD::SRA, dl, VT, LHS, Shift);
    HiRHS = DAG.getNode(ISD::SRA, dl, VT, RHS, Shift);
  } else {
    HiLHS = DAG.getConstant(0, dl, VT);
    HiRHS = DAG.getConstant(0, dl, VT);
  }

  // Attempt a libcall.
  SDValue Ret;
  TargetLowering::MakeLibCallOptions CallOptions;
  CallOptions.setIsSigned(Signed);
  CallOptions.setIsPostTypeLegalization(true);
  if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
    // Halves of WideVT are packed into registers in different order
    // depending on platform endianness. This is usually handled by
    // the C calling convention, but we can't defer to it in
    // the legalizer.
    SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
  } else {
    SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
  }
  assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
         "Ret value is a collection of constituent nodes holding result.");
  if (DAG.getDataLayout().isLittleEndian()) {
    // Same as above.
    Lo = Ret.getOperand(0);
    Hi = Ret.getOperand(1);
  } else {
    Lo = Ret.getOperand(1);
    Hi = Ret.getOperand(0);
  }
}

SDValue
TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
  assert((Node->getOpcode() == ISD::SMULFIX ||
          Node->getOpcode() == ISD::UMULFIX ||
          Node->getOpcode() == ISD::SMULFIXSAT ||
          Node->getOpcode() == ISD::UMULFIXSAT) &&
         "Expected a fixed point multiplication opcode");

  SDLoc dl(Node);
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  EVT VT = LHS.getValueType();
  unsigned Scale = Node->getConstantOperandVal(2);
  bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
                     Node->getOpcode() == ISD::UMULFIXSAT);
  bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
                 Node->getOpcode() == ISD::SMULFIXSAT);
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  unsigned VTSize = VT.getScalarSizeInBits();

  if (!Scale) {
    // [us]mul.fix(a, b, 0) -> mul(a, b)
    if (!Saturating) {
      if (isOperationLegalOrCustom(ISD::MUL, VT))
        return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
      SDValue Result =
          DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
      SDValue Product = Result.getValue(0);
      SDValue Overflow = Result.getValue(1);
      SDValue Zero = DAG.getConstant(0, dl, VT);

      APInt MinVal = APInt::getSignedMinValue(VTSize);
      APInt MaxVal = APInt::getSignedMaxValue(VTSize);
      SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
      SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
      // Xor the inputs, if resulting sign bit is 0 the product will be
      // positive, else negative.
      SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
      SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
      Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
      return DAG.getSelect(dl, VT, Overflow, Result, Product);
    } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
      SDValue Result =
          DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
      SDValue Product = Result.getValue(0);
      SDValue Overflow = Result.getValue(1);

      APInt MaxVal = APInt::getMaxValue(VTSize);
      SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
      return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
    }
  }

  assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
         "Expected scale to be less than the number of bits if signed or at "
         "most the number of bits if unsigned.");
  assert(LHS.getValueType() == RHS.getValueType() &&
         "Expected both operands to be the same type");

  // Get the upper and lower bits of the result.
  SDValue Lo, Hi;
  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
  unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
  EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());
  if (isOperationLegalOrCustom(LoHiOp, VT)) {
    SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
    Lo = Result.getValue(0);
    Hi = Result.getValue(1);
  } else if (isOperationLegalOrCustom(HiOp, VT)) {
    Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
  } else if (isOperationLegalOrCustom(ISD::MUL, WideVT)) {
    // Try for a multiplication using a wider type.
    unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    SDValue LHSExt = DAG.getNode(Ext, dl, WideVT, LHS);
    SDValue RHSExt = DAG.getNode(Ext, dl, WideVT, RHS);
    SDValue Res = DAG.getNode(ISD::MUL, dl, WideVT, LHSExt, RHSExt);
    Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
    SDValue Shifted =
        DAG.getNode(ISD::SRA, dl, WideVT, Res,
                    DAG.getShiftAmountConstant(VTSize, WideVT, dl));
    Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Shifted);
  } else if (VT.isVector()) {
    return SDValue();
  } else {
    forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
  }

  if (Scale == VTSize)
    // Result is just the top half since we'd be shifting by the width of the
    // operand. Overflow impossible so this works for both UMULFIX and
    // UMULFIXSAT.
    return Hi;

  // The result will need to be shifted right by the scale since both operands
  // are scaled. The result is given to us in 2 halves, so we only want part of
  // both in the result.
  SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
                               DAG.getShiftAmountConstant(Scale, VT, dl));
  if (!Saturating)
    return Result;

  if (!Signed) {
    // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
    // widened multiplication) aren't all zeroes.

    // Saturate to max if ((Hi >> Scale) != 0),
    // which is the same as if (Hi > ((1 << Scale) - 1))
    APInt MaxVal = APInt::getMaxValue(VTSize);
    SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
                                      dl, VT);
    Result = DAG.getSelectCC(dl, Hi, LowMask,
                             DAG.getConstant(MaxVal, dl, VT), Result,
                             ISD::SETUGT);

    return Result;
  }

  // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
  // widened multiplication) aren't all ones or all zeroes.

  SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
  SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);

  if (Scale == 0) {
    SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
                               DAG.getShiftAmountConstant(VTSize - 1, VT, dl));
    SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
    // Saturated to SatMin if wide product is negative, and SatMax if wide
    // product is positive ...
    SDValue Zero = DAG.getConstant(0, dl, VT);
    SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
                                               ISD::SETLT);
    // ... but only if we overflowed.
    return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
  }

  //  We handled Scale==0 above so all the bits to examine is in Hi.

  // Saturate to max if ((Hi >> (Scale - 1)) > 0),
  // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
  SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
                                    dl, VT);
  Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
  // Saturate to min if (Hi >> (Scale - 1)) < -1),
  // which is the same as if (HI < (-1 << (Scale - 1))
  SDValue HighMask =
      DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
                      dl, VT);
  Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
  return Result;
}

SDValue
TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
                                    SDValue LHS, SDValue RHS,
                                    unsigned Scale, SelectionDAG &DAG) const {
  assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
          Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
         "Expected a fixed point division opcode");

  EVT VT = LHS.getValueType();
  bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
  bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
  EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

  // If there is enough room in the type to upscale the LHS or downscale the
  // RHS before the division, we can perform it in this type without having to
  // resize. For signed operations, the LHS headroom is the number of
  // redundant sign bits, and for unsigned ones it is the number of zeroes.
  // The headroom for the RHS is the number of trailing zeroes.
  unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
                            : DAG.computeKnownBits(LHS).countMinLeadingZeros();
  unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();

  // For signed saturating operations, we need to be able to detect true integer
  // division overflow; that is, when you have MIN / -EPS. However, this
  // is undefined behavior and if we emit divisions that could take such
  // values it may cause undesired behavior (arithmetic exceptions on x86, for
  // example).
  // Avoid this by requiring an extra bit so that we never get this case.
  // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
  // signed saturating division, we need to emit a whopping 32-bit division.
  if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
    return SDValue();

  unsigned LHSShift = std::min(LHSLead, Scale);
  unsigned RHSShift = Scale - LHSShift;

  // At this point, we know that if we shift the LHS up by LHSShift and the
  // RHS down by RHSShift, we can emit a regular division with a final scaling
  // factor of Scale.

  if (LHSShift)
    LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
                      DAG.getShiftAmountConstant(LHSShift, VT, dl));
  if (RHSShift)
    RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
                      DAG.getShiftAmountConstant(RHSShift, VT, dl));

  SDValue Quot;
  if (Signed) {
    // For signed operations, if the resulting quotient is negative and the
    // remainder is nonzero, subtract 1 from the quotient to round towards
    // negative infinity.
    SDValue Rem;
    // FIXME: Ideally we would always produce an SDIVREM here, but if the
    // type isn't legal, SDIVREM cannot be expanded. There is no reason why
    // we couldn't just form a libcall, but the type legalizer doesn't do it.
    if (isTypeLegal(VT) &&
        isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
      Quot = DAG.getNode(ISD::SDIVREM, dl,
                         DAG.getVTList(VT, VT),
                         LHS, RHS);
      Rem = Quot.getValue(1);
      Quot = Quot.getValue(0);
    } else {
      Quot = DAG.getNode(ISD::SDIV, dl, VT,
                         LHS, RHS);
      Rem = DAG.getNode(ISD::SREM, dl, VT,
                        LHS, RHS);
    }
    SDValue Zero = DAG.getConstant(0, dl, VT);
    SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
    SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
    SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
    SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
    SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
                               DAG.getConstant(1, dl, VT));
    Quot = DAG.getSelect(dl, VT,
                         DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
                         Sub1, Quot);
  } else
    Quot = DAG.getNode(ISD::UDIV, dl, VT,
                       LHS, RHS);

  return Quot;
}

void TargetLowering::expandUADDSUBO(
    SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  bool IsAdd = Node->getOpcode() == ISD::UADDO;

  // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
  unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
  if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
    SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
    SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
                                    { LHS, RHS, CarryIn });
    Result = SDValue(NodeCarry.getNode(), 0);
    Overflow = SDValue(NodeCarry.getNode(), 1);
    return;
  }

  Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
                            LHS.getValueType(), LHS, RHS);

  EVT ResultType = Node->getValueType(1);
  EVT SetCCType = getSetCCResultType(
      DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
  SDValue SetCC;
  if (IsAdd && isOneConstant(RHS)) {
    // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
    // the live range of X. We assume comparing with 0 is cheap.
    // The general case (X + C) < C is not necessarily beneficial. Although we
    // reduce the live range of X, we may introduce the materialization of
    // constant C.
    SetCC =
        DAG.getSetCC(dl, SetCCType, Result,
                     DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ);
  } else if (IsAdd && isAllOnesConstant(RHS)) {
    // Special case: uaddo X, -1 overflows if X != 0.
    SetCC =
        DAG.getSetCC(dl, SetCCType, LHS,
                     DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE);
  } else {
    ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
    SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
  }
  Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
}

void TargetLowering::expandSADDSUBO(
    SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  bool IsAdd = Node->getOpcode() == ISD::SADDO;

  Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
                            LHS.getValueType(), LHS, RHS);

  EVT ResultType = Node->getValueType(1);
  EVT OType = getSetCCResultType(
      DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

  // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
  unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
  if (isOperationLegal(OpcSat, LHS.getValueType())) {
    SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
    SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
    Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
    return;
  }

  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());

  if (IsAdd) {
    // For an addition, the result should be less than one of the operands (LHS)
    // if and only if the other operand (RHS) is negative, otherwise there will
    // be overflow.
    SDValue ResultLowerThanLHS =
        DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
    SDValue RHSNegative = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETLT);
    Overflow = DAG.getBoolExtOrTrunc(
        DAG.getNode(ISD::XOR, dl, OType, RHSNegative, ResultLowerThanLHS), dl,
        ResultType, ResultType);
  } else {
    // For subtraction, overflow occurs when the signed comparison of operands
    // doesn't match the sign of the result.
    SDValue LHSLessThanRHS = DAG.getSetCC(dl, OType, LHS, RHS, ISD::SETLT);
    SDValue ResultNegative = DAG.getSetCC(dl, OType, Result, Zero, ISD::SETLT);
    Overflow = DAG.getBoolExtOrTrunc(
        DAG.getNode(ISD::XOR, dl, OType, LHSLessThanRHS, ResultNegative), dl,
        ResultType, ResultType);
  }
}

bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
                                SDValue &Overflow, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  EVT VT = Node->getValueType(0);
  EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue LHS = Node->getOperand(0);
  SDValue RHS = Node->getOperand(1);
  bool isSigned = Node->getOpcode() == ISD::SMULO;

  // For power-of-two multiplications we can use a simpler shift expansion.
  if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
    const APInt &C = RHSC->getAPIntValue();
    // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
    if (C.isPowerOf2()) {
      // smulo(x, signed_min) is same as umulo(x, signed_min).
      bool UseArithShift = isSigned && !C.isMinSignedValue();
      SDValue ShiftAmt = DAG.getShiftAmountConstant(C.logBase2(), VT, dl);
      Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
      Overflow = DAG.getSetCC(dl, SetCCVT,
          DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
                      dl, VT, Result, ShiftAmt),
          LHS, ISD::SETNE);
      return true;
    }
  }

  SDValue BottomHalf;
  SDValue TopHalf;
  EVT WideVT = VT.widenIntegerElementType(*DAG.getContext());

  static const unsigned Ops[2][3] =
      { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
        { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
  if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
    BottomHalf = DAG.getNode(Ops[isSigned][0], dl, DAG.getVTList(VT, VT), LHS,
                             RHS);
    TopHalf = BottomHalf.getValue(1);
  } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
    BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
    TopHalf = DAG.getNode(Ops[isSigned][1], dl, VT, LHS, RHS);
  } else if (isTypeLegal(WideVT)) {
    LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
    RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
    SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
    BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
    SDValue ShiftAmt =
        DAG.getShiftAmountConstant(VT.getScalarSizeInBits(), WideVT, dl);
    TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
                          DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
  } else {
    if (VT.isVector())
      return false;

    forceExpandWideMUL(DAG, dl, isSigned, LHS, RHS, BottomHalf, TopHalf);
  }

  Result = BottomHalf;
  if (isSigned) {
    SDValue ShiftAmt = DAG.getShiftAmountConstant(
        VT.getScalarSizeInBits() - 1, BottomHalf.getValueType(), dl);
    SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
    Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
  } else {
    Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
                            DAG.getConstant(0, dl, VT), ISD::SETNE);
  }

  // Truncate the result if SetCC returns a larger type than needed.
  EVT RType = Node->getValueType(1);
  if (RType.bitsLT(Overflow.getValueType()))
    Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);

  assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
         "Unexpected result type for S/UMULO legalization");
  return true;
}

SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
  SDValue Op = Node->getOperand(0);
  SDNodeFlags Flags = Node->getFlags();
  EVT VT = Op.getValueType();

  // Try to use a shuffle reduction for power of two vectors.
  if (VT.isPow2VectorType()) {
    // See if the reduction opcode is safe to use with widened types.
    bool WidenSrc = false;
    switch (Node->getOpcode()) {
    case ISD::VECREDUCE_FADD:
    case ISD::VECREDUCE_FMUL:
    case ISD::VECREDUCE_ADD:
    case ISD::VECREDUCE_MUL:
    case ISD::VECREDUCE_AND:
    case ISD::VECREDUCE_OR:
    case ISD::VECREDUCE_XOR:
    case ISD::VECREDUCE_SMAX:
    case ISD::VECREDUCE_SMIN:
    case ISD::VECREDUCE_UMAX:
    case ISD::VECREDUCE_UMIN:
      WidenSrc = VT.isFixedLengthVector();
      break;
    }

    while (VT.getVectorElementCount().isKnownMultipleOf(2)) {
      EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
      if (!isOperationLegalOrCustom(BaseOpcode, HalfVT)) {
        if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
          // Attempt to widen the source vectors to a legal op.
          EVT WideVT = getTypeToTransformTo(*DAG.getContext(), HalfVT);
          if (WideVT.isVector() &&
              WideVT.getScalarType() == HalfVT.getScalarType() &&
              WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
              isOperationLegalOrCustom(BaseOpcode, WideVT)) {
            SDValue Lo, Hi;
            std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
            Lo = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Lo, 0);
            Hi = DAG.getInsertSubvector(dl, DAG.getPOISON(WideVT), Hi, 0);
            Op = DAG.getNode(BaseOpcode, dl, WideVT, Lo, Hi, Flags);
            Op = DAG.getExtractSubvector(dl, HalfVT, Op, 0);
            VT = HalfVT;
            continue;
          }
        }
        break;
      }

      SDValue Lo, Hi;
      std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
      Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi, Flags);
      VT = HalfVT;

      // Stop if splitting is enough to make the reduction legal.
      if (isOperationLegalOrCustom(Node->getOpcode(), HalfVT))
        return DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Op,
                           Flags);
    }
  }

  if (VT.isScalableVector())
    reportFatalInternalError(
        "Expanding reductions for scalable vectors is undefined.");

  EVT EltVT = VT.getVectorElementType();
  unsigned NumElts = VT.getVectorNumElements();

  SmallVector<SDValue, 8> Ops;
  DAG.ExtractVectorElements(Op, Ops, 0, NumElts);

  SDValue Res = Ops[0];
  for (unsigned i = 1; i < NumElts; i++)
    Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);

  // Result type may be wider than element type.
  if (EltVT != Node->getValueType(0))
    Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
  return Res;
}

SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc dl(Node);
  SDValue AccOp = Node->getOperand(0);
  SDValue VecOp = Node->getOperand(1);
  SDNodeFlags Flags = Node->getFlags();

  EVT VT = VecOp.getValueType();
  EVT EltVT = VT.getVectorElementType();

  if (VT.isScalableVector())
    report_fatal_error(
        "Expanding reductions for scalable vectors is undefined.");

  unsigned NumElts = VT.getVectorNumElements();

  SmallVector<SDValue, 8> Ops;
  DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);

  unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

  SDValue Res = AccOp;
  for (unsigned i = 0; i < NumElts; i++)
    Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);

  return Res;
}

bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
                               SelectionDAG &DAG) const {
  EVT VT = Node->getValueType(0);
  SDLoc dl(Node);
  bool isSigned = Node->getOpcode() == ISD::SREM;
  unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
  SDValue Dividend = Node->getOperand(0);
  SDValue Divisor = Node->getOperand(1);
  if (isOperationLegalOrCustom(DivRemOpc, VT)) {
    SDVTList VTs = DAG.getVTList(VT, VT);
    Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
    return true;
  }
  if (isOperationLegalOrCustom(DivOpc, VT)) {
    // X % Y -> X-X/Y*Y
    SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
    SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
    Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
    return true;
  }
  return false;
}

SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
                                            SelectionDAG &DAG) const {
  bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
  SDLoc dl(SDValue(Node, 0));
  SDValue Src = Node->getOperand(0);

  // DstVT is the result type, while SatVT is the size to which we saturate
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Node->getValueType(0);

  EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
  unsigned SatWidth = SatVT.getScalarSizeInBits();
  unsigned DstWidth = DstVT.getScalarSizeInBits();
  assert(SatWidth <= DstWidth &&
         "Expected saturation width smaller than result width");

  // Determine minimum and maximum integer values and their corresponding
  // floating-point values.
  APInt MinInt, MaxInt;
  if (IsSigned) {
    MinInt = APInt::getSignedMinValue(SatWidth).sext(DstWidth);
    MaxInt = APInt::getSignedMaxValue(SatWidth).sext(DstWidth);
  } else {
    MinInt = APInt::getMinValue(SatWidth).zext(DstWidth);
    MaxInt = APInt::getMaxValue(SatWidth).zext(DstWidth);
  }

  // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
  // libcall emission cannot handle this. Large result types will fail.
  if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
    Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
    SrcVT = Src.getValueType();
  }

  const fltSemantics &Sem = SrcVT.getFltSemantics();
  APFloat MinFloat(Sem);
  APFloat MaxFloat(Sem);

  APFloat::opStatus MinStatus =
      MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
  APFloat::opStatus MaxStatus =
      MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
  bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
                             !(MaxStatus & APFloat::opStatus::opInexact);

  SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
  SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);

  // If the integer bounds are exactly representable as floats and min/max are
  // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
  // of comparisons and selects.
  auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
                        bool MayPropagateNaN) {
    bool MinMaxLegal = isOperationLegalOrCustom(MinOpcode, SrcVT) &&
                       isOperationLegalOrCustom(MaxOpcode, SrcVT);
    if (!MinMaxLegal)
      return SDValue();

    SDValue Clamped = Src;

    // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
    // then the result is MinFloat.
    Clamped = DAG.getNode(MaxOpcode, dl, SrcVT, Clamped, MinFloatNode);
    // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
    Clamped = DAG.getNode(MinOpcode, dl, SrcVT, Clamped, MaxFloatNode);
    // Convert clamped value to integer.
    SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
                                  dl, DstVT, Clamped);

    // If !MayPropagateNan and the conversion is unsigned case we're done,
    // because we mapped NaN to MinFloat, which will cast to zero.
    if (!MayPropagateNaN && !IsSigned)
      return FpToInt;

    // Otherwise, select 0 if Src is NaN.
    SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
    EVT SetCCVT =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
    SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
    return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
  };
  if (AreExactFloatBounds) {
    if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
                                 /*MayPropagateNaN=*/false))
      return Res;
    // These may propagate NaN for sNaN operands.
    if (SDValue Res =
            EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
      return Res;
    // These always propagate NaN.
    if (SDValue Res =
            EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
      return Res;
  }

  SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
  SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);

  // Result of direct conversion. The assumption here is that the operation is
  // non-trapping and it's fine to apply it to an out-of-range value if we
  // select it away later.
  SDValue FpToInt =
      DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);

  SDValue Select = FpToInt;

  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);

  // If Src ULT MinFloat, select MinInt. In particular, this also selects
  // MinInt if Src is NaN.
  SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT);
  Select = DAG.getSelect(dl, DstVT, ULT, MinIntNode, Select);
  // If Src OGT MaxFloat, select MaxInt.
  SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT);
  Select = DAG.getSelect(dl, DstVT, OGT, MaxIntNode, Select);

  // In the unsigned case we are done, because we mapped NaN to MinInt, which
  // is already zero.
  if (!IsSigned)
    return Select;

  // Otherwise, select 0 if Src is NaN.
  SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
  SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO);
  return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select);
}

SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
                                                const SDLoc &dl,
                                                SelectionDAG &DAG) const {
  EVT OperandVT = Op.getValueType();
  if (OperandVT.getScalarType() == ResultVT.getScalarType())
    return Op;
  EVT ResultIntVT = ResultVT.changeTypeToInteger();
  // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
  // can induce double-rounding which may alter the results. We can
  // correct for this using a trick explained in: Boldo, Sylvie, and
  // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
  // World Congress. 2005.
  SDValue Narrow = DAG.getFPExtendOrRound(Op, dl, ResultVT);
  SDValue NarrowAsWide = DAG.getFPExtendOrRound(Narrow, dl, OperandVT);

  // We can keep the narrow value as-is if narrowing was exact (no
  // rounding error), the wide value was NaN (the narrow value is also
  // NaN and should be preserved) or if we rounded to the odd value.
  SDValue NarrowBits = DAG.getNode(ISD::BITCAST, dl, ResultIntVT, Narrow);
  SDValue One = DAG.getConstant(1, dl, ResultIntVT);
  SDValue NegativeOne = DAG.getAllOnesConstant(dl, ResultIntVT);
  SDValue And = DAG.getNode(ISD::AND, dl, ResultIntVT, NarrowBits, One);
  EVT ResultIntVTCCVT = getSetCCResultType(
      DAG.getDataLayout(), *DAG.getContext(), And.getValueType());
  SDValue Zero = DAG.getConstant(0, dl, ResultIntVT);
  // The result is already odd so we don't need to do anything.
  SDValue AlreadyOdd = DAG.getSetCC(dl, ResultIntVTCCVT, And, Zero, ISD::SETNE);

  EVT WideSetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                                       Op.getValueType());
  // We keep results which are exact, odd or NaN.
  SDValue KeepNarrow =
      DAG.getSetCC(dl, WideSetCCVT, Op, NarrowAsWide, ISD::SETUEQ);
  KeepNarrow = DAG.getNode(ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
  // We morally performed a round-down if AbsNarrow is smaller than
  // AbsWide.
  SDValue AbsWide = DAG.getNode(ISD::FABS, dl, OperandVT, Op);
  SDValue AbsNarrowAsWide = DAG.getNode(ISD::FABS, dl, OperandVT, NarrowAsWide);
  SDValue NarrowIsRd =
      DAG.getSetCC(dl, WideSetCCVT, AbsWide, AbsNarrowAsWide, ISD::SETOGT);
  // If the narrow value is odd or exact, pick it.
  // Otherwise, narrow is even and corresponds to either the rounded-up
  // or rounded-down value. If narrow is the rounded-down value, we want
  // the rounded-up value as it will be odd.
  SDValue Adjust = DAG.getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
  SDValue Adjusted = DAG.getNode(ISD::ADD, dl, ResultIntVT, NarrowBits, Adjust);
  Op = DAG.getSelect(dl, ResultIntVT, KeepNarrow, NarrowBits, Adjusted);
  return DAG.getNode(ISD::BITCAST, dl, ResultVT, Op);
}

SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
  assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
  SDValue Op = Node->getOperand(0);
  EVT VT = Node->getValueType(0);
  SDLoc dl(Node);
  if (VT.getScalarType() == MVT::bf16) {
    if (Node->getConstantOperandVal(1) == 1) {
      return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0));
    }
    EVT OperandVT = Op.getValueType();
    SDValue IsNaN = DAG.getSetCC(
        dl,
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OperandVT),
        Op, Op, ISD::SETUO);

    // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
    // can induce double-rounding which may alter the results. We can
    // correct for this using a trick explained in: Boldo, Sylvie, and
    // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
    // World Congress. 2005.
    EVT F32 = VT.changeElementType(*DAG.getContext(), MVT::f32);
    EVT I32 = F32.changeTypeToInteger();
    Op = expandRoundInexactToOdd(F32, Op, dl, DAG);
    Op = DAG.getNode(ISD::BITCAST, dl, I32, Op);

    // Conversions should set NaN's quiet bit. This also prevents NaNs from
    // turning into infinities.
    SDValue NaN =
        DAG.getNode(ISD::OR, dl, I32, Op, DAG.getConstant(0x400000, dl, I32));

    // Factor in the contribution of the low 16 bits.
    SDValue One = DAG.getConstant(1, dl, I32);
    SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Op,
                              DAG.getShiftAmountConstant(16, I32, dl));
    Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One);
    SDValue RoundingBias =
        DAG.getNode(ISD::ADD, dl, I32, Lsb, DAG.getConstant(0x7fff, dl, I32));
    SDValue Add = DAG.getNode(ISD::ADD, dl, I32, Op, RoundingBias);

    // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
    // 0x80000000.
    Op = DAG.getSelect(dl, I32, IsNaN, NaN, Add);

    // Now that we have rounded, shift the bits into position.
    Op = DAG.getNode(ISD::SRL, dl, I32, Op,
                     DAG.getShiftAmountConstant(16, I32, dl));
    EVT I16 = I32.changeElementType(*DAG.getContext(), MVT::i16);
    Op = DAG.getNode(ISD::TRUNCATE, dl, I16, Op);
    return DAG.getNode(ISD::BITCAST, dl, VT, Op);
  }
  return SDValue();
}

SDValue TargetLowering::expandVectorSplice(SDNode *Node,
                                           SelectionDAG &DAG) const {
  assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
          Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
         "Unexpected opcode!");
  assert((Node->getValueType(0).isScalableVector() ||
          !isa<ConstantSDNode>(Node->getOperand(2))) &&
         "Fixed length vector types with constant offsets expected to use "
         "SHUFFLE_VECTOR!");

  EVT VT = Node->getValueType(0);
  SDValue V1 = Node->getOperand(0);
  SDValue V2 = Node->getOperand(1);
  SDValue Offset = Node->getOperand(2);
  SDLoc DL(Node);

  // Expand through memory thusly:
  //  Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
  //  Store V1, Ptr
  //  Store V2, Ptr + sizeof(V1)
  //  if (VECTOR_SPLICE_LEFT)
  //    Ptr = Ptr + (Offset * sizeof(VT.Elt))
  //  else
  //    Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
  //  Res = Load Ptr

  Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);

  EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
                               VT.getVectorElementCount() * 2);
  SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
  EVT PtrVT = StackPtr.getValueType();
  auto &MF = DAG.getMachineFunction();
  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);

  // Store the lo part of CONCAT_VECTORS(V1, V2)
  SDValue StoreV1 =
      DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo, Alignment);
  // Store the hi part of CONCAT_VECTORS(V1, V2)
  SDValue VTBytes = DAG.getTypeSize(DL, PtrVT, VT.getStoreSize());
  SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, VTBytes);
  SDValue StoreV2 =
      DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo, Alignment);

  // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
  SDValue EltByteSize =
      DAG.getTypeSize(DL, PtrVT, VT.getVectorElementType().getStoreSize());
  Offset = DAG.getZExtOrTrunc(Offset, DL, PtrVT);
  SDValue TrailingBytes = DAG.getNode(ISD::MUL, DL, PtrVT, Offset, EltByteSize);

  TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VTBytes);

  if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
    StackPtr = DAG.getMemBasePlusOffset(StackPtr, TrailingBytes, DL);
  else
    StackPtr = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);

  // Load the spliced result
  return DAG.getLoad(VT, DL, StoreV2, StackPtr,
                     MachinePointerInfo::getUnknownStack(MF), Alignment);
}

SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Node);
  SDValue Vec = Node->getOperand(0);
  SDValue Mask = Node->getOperand(1);
  SDValue Passthru = Node->getOperand(2);

  EVT VecVT = Vec.getValueType();
  EVT ScalarVT = VecVT.getScalarType();
  EVT MaskVT = Mask.getValueType();
  EVT MaskScalarVT = MaskVT.getScalarType();

  // Needs to be handled by targets that have scalable vector types.
  if (VecVT.isScalableVector())
    report_fatal_error("Cannot expand masked_compress for scalable vectors.");

  Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT.getStoreSize(), Alignment);
  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
  MachinePointerInfo PtrInfo =
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);

  MVT PositionVT = getVectorIdxTy(DAG.getDataLayout());
  SDValue Chain = DAG.getEntryNode();
  SDValue OutPos = DAG.getConstant(0, DL, PositionVT);

  bool HasPassthru = !Passthru.isUndef();

  // If we have a passthru vector, store it on the stack, overwrite the matching
  // positions and then re-write the last element that was potentially
  // overwritten even though mask[i] = false.
  if (HasPassthru)
    Chain = DAG.getStore(Chain, DL, Passthru, StackPtr, PtrInfo, Alignment);

  SDValue LastWriteVal;
  APInt PassthruSplatVal;
  bool IsSplatPassthru =
      ISD::isConstantSplatVector(Passthru.getNode(), PassthruSplatVal);

  if (IsSplatPassthru) {
    // As we do not know which position we wrote to last, we cannot simply
    // access that index from the passthru vector. So we first check if passthru
    // is a splat vector, to use any element ...
    LastWriteVal = DAG.getConstant(PassthruSplatVal, DL, ScalarVT);
  } else if (HasPassthru) {
    // ... if it is not a splat vector, we need to get the passthru value at
    // position = popcount(mask) and re-load it from the stack before it is
    // overwritten in the loop below.
    EVT PopcountVT = ScalarVT.changeTypeToInteger();
    SDValue Popcount = DAG.getNode(
        ISD::TRUNCATE, DL,
        MaskVT.changeVectorElementType(*DAG.getContext(), MVT::i1), Mask);
    Popcount = DAG.getNode(
        ISD::ZERO_EXTEND, DL,
        MaskVT.changeVectorElementType(*DAG.getContext(), PopcountVT),
        Popcount);
    Popcount = DAG.getNode(ISD::VECREDUCE_ADD, DL, PopcountVT, Popcount);
    SDValue LastElmtPtr =
        getVectorElementPointer(DAG, StackPtr, VecVT, Popcount);
    LastWriteVal = DAG.getLoad(
        ScalarVT, DL, Chain, LastElmtPtr,
        MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
    Chain = LastWriteVal.getValue(1);
  }

  unsigned NumElms = VecVT.getVectorNumElements();
  for (unsigned I = 0; I < NumElms; I++) {
    SDValue ValI = DAG.getExtractVectorElt(DL, ScalarVT, Vec, I);
    SDValue OutPtr = getVectorElementPointer(DAG, StackPtr, VecVT, OutPos);
    Chain = DAG.getStore(
        Chain, DL, ValI, OutPtr,
        MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));

    // Get the mask value and add it to the current output position. This
    // either increments by 1 if MaskI is true or adds 0 otherwise.
    // Freeze in case we have poison/undef mask entries.
    SDValue MaskI = DAG.getExtractVectorElt(DL, MaskScalarVT, Mask, I);
    MaskI = DAG.getFreeze(MaskI);
    MaskI = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, MaskI);
    MaskI = DAG.getNode(ISD::ZERO_EXTEND, DL, PositionVT, MaskI);
    OutPos = DAG.getNode(ISD::ADD, DL, PositionVT, OutPos, MaskI);

    if (HasPassthru && I == NumElms - 1) {
      SDValue EndOfVector =
          DAG.getConstant(VecVT.getVectorNumElements() - 1, DL, PositionVT);
      SDValue AllLanesSelected =
          DAG.getSetCC(DL, MVT::i1, OutPos, EndOfVector, ISD::CondCode::SETUGT);
      OutPos = DAG.getNode(ISD::UMIN, DL, PositionVT, OutPos, EndOfVector);
      OutPtr = getVectorElementPointer(DAG, StackPtr, VecVT, OutPos);

      // Re-write the last ValI if all lanes were selected. Otherwise,
      // overwrite the last write it with the passthru value.
      LastWriteVal = DAG.getSelect(DL, ScalarVT, AllLanesSelected, ValI,
                                   LastWriteVal, SDNodeFlags::Unpredictable);
      Chain = DAG.getStore(
          Chain, DL, LastWriteVal, OutPtr,
          MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
    }
  }

  return DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo, Alignment);
}

SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
  SDLoc DL(Node);
  EVT VT = Node->getValueType(0);

  bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
  auto [Mask, StepVec] =
      getLegalMaskAndStepVector(Node->getOperand(0), ZeroIsPoison, DL, DAG);

  // No legal step vector: split mask in half and recombine results.
  // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
  // (== LoNumElts when no active lane), allowing the SETNE comparison.
  // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
  if (!StepVec) {
    EVT ResVT = Node->getValueType(0);
    auto [MaskLo, MaskHi] = DAG.SplitVector(Node->getOperand(0), DL);
    SDValue LoNumElts = DAG.getElementCount(
        DL, ResVT, MaskLo.getValueType().getVectorElementCount());
    SDValue ResLo = DAG.getNode(ISD::CTTZ_ELTS, DL, ResVT, MaskLo);
    SDValue ResHi = DAG.getNode(Node->getOpcode(), DL, ResVT, MaskHi);
    SDValue ResLoNotNumElts = DAG.getSetCC(
        DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResVT),
        ResLo, LoNumElts, ISD::SETNE);
    // Per LangRef, ResVT must be wide enough to hold the total element count,
    // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
    // the count is only required to fit unsigned.
    SDValue Sum = DAG.getNode(ISD::ADD, DL, ResVT, LoNumElts, ResHi,
                              SDNodeFlags::NoUnsignedWrap);
    return DAG.getSelect(DL, ResVT, ResLoNotNumElts, ResLo, Sum);
  }

  EVT StepVecVT = StepVec.getValueType();
  EVT StepVT = StepVecVT.getVectorElementType();

  // Promote the scalar result type early to avoid redundant zexts.
  if (getTypeAction(StepVT.getSimpleVT()) == TypePromoteInteger)
    StepVT = getTypeToTransformTo(*DAG.getContext(), StepVT);

  SDValue VL =
      DAG.getElementCount(DL, StepVT, StepVecVT.getVectorElementCount());
  SDValue SplatVL = DAG.getSplat(StepVecVT, DL, VL);
  StepVec = DAG.getNode(ISD::SUB, DL, StepVecVT, SplatVL, StepVec);
  SDValue Zeroes = DAG.getConstant(0, DL, StepVecVT);
  SDValue Select = DAG.getSelect(DL, StepVecVT, Mask, StepVec, Zeroes);
  SDValue Max = DAG.getNode(ISD::VECREDUCE_UMAX, DL,
                            StepVecVT.getVectorElementType(), Select);
  SDValue Sub = DAG.getNode(ISD::SUB, DL, StepVT, VL,
                            DAG.getZExtOrTrunc(Max, DL, StepVT));

  return DAG.getZExtOrTrunc(Sub, DL, VT);
}

SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
                                               SelectionDAG &DAG) const {
  SDLoc DL(N);
  SDValue Acc = N->getOperand(0);
  SDValue MulLHS = N->getOperand(1);
  SDValue MulRHS = N->getOperand(2);
  EVT AccVT = Acc.getValueType();
  EVT MulOpVT = MulLHS.getValueType();

  EVT ExtMulOpVT =
      EVT::getVectorVT(*DAG.getContext(), AccVT.getVectorElementType(),
                       MulOpVT.getVectorElementCount());

  unsigned ExtOpcLHS, ExtOpcRHS;
  switch (N->getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::PARTIAL_REDUCE_UMLA:
    ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
    break;
  case ISD::PARTIAL_REDUCE_SMLA:
    ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
    break;
  case ISD::PARTIAL_REDUCE_FMLA:
    ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
    break;
  }

  if (ExtMulOpVT != MulOpVT) {
    MulLHS = DAG.getNode(ExtOpcLHS, DL, ExtMulOpVT, MulLHS);
    MulRHS = DAG.getNode(ExtOpcRHS, DL, ExtMulOpVT, MulRHS);
  }
  SDValue Input = MulLHS;
  if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
    if (!llvm::isOneOrOneSplatFP(MulRHS))
      Input = DAG.getNode(ISD::FMUL, DL, ExtMulOpVT, MulLHS, MulRHS);
  } else if (!llvm::isOneOrOneSplat(MulRHS)) {
    Input = DAG.getNode(ISD::MUL, DL, ExtMulOpVT, MulLHS, MulRHS);
  }

  unsigned Stride = AccVT.getVectorMinNumElements();
  unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;

  // Collect all of the subvectors
  std::deque<SDValue> Subvectors = {Acc};
  for (unsigned I = 0; I < ScaleFactor; I++)
    Subvectors.push_back(DAG.getExtractSubvector(DL, AccVT, Input, I * Stride));

  unsigned FlatNode =
      N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;

  // Flatten the subvector tree
  while (Subvectors.size() > 1) {
    Subvectors.push_back(
        DAG.getNode(FlatNode, DL, AccVT, {Subvectors[0], Subvectors[1]}));
    Subvectors.pop_front();
    Subvectors.pop_front();
  }

  assert(Subvectors.size() == 1 &&
         "There should only be one subvector after tree flattening");

  return Subvectors[0];
}

/// Given a store node \p StoreNode, return true if it is safe to fold that node
/// into \p FPNode, which expands to a library call with output pointers.
static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
                                                  SDNode *FPNode) {
  SmallVector<const SDNode *, 8> Worklist;
  SmallVector<const SDNode *, 8> DeferredNodes;
  SmallPtrSet<const SDNode *, 16> Visited;

  // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
  for (SDValue Op : StoreNode->ops())
    if (Op.getNode() != FPNode)
      Worklist.push_back(Op.getNode());

  unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
  while (!Worklist.empty()) {
    const SDNode *Node = Worklist.pop_back_val();
    auto [_, Inserted] = Visited.insert(Node);
    if (!Inserted)
      continue;

    if (MaxSteps > 0 && Visited.size() >= MaxSteps)
      return false;

    // Reached the FPNode (would result in a cycle).
    // OR Reached CALLSEQ_START (would result in nested call sequences).
    if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
      return false;

    if (Node->getOpcode() == ISD::CALLSEQ_END) {
      // Defer looking into call sequences (so we can check we're outside one).
      // We still need to look through these for the predecessor check.
      DeferredNodes.push_back(Node);
      continue;
    }

    for (SDValue Op : Node->ops())
      Worklist.push_back(Op.getNode());
  }

  // True if we're outside a call sequence and don't have the FPNode as a
  // predecessor. No cycles or nested call sequences possible.
  return !SDNode::hasPredecessorHelper(FPNode, Visited, DeferredNodes,
                                       MaxSteps);
}

bool TargetLowering::expandMultipleResultFPLibCall(
    SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
    SmallVectorImpl<SDValue> &Results,
    std::optional<unsigned> CallRetResNo) const {
  if (LC == RTLIB::UNKNOWN_LIBCALL)
    return false;

  RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(LC);
  if (LibcallImpl == RTLIB::Unsupported)
    return false;

  LLVMContext &Ctx = *DAG.getContext();
  EVT VT = Node->getValueType(0);
  unsigned NumResults = Node->getNumValues();

  // Find users of the node that store the results (and share input chains). The
  // destination pointers can be used instead of creating stack allocations.
  SDValue StoresInChain;
  SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
  for (SDNode *User : Node->users()) {
    if (!ISD::isNormalStore(User))
      continue;
    auto *ST = cast<StoreSDNode>(User);
    SDValue StoreValue = ST->getValue();
    unsigned ResNo = StoreValue.getResNo();
    // Ensure the store corresponds to an output pointer.
    if (CallRetResNo == ResNo)
      continue;
    // Ensure the store to the default address space and not atomic or volatile.
    if (!ST->isSimple() || ST->getAddressSpace() != 0)
      continue;
    // Ensure all store chains are the same (so they don't alias).
    if (StoresInChain && ST->getChain() != StoresInChain)
      continue;
    // Ensure the store is properly aligned.
    Type *StoreType = StoreValue.getValueType().getTypeForEVT(Ctx);
    if (ST->getAlign() <
        DAG.getDataLayout().getABITypeAlign(StoreType->getScalarType()))
      continue;
    // Avoid:
    //  1. Creating cyclic dependencies.
    //  2. Expanding the node to a call within a call sequence.
    if (!canFoldStoreIntoLibCallOutputPointers(ST, Node))
      continue;
    ResultStores[ResNo] = ST;
    StoresInChain = ST->getChain();
  }

  ArgListTy Args;

  // Pass the arguments.
  for (const SDValue &Op : Node->op_values()) {
    EVT ArgVT = Op.getValueType();
    Type *ArgTy = ArgVT.getTypeForEVT(Ctx);
    Args.emplace_back(Op, ArgTy);
  }

  // Pass the output pointers.
  SmallVector<SDValue, 2> ResultPtrs(NumResults);
  Type *PointerTy = PointerType::getUnqual(Ctx);
  for (auto [ResNo, ST] : llvm::enumerate(ResultStores)) {
    if (ResNo == CallRetResNo)
      continue;
    EVT ResVT = Node->getValueType(ResNo);
    SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(ResVT);
    ResultPtrs[ResNo] = ResultPtr;
    Args.emplace_back(ResultPtr, PointerTy);
  }

  SDLoc DL(Node);

  if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(LibcallImpl)) {
    // Pass the vector mask (if required).
    EVT MaskVT = getSetCCResultType(DAG.getDataLayout(), Ctx, VT);
    SDValue Mask = DAG.getBoolConstant(true, DL, MaskVT, VT);
    Args.emplace_back(Mask, MaskVT.getTypeForEVT(Ctx));
  }

  Type *RetType = CallRetResNo.has_value()
                      ? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
                      : Type::getVoidTy(Ctx);
  SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
  SDValue Callee =
      DAG.getExternalSymbol(LibcallImpl, getPointerTy(DAG.getDataLayout()));
  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
      getLibcallImplCallingConv(LibcallImpl), RetType, Callee, std::move(Args));

  auto [Call, CallChain] = LowerCallTo(CLI);

  for (auto [ResNo, ResultPtr] : llvm::enumerate(ResultPtrs)) {
    if (ResNo == CallRetResNo) {
      Results.push_back(Call);
      continue;
    }
    MachinePointerInfo PtrInfo;
    SDValue LoadResult = DAG.getLoad(Node->getValueType(ResNo), DL, CallChain,
                                     ResultPtr, PtrInfo);
    SDValue OutChain = LoadResult.getValue(1);

    if (StoreSDNode *ST = ResultStores[ResNo]) {
      // Replace store with the library call.
      DAG.ReplaceAllUsesOfValueWith(SDValue(ST, 0), OutChain);
      PtrInfo = ST->getPointerInfo();
    } else {
      PtrInfo = MachinePointerInfo::getFixedStack(
          DAG.getMachineFunction(),
          cast<FrameIndexSDNode>(ResultPtr)->getIndex());
    }

    Results.push_back(LoadResult);
  }

  return true;
}

bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
                                           SDValue &LHS, SDValue &RHS,
                                           SDValue &CC, SDValue Mask,
                                           SDValue EVL, bool &NeedInvert,
                                           const SDLoc &dl, SDValue &Chain,
                                           bool IsSignaling) const {
  MVT OpVT = LHS.getSimpleValueType();
  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
  NeedInvert = false;
  assert(!EVL == !Mask && "VP Mask and EVL must either both be set or unset");
  bool IsNonVP = !EVL;
  switch (getCondCodeAction(CCCode, OpVT)) {
  default:
    llvm_unreachable("Unknown condition code action!");
  case TargetLowering::Legal:
    // Nothing to do.
    break;
  case TargetLowering::Expand: {
    ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
      std::swap(LHS, RHS);
      CC = DAG.getCondCode(InvCC);
      return true;
    }
    // Swapping operands didn't work. Try inverting the condition.
    bool NeedSwap = false;
    InvCC = getSetCCInverse(CCCode, OpVT);
    if (!isCondCodeLegalOrCustom(InvCC, OpVT)) {
      // If inverting the condition is not enough, try swapping operands
      // on top of it.
      InvCC = ISD::getSetCCSwappedOperands(InvCC);
      NeedSwap = true;
    }
    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
      CC = DAG.getCondCode(InvCC);
      NeedInvert = true;
      if (NeedSwap)
        std::swap(LHS, RHS);
      return true;
    }

    // Special case: expand i1 comparisons using logical operations.
    if (OpVT == MVT::i1) {
      SDValue Ret;
      switch (CCCode) {
      default:
        llvm_unreachable("Unknown integer setcc!");
      case ISD::SETEQ: // X == Y  -->  ~(X ^ Y)
        Ret = DAG.getNOT(dl, DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS),
                         MVT::i1);
        break;
      case ISD::SETNE: // X != Y  -->  (X ^ Y)
        Ret = DAG.getNode(ISD::XOR, dl, MVT::i1, LHS, RHS);
        break;
      case ISD::SETGT:  // X >s Y  -->  X == 0 & Y == 1  -->  ~X & Y
      case ISD::SETULT: // X <u Y  -->  X == 0 & Y == 1  -->  ~X & Y
        Ret = DAG.getNode(ISD::AND, dl, MVT::i1, RHS,
                          DAG.getNOT(dl, LHS, MVT::i1));
        break;
      case ISD::SETLT:  // X <s Y  -->  X == 1 & Y == 0  -->  ~Y & X
      case ISD::SETUGT: // X >u Y  -->  X == 1 & Y == 0  -->  ~Y & X
        Ret = DAG.getNode(ISD::AND, dl, MVT::i1, LHS,
                          DAG.getNOT(dl, RHS, MVT::i1));
        break;
      case ISD::SETULE: // X <=u Y  -->  X == 0 | Y == 1  -->  ~X | Y
      case ISD::SETGE:  // X >=s Y  -->  X == 0 | Y == 1  -->  ~X | Y
        Ret = DAG.getNode(ISD::OR, dl, MVT::i1, RHS,
                          DAG.getNOT(dl, LHS, MVT::i1));
        break;
      case ISD::SETUGE: // X >=u Y  -->  X == 1 | Y == 0  -->  ~Y | X
      case ISD::SETLE:  // X <=s Y  -->  X == 1 | Y == 0  -->  ~Y | X
        Ret = DAG.getNode(ISD::OR, dl, MVT::i1, LHS,
                          DAG.getNOT(dl, RHS, MVT::i1));
        break;
      }

      LHS = DAG.getZExtOrTrunc(Ret, dl, VT);
      RHS = SDValue();
      CC = SDValue();
      return true;
    }

    ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
    unsigned Opc = 0;
    switch (CCCode) {
    default:
      llvm_unreachable("Don't know how to expand this condition!");
    case ISD::SETUO:
      if (isCondCodeLegal(ISD::SETUNE, OpVT)) {
        CC1 = ISD::SETUNE;
        CC2 = ISD::SETUNE;
        Opc = ISD::OR;
        break;
      }
      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
             "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
      NeedInvert = true;
      [[fallthrough]];
    case ISD::SETO:
      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
             "If SETO is expanded, SETOEQ must be legal!");
      CC1 = ISD::SETOEQ;
      CC2 = ISD::SETOEQ;
      Opc = ISD::AND;
      break;
    case ISD::SETONE:
    case ISD::SETUEQ:
      // If the SETUO or SETO CC isn't legal, we might be able to use
      // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
      // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
      // the operands.
      CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
      if (!isCondCodeLegal(CC2, OpVT) && (isCondCodeLegal(ISD::SETOGT, OpVT) ||
                                          isCondCodeLegal(ISD::SETOLT, OpVT))) {
        CC1 = ISD::SETOGT;
        CC2 = ISD::SETOLT;
        Opc = ISD::OR;
        NeedInvert = ((unsigned)CCCode & 0x8U);
        break;
      }
      [[fallthrough]];
    case ISD::SETOEQ:
    case ISD::SETOGT:
    case ISD::SETOGE:
    case ISD::SETOLT:
    case ISD::SETOLE:
    case ISD::SETUNE:
    case ISD::SETUGT:
    case ISD::SETUGE:
    case ISD::SETULT:
    case ISD::SETULE:
      // If we are floating point, assign and break, otherwise fall through.
      if (!OpVT.isInteger()) {
        // We can use the 4th bit to tell if we are the unordered
        // or ordered version of the opcode.
        CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
        Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
        CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
        break;
      }
      // Fallthrough if we are unsigned integer.
      [[fallthrough]];
    case ISD::SETLE:
    case ISD::SETGT:
    case ISD::SETGE:
    case ISD::SETLT:
    case ISD::SETNE:
    case ISD::SETEQ:
      // If all combinations of inverting the condition and swapping operands
      // didn't work then we have no means to expand the condition.
      llvm_unreachable("Don't know how to expand this condition!");
    }

    SDValue SetCC1, SetCC2;
    if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
      // If we aren't the ordered or unorder operation,
      // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
      if (IsNonVP) {
        SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
        SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
      } else {
        SetCC1 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC1, Mask, EVL);
        SetCC2 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC2, Mask, EVL);
      }
    } else {
      // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
      if (IsNonVP) {
        SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
        SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
      } else {
        SetCC1 = DAG.getSetCCVP(dl, VT, LHS, LHS, CC1, Mask, EVL);
        SetCC2 = DAG.getSetCCVP(dl, VT, RHS, RHS, CC2, Mask, EVL);
      }
    }
    if (Chain)
      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
                          SetCC2.getValue(1));
    if (IsNonVP)
      LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
    else {
      // Transform the binary opcode to the VP equivalent.
      assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode");
      Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND;
      LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL);
    }
    RHS = SDValue();
    CC = SDValue();
    return true;
  }
  }
  return false;
}

SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
                                                      SelectionDAG &DAG) const {
  EVT VT = Node->getValueType(0);
  // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
  // split into two equal parts.
  if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(2))
    return SDValue();

  // Restrict expansion to cases where both parts can be concatenated.
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
  if (LoVT != HiVT || !isTypeLegal(LoVT))
    return SDValue();

  SDLoc DL(Node);
  unsigned Opcode = Node->getOpcode();

  // Don't expand if the result is likely to be unrolled anyway.
  if (!isOperationLegalOrCustomOrPromote(Opcode, LoVT))
    return SDValue();

  SmallVector<SDValue, 4> LoOps, HiOps;
  for (const SDValue &V : Node->op_values()) {
    auto [Lo, Hi] = DAG.SplitVector(V, DL, LoVT, HiVT);
    LoOps.push_back(Lo);
    HiOps.push_back(Hi);
  }

  SDValue SplitOpLo = DAG.getNode(Opcode, DL, LoVT, LoOps, Node->getFlags());
  SDValue SplitOpHi = DAG.getNode(Opcode, DL, HiVT, HiOps, Node->getFlags());
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SplitOpLo, SplitOpHi);
}

SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
                                                     const SDLoc &DL,
                                                     EVT InVecVT, SDValue EltNo,
                                                     LoadSDNode *OriginalLoad,
                                                     SelectionDAG &DAG) const {
  assert(OriginalLoad->isSimple());

  EVT VecEltVT = InVecVT.getVectorElementType();

  // If the vector element type is not a multiple of a byte then we are unable
  // to correctly compute an address to load only the extracted element as a
  // scalar.
  if (!VecEltVT.isByteSized())
    return SDValue();

  ISD::LoadExtType ExtTy =
      ResultVT.bitsGT(VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
  if (!isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
    return SDValue();

  std::optional<unsigned> ByteOffset;
  Align Alignment = OriginalLoad->getAlign();
  MachinePointerInfo MPI;
  if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
    int Elt = ConstEltNo->getZExtValue();
    ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
    MPI = OriginalLoad->getPointerInfo().getWithOffset(*ByteOffset);
    Alignment = commonAlignment(Alignment, *ByteOffset);
  } else {
    // Discard the pointer info except the address space because the memory
    // operand can't represent this new access since the offset is variable.
    MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
    Alignment = commonAlignment(Alignment, VecEltVT.getSizeInBits() / 8);
  }

  if (!shouldReduceLoadWidth(OriginalLoad, ExtTy, VecEltVT, ByteOffset))
    return SDValue();

  unsigned IsFast = 0;
  if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VecEltVT,
                          OriginalLoad->getAddressSpace(), Alignment,
                          OriginalLoad->getMemOperand()->getFlags(), &IsFast) ||
      !IsFast)
    return SDValue();

  // The original DAG loaded the entire vector from memory, so arithmetic
  // within it must be inbounds.
  SDValue NewPtr = getInboundsVectorElementPointer(
      DAG, OriginalLoad->getBasePtr(), InVecVT, EltNo);

  // We are replacing a vector load with a scalar load. The new load must have
  // identical memory op ordering to the original.
  SDValue Load;
  if (ResultVT.bitsGT(VecEltVT)) {
    // If the result type of vextract is wider than the load, then issue an
    // extending load instead.
    ISD::LoadExtType ExtType =
        isLoadLegal(ResultVT, VecEltVT, Alignment,
                    OriginalLoad->getAddressSpace(), ISD::ZEXTLOAD, false)
            ? ISD::ZEXTLOAD
            : ISD::EXTLOAD;
    Load = DAG.getExtLoad(ExtType, DL, ResultVT, OriginalLoad->getChain(),
                          NewPtr, MPI, VecEltVT, Alignment,
                          OriginalLoad->getMemOperand()->getFlags(),
                          OriginalLoad->getAAInfo());
    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
  } else {
    // The result type is narrower or the same width as the vector element
    Load = DAG.getLoad(VecEltVT, DL, OriginalLoad->getChain(), NewPtr, MPI,
                       Alignment, OriginalLoad->getMemOperand()->getFlags(),
                       OriginalLoad->getAAInfo());
    DAG.makeEquivalentMemoryOrdering(OriginalLoad, Load);
    if (ResultVT.bitsLT(VecEltVT))
      Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load);
    else
      Load = DAG.getBitcast(ResultVT, Load);
  }

  return Load;
}

// Set type id for call site info and metadata 'call_target'.
// We are filtering for:
// a) The call-graph-section use case that wants to know about indirect
//    calls, or
// b) We want to annotate indirect calls.
void TargetLowering::setTypeIdForCallsiteInfo(
    const CallBase *CB, MachineFunction &MF,
    MachineFunction::CallSiteInfo &CSInfo) const {
  if (CB && CB->isIndirectCall() &&
      (MF.getTarget().Options.EmitCallGraphSection ||
       MF.getTarget().Options.EmitCallSiteInfo))
    CSInfo = MachineFunction::CallSiteInfo(*CB);
}