//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements the WebAssemblyTargetLowering class.
///
//===----------------------------------------------------------------------===//

#include "WebAssemblyISelLowering.h"
#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
#include "Utils/WebAssemblyTypeUtilities.h"
#include "WebAssemblyMachineFunctionInfo.h"
#include "WebAssemblySubtarget.h"
#include "WebAssemblyTargetMachine.h"
#include "WebAssemblyUtilities.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
using namespace llvm;

#define DEBUG_TYPE "wasm-lower"

WebAssemblyTargetLowering::WebAssemblyTargetLowering(
    const TargetMachine &TM, const WebAssemblySubtarget &STI)
    : TargetLowering(TM, STI), Subtarget(&STI) {
  auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;

  // Set the load count for memcmp expand optimization
  MaxLoadsPerMemcmp = 8;
  MaxLoadsPerMemcmpOptSize = 4;

  // Booleans always contain 0 or 1.
  setBooleanContents(ZeroOrOneBooleanContent);
  // Except in SIMD vectors
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
  // We don't know the microarchitecture here, so just reduce register pressure.
  setSchedulingPreference(Sched::RegPressure);
  // Tell ISel that we have a stack pointer.
  setStackPointerRegisterToSaveRestore(
      Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
  // Set up the register classes.
  addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
  addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
  addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
  addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
  if (Subtarget->hasSIMD128()) {
    addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
    addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
    addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
    addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
    addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
    addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
  }
  if (Subtarget->hasFP16()) {
    addRegisterClass(MVT::v8f16, &WebAssembly::V128RegClass);
  }
  if (Subtarget->hasReferenceTypes()) {
    addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
    addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
    if (Subtarget->hasExceptionHandling()) {
      addRegisterClass(MVT::exnref, &WebAssembly::EXNREFRegClass);
    }
  }
  // Compute derived properties from the register classes.
  computeRegisterProperties(Subtarget->getRegisterInfo());

  // Transform loads and stores to pointers in address space 1 to loads and
  // stores to WebAssembly global variables, outside linear memory.
  for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
    setOperationAction(ISD::LOAD, T, Custom);
    setOperationAction(ISD::STORE, T, Custom);
  }
  if (Subtarget->hasSIMD128()) {
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64}) {
      setOperationAction(ISD::LOAD, T, Custom);
      setOperationAction(ISD::STORE, T, Custom);
    }
  }
  if (Subtarget->hasFP16()) {
    setOperationAction(ISD::LOAD, MVT::v8f16, Custom);
    setOperationAction(ISD::STORE, MVT::v8f16, Custom);
  }
  if (Subtarget->hasReferenceTypes()) {
    // We need custom load and store lowering for both externref, funcref and
    // Other. The MVT::Other here represents tables of reference types.
    for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
      setOperationAction(ISD::LOAD, T, Custom);
      setOperationAction(ISD::STORE, T, Custom);
    }
  }

  setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);
  setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
  setOperationAction(ISD::JumpTable, MVTPtr, Custom);
  setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
  setOperationAction(ISD::BRIND, MVT::Other, Custom);
  setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);

  // Take the default expansion for va_arg, va_copy, and va_end. There is no
  // default action for va_start, so we do that custom.
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction(ISD::VAARG, MVT::Other, Expand);
  setOperationAction(ISD::VACOPY, MVT::Other, Expand);
  setOperationAction(ISD::VAEND, MVT::Other, Expand);

  for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {
    if (!Subtarget->hasFP16() && T == MVT::v8f16) {
      continue;
    }
    // Don't expand the floating-point types to constant pools.
    setOperationAction(ISD::ConstantFP, T, Legal);
    // Expand floating-point comparisons.
    for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
                    ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
      setCondCodeAction(CC, T, Expand);
    // Expand floating-point library function operators.
    for (auto Op : {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FMA})
      setOperationAction(Op, T, Expand);
    // Expand vector FREM, but use a libcall rather than an expansion for scalar
    if (MVT(T).isVector())
      setOperationAction(ISD::FREM, T, Expand);
    else
      setOperationAction(ISD::FREM, T, LibCall);
    // Note supported floating-point library function operators that otherwise
    // default to expand.
    for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
                    ISD::FRINT, ISD::FROUNDEVEN})
      setOperationAction(Op, T, Legal);
    // Support minimum and maximum, which otherwise default to expand.
    setOperationAction(ISD::FMINIMUM, T, Legal);
    setOperationAction(ISD::FMAXIMUM, T, Legal);
    if (Subtarget->hasSIMD128() && MVT(T).isVector()) {
      setOperationAction(ISD::PSEUDO_FMIN, T, Legal);
      setOperationAction(ISD::PSEUDO_FMAX, T, Legal);
    }
    // When experimental v8f16 support is enabled these instructions don't need
    // to be expanded.
    if (T != MVT::v8f16) {
      setOperationAction(ISD::FP16_TO_FP, T, Expand);
      setOperationAction(ISD::FP_TO_FP16, T, Expand);
    }
    if (Subtarget->hasFP16() && T == MVT::f32) {
      setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Legal);
      setTruncStoreAction(T, MVT::f16, Legal);
    } else {
      setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
      setTruncStoreAction(T, MVT::f16, Expand);
    }
  }

  // Expand unavailable integer operations.
  for (auto Op :
       {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
        ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
        ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
    for (auto T : {MVT::i32, MVT::i64})
      setOperationAction(Op, T, Expand);
    if (Subtarget->hasSIMD128())
      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
        setOperationAction(Op, T, Expand);
  }

  if (Subtarget->hasWideArithmetic()) {
    setOperationAction(ISD::ADD, MVT::i128, Custom);
    setOperationAction(ISD::SUB, MVT::i128, Custom);
    setOperationAction(ISD::SMUL_LOHI, MVT::i64, Custom);
    setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom);
    setOperationAction(ISD::UADDO, MVT::i64, Custom);
  }

  if (Subtarget->hasNontrappingFPToInt())
    for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
      for (auto T : {MVT::i32, MVT::i64})
        setOperationAction(Op, T, Custom);

  if (Subtarget->hasRelaxedSIMD()) {
    setOperationAction(
        {ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM},
        {MVT::v4f32, MVT::v2f64}, Custom);
  }

  // Combine expands these operations, because wasi-libc and emscripten do not
  // yet have the dedicated libcalls.
  setTargetDAGCombine(
      {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM});

  // SIMD-specific configuration
  if (Subtarget->hasSIMD128()) {

    setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);

    // Combine wide-vector muls, with extend inputs, to extmul_half.
    setTargetDAGCombine(ISD::MUL);
    setTargetDAGCombine(ISD::SHL);

    // Combine vector mask reductions into alltrue/anytrue
    setTargetDAGCombine(ISD::SETCC);

    // Convert vector to integer bitcasts to bitmask
    setTargetDAGCombine(ISD::BITCAST);

    // Hoist bitcasts out of shuffles
    setTargetDAGCombine(ISD::VECTOR_SHUFFLE);

    // Combine extends of extract_subvectors into widening ops
    setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND});

    // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
    // conversions ops
    setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND,
                         ISD::EXTRACT_SUBVECTOR});

    // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
    // into conversion ops
    setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
                         ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_ROUND,
                         ISD::CONCAT_VECTORS});

    setTargetDAGCombine(ISD::TRUNCATE);

    // Support saturating add/sub for i8x16 and i16x8
    for (auto Op : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
      for (auto T : {MVT::v16i8, MVT::v8i16})
        setOperationAction(Op, T, Legal);

    // Support integer abs
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(ISD::ABS, T, Legal);

    // Custom lower BUILD_VECTORs to minimize number of replace_lanes
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64})
      setOperationAction(ISD::BUILD_VECTOR, T, Custom);

    if (Subtarget->hasFP16()) {
      setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::f16, Custom);
      setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
    }

    // We have custom shuffle lowering to expose the shuffle mask
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64})
      setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);

    if (Subtarget->hasFP16())
      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f16, Custom);

    // Support splatting
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64})
      setOperationAction(ISD::SPLAT_VECTOR, T, Legal);

    setOperationAction(ISD::AVGCEILU, {MVT::v8i16, MVT::v16i8}, Legal);

    // Custom lowering since wasm shifts must have a scalar shift amount
    for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
        setOperationAction(Op, T, Custom);

    // Custom lower lane accesses to expand out variable indices
    for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                     MVT::v2f64})
        setOperationAction(Op, T, Custom);

    // There is no i8x16.mul instruction
    setOperationAction(ISD::MUL, MVT::v16i8, Expand);

    // Expand integer operations supported for scalars but not SIMD
    for (auto Op :
         {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
        setOperationAction(Op, T, Expand);

    // But we do have integer min and max operations
    for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
        setOperationAction(Op, T, Legal);

    // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
    setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);
    setOperationAction(ISD::CTLZ, MVT::v16i8, Expand);
    setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);

    // Custom lower bit counting operations for other types to scalarize them.
    for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
      for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
        setOperationAction(Op, T, Custom);

    // Expand float operations supported for scalars but not SIMD
    for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
                    ISD::FEXP, ISD::FEXP2, ISD::FEXP10})
      for (auto T : {MVT::v4f32, MVT::v2f64})
        setOperationAction(Op, T, Expand);

    // Unsigned comparison operations are unavailable for i64x2 vectors.
    for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
      setCondCodeAction(CC, MVT::v2i64, Custom);

    // 64x2 conversions are not in the spec
    for (auto Op :
         {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
      for (auto T : {MVT::v2i64, MVT::v2f64})
        setOperationAction(Op, T, Expand);

    // But saturating fp_to_int conversions are
    for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) {
      setOperationAction(Op, MVT::v4i32, Custom);
      if (Subtarget->hasFP16()) {
        setOperationAction(Op, MVT::v8i16, Custom);
      }
    }

    // Support vector extending
    for (auto T : MVT::integer_fixedlen_vector_valuetypes()) {
      setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Custom);
    }

    if (Subtarget->hasFP16()) {
      setOperationAction(ISD::FMA, MVT::v8f16, Legal);
    }

    if (Subtarget->hasRelaxedSIMD()) {
      setOperationAction(ISD::FMULADD, MVT::v4f32, Legal);
      setOperationAction(ISD::FMULADD, MVT::v2f64, Legal);
    }

    // Partial MLA reductions.
    for (auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {
      setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v16i8, Legal);
      setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v8i16, Legal);
    }
  }

  // As a special case, these operators use the type to mean the type to
  // sign-extend from.
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
  if (!Subtarget->hasSignExt()) {
    // Sign extends are legal only when extending a vector extract
    auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
    for (auto T : {MVT::i8, MVT::i16, MVT::i32})
      setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
  }
  for (auto T : MVT::integer_fixedlen_vector_valuetypes())
    setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);

  // Dynamic stack allocation: use the default expansion.
  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);

  setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
  setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
  setOperationAction(ISD::CopyToReg, MVT::Other, Custom);

  // Expand these forms; we pattern-match the forms that we can handle in isel.
  for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
    for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
      setOperationAction(Op, T, Expand);

  if (Subtarget->hasReferenceTypes())
    for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
      for (auto T : {MVT::externref, MVT::funcref})
        setOperationAction(Op, T, Expand);

  // There is no vector conditional select instruction
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64, MVT::v8f16})
    setOperationAction(ISD::SELECT_CC, T, Expand);

  // We have custom switch handling.
  setOperationAction(ISD::BR_JT, MVT::Other, Custom);

  // WebAssembly doesn't have:
  //  - Floating-point extending loads.
  //  - Floating-point truncating stores.
  //  - i1 extending loads.
  //  - truncating SIMD stores and most extending loads
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
  for (auto T : MVT::integer_valuetypes())
    for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
      setLoadExtAction(Ext, T, MVT::i1, Promote);
  if (Subtarget->hasSIMD128()) {
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
                   MVT::v2f64}) {
      for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
        if (MVT(T) != MemT) {
          setTruncStoreAction(T, MemT, Expand);
          for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
            setLoadExtAction(Ext, T, MemT, Expand);
        }
      }
    }
    // But some vector extending loads are legal
    for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
      setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
      setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
      setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
    }
    setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
  }

  // Don't do anything clever with build_pairs
  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);

  // Trap lowers to wasm unreachable
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);

  // Exception handling intrinsics
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);

  setMaxAtomicSizeInBitsSupported(64);

  // Always convert switches to br_tables unless there is only one case, which
  // is equivalent to a simple branch. This reduces code size for wasm, and we
  // defer possible jump table optimizations to the VM.
  setMinimumJumpTableEntries(2);
}

TargetLowering::AtomicExpansionKind
WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(
    const AtomicRMWInst *AI) const {
  // We have wasm instructions for these
  switch (AI->getOperation()) {
  case AtomicRMWInst::Add:
  case AtomicRMWInst::Sub:
  case AtomicRMWInst::And:
  case AtomicRMWInst::Or:
  case AtomicRMWInst::Xor:
  case AtomicRMWInst::Xchg:
    return AtomicExpansionKind::None;
  default:
    break;
  }
  return AtomicExpansionKind::CmpXChg;
}

bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
  // Implementation copied from X86TargetLowering.
  unsigned Opc = VecOp.getOpcode();

  // Assume target opcodes can't be scalarized.
  // TODO - do we have any exceptions?
  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
    return false;

  // If the vector op is not supported, try to convert to scalar.
  EVT VecVT = VecOp.getValueType();
  if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
    return true;

  // If the vector op is supported, but the scalar op is not, the transform may
  // not be worthwhile.
  EVT ScalarVT = VecVT.getScalarType();
  return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
}

FastISel *WebAssemblyTargetLowering::createFastISel(
    FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
    const LibcallLoweringInfo *LibcallLowering) const {
  return WebAssembly::createFastISel(FuncInfo, LibInfo, LibcallLowering);
}

MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
                                                      EVT VT) const {
  unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
  if (BitWidth > 1 && BitWidth < 8)
    BitWidth = 8;

  if (BitWidth > 64) {
    // The shift will be lowered to a libcall, and compiler-rt libcalls expect
    // the count to be an i32.
    BitWidth = 32;
    assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&
           "32-bit shift counts ought to be enough for anyone");
  }

  MVT Result = MVT::getIntegerVT(BitWidth);
  assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&
         "Unable to represent scalar shift amount type");
  return Result;
}

// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
// undefined result on invalid/overflow, to the WebAssembly opcode, which
// traps on invalid/overflow.
static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
                                       MachineBasicBlock *BB,
                                       const TargetInstrInfo &TII,
                                       bool IsUnsigned, bool Int64,
                                       bool Float64, unsigned LoweredOpcode) {
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();

  Register OutReg = MI.getOperand(0).getReg();
  Register InReg = MI.getOperand(1).getReg();

  unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
  unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
  unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
  unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
  unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
  unsigned Eqz = WebAssembly::EQZ_I32;
  unsigned And = WebAssembly::AND_I32;
  int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
  int64_t Substitute = IsUnsigned ? 0 : Limit;
  double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
  auto &Context = BB->getParent()->getFunction().getContext();
  Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);

  const BasicBlock *LLVMBB = BB->getBasicBlock();
  MachineFunction *F = BB->getParent();
  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
  MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);

  MachineFunction::iterator It = ++BB->getIterator();
  F->insert(It, FalseMBB);
  F->insert(It, TrueMBB);
  F->insert(It, DoneMBB);

  // Transfer the remainder of BB and its successor edges to DoneMBB.
  DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

  BB->addSuccessor(TrueMBB);
  BB->addSuccessor(FalseMBB);
  TrueMBB->addSuccessor(DoneMBB);
  FalseMBB->addSuccessor(DoneMBB);

  unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
  Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
  Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
  CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
  TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));

  MI.eraseFromParent();
  // For signed numbers, we can do a single comparison to determine whether
  // fabs(x) is within range.
  if (IsUnsigned) {
    Tmp0 = InReg;
  } else {
    BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
  }
  BuildMI(BB, DL, TII.get(FConst), Tmp1)
      .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
  BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);

  // For unsigned numbers, we have to do a separate comparison with zero.
  if (IsUnsigned) {
    Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
    Register SecondCmpReg =
        MRI.createVirtualRegister(&WebAssembly::I32RegClass);
    Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
    BuildMI(BB, DL, TII.get(FConst), Tmp1)
        .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
    BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
    BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
    CmpReg = AndReg;
  }

  BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);

  // Create the CFG diamond to select between doing the conversion or using
  // the substitute value.
  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
  BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
  BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
  BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
  BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
      .addReg(FalseReg)
      .addMBB(FalseMBB)
      .addReg(TrueReg)
      .addMBB(TrueMBB);

  return DoneMBB;
}

// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`
// instruction to handle the zero-length case.
static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL,
                                      MachineBasicBlock *BB,
                                      const TargetInstrInfo &TII, bool Int64) {
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();

  MachineOperand DstMem = MI.getOperand(0);
  MachineOperand SrcMem = MI.getOperand(1);
  MachineOperand Dst = MI.getOperand(2);
  MachineOperand Src = MI.getOperand(3);
  MachineOperand Len = MI.getOperand(4);

  // If the length is a constant, we don't actually need the check.
  if (MachineInstr *Def = MRI.getVRegDef(Len.getReg())) {
    if (Def->getOpcode() == WebAssembly::CONST_I32 ||
        Def->getOpcode() == WebAssembly::CONST_I64) {
      if (Def->getOperand(1).getImm() == 0) {
        // A zero-length memcpy is a no-op.
        MI.eraseFromParent();
        return BB;
      }
      // A non-zero-length memcpy doesn't need a zero check.
      unsigned MemoryCopy =
          Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
      BuildMI(*BB, MI, DL, TII.get(MemoryCopy))
          .add(DstMem)
          .add(SrcMem)
          .add(Dst)
          .add(Src)
          .add(Len);
      MI.eraseFromParent();
      return BB;
    }
  }

  // We're going to add an extra use to `Len` to test if it's zero; that
  // use shouldn't be a kill, even if the original use is.
  MachineOperand NoKillLen = Len;
  NoKillLen.setIsKill(false);

  // Decide on which `MachineInstr` opcode we're going to use.
  unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
  unsigned MemoryCopy =
      Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;

  // Create two new basic blocks; one for the new `memory.fill` that we can
  // branch over, and one for the rest of the instructions after the original
  // `memory.fill`.
  const BasicBlock *LLVMBB = BB->getBasicBlock();
  MachineFunction *F = BB->getParent();
  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);

  MachineFunction::iterator It = ++BB->getIterator();
  F->insert(It, TrueMBB);
  F->insert(It, DoneMBB);

  // Transfer the remainder of BB and its successor edges to DoneMBB.
  DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

  // Connect the CFG edges.
  BB->addSuccessor(TrueMBB);
  BB->addSuccessor(DoneMBB);
  TrueMBB->addSuccessor(DoneMBB);

  // Create a virtual register for the `Eqz` result.
  unsigned EqzReg;
  EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);

  // Erase the original `memory.copy`.
  MI.eraseFromParent();

  // Test if `Len` is zero.
  BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);

  // Insert a new `memory.copy`.
  BuildMI(TrueMBB, DL, TII.get(MemoryCopy))
      .add(DstMem)
      .add(SrcMem)
      .add(Dst)
      .add(Src)
      .add(Len);

  // Create the CFG triangle.
  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
  BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);

  return DoneMBB;
}

// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`
// instruction to handle the zero-length case.
static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL,
                                      MachineBasicBlock *BB,
                                      const TargetInstrInfo &TII, bool Int64) {
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();

  MachineOperand Mem = MI.getOperand(0);
  MachineOperand Dst = MI.getOperand(1);
  MachineOperand Val = MI.getOperand(2);
  MachineOperand Len = MI.getOperand(3);

  // If the length is a constant, we don't actually need the check.
  if (MachineInstr *Def = MRI.getVRegDef(Len.getReg())) {
    if (Def->getOpcode() == WebAssembly::CONST_I32 ||
        Def->getOpcode() == WebAssembly::CONST_I64) {
      if (Def->getOperand(1).getImm() == 0) {
        // A zero-length memset is a no-op.
        MI.eraseFromParent();
        return BB;
      }
      // A non-zero-length memset doesn't need a zero check.
      unsigned MemoryFill =
          Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
      BuildMI(*BB, MI, DL, TII.get(MemoryFill))
          .add(Mem)
          .add(Dst)
          .add(Val)
          .add(Len);
      MI.eraseFromParent();
      return BB;
    }
  }

  // We're going to add an extra use to `Len` to test if it's zero; that
  // use shouldn't be a kill, even if the original use is.
  MachineOperand NoKillLen = Len;
  NoKillLen.setIsKill(false);

  // Decide on which `MachineInstr` opcode we're going to use.
  unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
  unsigned MemoryFill =
      Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;

  // Create two new basic blocks; one for the new `memory.fill` that we can
  // branch over, and one for the rest of the instructions after the original
  // `memory.fill`.
  const BasicBlock *LLVMBB = BB->getBasicBlock();
  MachineFunction *F = BB->getParent();
  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);

  MachineFunction::iterator It = ++BB->getIterator();
  F->insert(It, TrueMBB);
  F->insert(It, DoneMBB);

  // Transfer the remainder of BB and its successor edges to DoneMBB.
  DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

  // Connect the CFG edges.
  BB->addSuccessor(TrueMBB);
  BB->addSuccessor(DoneMBB);
  TrueMBB->addSuccessor(DoneMBB);

  // Create a virtual register for the `Eqz` result.
  unsigned EqzReg;
  EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);

  // Erase the original `memory.fill`.
  MI.eraseFromParent();

  // Test if `Len` is zero.
  BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);

  // Insert a new `memory.copy`.
  BuildMI(TrueMBB, DL, TII.get(MemoryFill)).add(Mem).add(Dst).add(Val).add(Len);

  // Create the CFG triangle.
  BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
  BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);

  return DoneMBB;
}

static MachineBasicBlock *
LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
                 const WebAssemblySubtarget *Subtarget,
                 const TargetInstrInfo &TII) {
  assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
         CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
  bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
  MachineInstr &CallParams = *CallResults.getPrevNode();
  assert(CallParams.getOpcode() ==
         (IsRetCall ? WebAssembly::RET_CALL_PARAMS : WebAssembly::CALL_PARAMS));

  bool IsIndirect =
      CallParams.getOperand(0).isReg() || CallParams.getOperand(0).isFI();

  bool IsFuncrefCall = false;
  if (IsIndirect && CallParams.getOperand(0).isReg()) {
    Register Reg = CallParams.getOperand(0).getReg();
    const MachineFunction *MF = BB->getParent();
    const MachineRegisterInfo &MRI = MF->getRegInfo();
    const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
    IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
    assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
  }

  unsigned CallOp;
  if (IsIndirect && IsRetCall) {
    CallOp = WebAssembly::RET_CALL_INDIRECT;
  } else if (IsIndirect) {
    CallOp = WebAssembly::CALL_INDIRECT;
  } else if (IsRetCall) {
    CallOp = WebAssembly::RET_CALL;
  } else {
    CallOp = WebAssembly::CALL;
  }

  MachineFunction &MF = *BB->getParent();
  const MCInstrDesc &MCID = TII.get(CallOp);
  MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));

  // Move the function pointer to the end of the arguments for indirect calls
  if (IsIndirect) {
    auto FnPtr = CallParams.getOperand(0);
    CallParams.removeOperand(0);

    // For funcrefs, call_indirect is done through __funcref_call_table and the
    // funcref is always installed in slot 0 of the table, therefore instead of
    // having the function pointer added at the end of the params list, a zero
    // (the index in
    // __funcref_call_table is added).
    if (IsFuncrefCall) {
      Register RegZero =
          MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
      MachineInstrBuilder MIBC0 =
          BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);

      BB->insert(CallResults.getIterator(), MIBC0);
      MachineInstrBuilder(MF, CallParams).addReg(RegZero);
    } else
      CallParams.addOperand(FnPtr);
  }

  for (auto Def : CallResults.defs())
    MIB.add(Def);

  if (IsIndirect) {
    // Placeholder for the type index.
    // This gets replaced with the correct value in WebAssemblyMCInstLower.cpp
    MIB.addImm(0);
    // The table into which this call_indirect indexes.
    MCSymbolWasm *Table = IsFuncrefCall
                              ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
                                    MF.getContext(), Subtarget)
                              : WebAssembly::getOrCreateFunctionTableSymbol(
                                    MF.getContext(), Subtarget);
    if (Subtarget->hasCallIndirectOverlong()) {
      MIB.addSym(Table);
    } else {
      // For the MVP there is at most one table whose number is 0, but we can't
      // write a table symbol or issue relocations.  Instead we just ensure the
      // table is live and write a zero.
      Table->setNoStrip();
      MIB.addImm(0);
    }
  }

  // Avoid duplicating the implicit operands.
  for (auto Use : CallParams.explicit_uses())
    MIB.add(Use);

  BB->insert(CallResults.getIterator(), MIB);
  CallParams.eraseFromParent();
  CallResults.eraseFromParent();

  // If this is a funcref call, to avoid hidden GC roots, we need to clear the
  // table slot with ref.null upon call_indirect return.
  //
  // This generates the following code, which comes right after a call_indirect
  // of a funcref:
  //
  //    i32.const 0
  //    ref.null func
  //    table.set __funcref_call_table
  if (IsIndirect && IsFuncrefCall) {
    MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
        MF.getContext(), Subtarget);
    Register RegZero =
        MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
    MachineInstr *Const0 =
        BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
    BB->insertAfter(MIB.getInstr()->getIterator(), Const0);

    Register RegFuncref =
        MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
    MachineInstr *RefNull =
        BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
    BB->insertAfter(Const0->getIterator(), RefNull);

    MachineInstr *TableSet =
        BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
            .addSym(Table)
            .addReg(RegZero)
            .addReg(RegFuncref);
    BB->insertAfter(RefNull->getIterator(), TableSet);
  }

  return BB;
}

MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
    MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
  DebugLoc DL = MI.getDebugLoc();

  switch (MI.getOpcode()) {
  default:
    llvm_unreachable("Unexpected instr type to insert");
  case WebAssembly::FP_TO_SINT_I32_F32:
    return LowerFPToInt(MI, DL, BB, TII, false, false, false,
                        WebAssembly::I32_TRUNC_S_F32);
  case WebAssembly::FP_TO_UINT_I32_F32:
    return LowerFPToInt(MI, DL, BB, TII, true, false, false,
                        WebAssembly::I32_TRUNC_U_F32);
  case WebAssembly::FP_TO_SINT_I64_F32:
    return LowerFPToInt(MI, DL, BB, TII, false, true, false,
                        WebAssembly::I64_TRUNC_S_F32);
  case WebAssembly::FP_TO_UINT_I64_F32:
    return LowerFPToInt(MI, DL, BB, TII, true, true, false,
                        WebAssembly::I64_TRUNC_U_F32);
  case WebAssembly::FP_TO_SINT_I32_F64:
    return LowerFPToInt(MI, DL, BB, TII, false, false, true,
                        WebAssembly::I32_TRUNC_S_F64);
  case WebAssembly::FP_TO_UINT_I32_F64:
    return LowerFPToInt(MI, DL, BB, TII, true, false, true,
                        WebAssembly::I32_TRUNC_U_F64);
  case WebAssembly::FP_TO_SINT_I64_F64:
    return LowerFPToInt(MI, DL, BB, TII, false, true, true,
                        WebAssembly::I64_TRUNC_S_F64);
  case WebAssembly::FP_TO_UINT_I64_F64:
    return LowerFPToInt(MI, DL, BB, TII, true, true, true,
                        WebAssembly::I64_TRUNC_U_F64);
  case WebAssembly::MEMCPY_A32:
    return LowerMemcpy(MI, DL, BB, TII, false);
  case WebAssembly::MEMCPY_A64:
    return LowerMemcpy(MI, DL, BB, TII, true);
  case WebAssembly::MEMSET_A32:
    return LowerMemset(MI, DL, BB, TII, false);
  case WebAssembly::MEMSET_A64:
    return LowerMemset(MI, DL, BB, TII, true);
  case WebAssembly::CALL_RESULTS:
  case WebAssembly::RET_CALL_RESULTS:
    return LowerCallResults(MI, DL, BB, Subtarget, TII);
  }
}

std::pair<unsigned, const TargetRegisterClass *>
WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
    const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
  // First, see if this is a constraint that directly corresponds to a
  // WebAssembly register class.
  if (Constraint.size() == 1) {
    switch (Constraint[0]) {
    case 'r':
      assert(VT != MVT::iPTR && "Pointer MVT not expected here");
      if (Subtarget->hasSIMD128() && VT.isVector()) {
        if (VT.getSizeInBits() == 128)
          return std::make_pair(0U, &WebAssembly::V128RegClass);
      }
      if (VT.isInteger() && !VT.isVector()) {
        if (VT.getSizeInBits() <= 32)
          return std::make_pair(0U, &WebAssembly::I32RegClass);
        if (VT.getSizeInBits() <= 64)
          return std::make_pair(0U, &WebAssembly::I64RegClass);
      }
      if (VT.isFloatingPoint() && !VT.isVector()) {
        switch (VT.getSizeInBits()) {
        case 32:
          return std::make_pair(0U, &WebAssembly::F32RegClass);
        case 64:
          return std::make_pair(0U, &WebAssembly::F64RegClass);
        default:
          break;
        }
      }
      break;
    default:
      break;
    }
  }

  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
}

bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
  // Assume ctz is a relatively cheap operation.
  return true;
}

bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
  // Assume clz is a relatively cheap operation.
  return true;
}

bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                      const AddrMode &AM,
                                                      Type *Ty, unsigned AS,
                                                      Instruction *I) const {
  // WebAssembly offsets are added as unsigned without wrapping. The
  // isLegalAddressingMode gives us no way to determine if wrapping could be
  // happening, so we approximate this by accepting only non-negative offsets.
  if (AM.BaseOffs < 0)
    return false;

  // WebAssembly has no scale register operands.
  if (AM.Scale != 0)
    return false;

  // Everything else is legal.
  return true;
}

bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
    EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
    MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
  // WebAssembly supports unaligned accesses, though it should be declared
  // with the p2align attribute on loads and stores which do so, and there
  // may be a performance impact. We tell LLVM they're "fast" because
  // for the kinds of things that LLVM uses this for (merging adjacent stores
  // of constants, etc.), WebAssembly implementations will either want the
  // unaligned access or they'll split anyway.
  if (Fast)
    *Fast = 1;
  return true;
}

bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
                                              AttributeList Attr) const {
  // The current thinking is that wasm engines will perform this optimization,
  // so we can save on code size.
  return true;
}

bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
  EVT ExtT = ExtVal.getValueType();
  SDValue N0 = peekThroughFreeze(ExtVal->getOperand(0));
  auto *Load = dyn_cast<LoadSDNode>(N0);
  if (!Load)
    return false;
  EVT MemT = Load->getValueType(0);
  return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
         (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
         (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
}

bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
    const GlobalAddressSDNode *GA) const {
  // Wasm doesn't support function addresses with offsets
  const GlobalValue *GV = GA->getGlobal();
  return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
}

EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
                                                  LLVMContext &C,
                                                  EVT VT) const {
  if (VT.isVector()) {
    if (VT.getVectorElementType() == MVT::f16 && !Subtarget->hasFP16())
      return VT.changeElementType(C, MVT::i1);

    return VT.changeVectorElementTypeToInteger();
  }

  // So far, all branch instructions in Wasm take an I32 condition.
  // The default TargetLowering::getSetCCResultType returns the pointer size,
  // which would be useful to reduce instruction counts when testing
  // against 64-bit pointers/values if at some point Wasm supports that.
  return EVT::getIntegerVT(C, 32);
}

void WebAssemblyTargetLowering::getTgtMemIntrinsic(
    SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  IntrinsicInfo Info;
  switch (Intrinsic) {
  case Intrinsic::wasm_memory_atomic_notify:
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(4);
    // atomic.notify instruction does not really load the memory specified with
    // this argument, but MachineMemOperand should either be load or store, so
    // we set this to a load.
    // FIXME Volatile isn't really correct, but currently all LLVM atomic
    // instructions are treated as volatiles in the backend, so we should be
    // consistent. The same applies for wasm_atomic_wait intrinsics too.
    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::wasm_memory_atomic_wait32:
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(4);
    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::wasm_memory_atomic_wait64:
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i64;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(8);
    Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::wasm_loadf16_f32:
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::f16;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(2);
    Info.flags = MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::wasm_storef16_f32:
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = MVT::f16;
    Info.ptrVal = I.getArgOperand(1);
    Info.offset = 0;
    Info.align = Align(2);
    Info.flags = MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  default:
    return;
  }
}

void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
    const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
    const SelectionDAG &DAG, unsigned Depth) const {
  switch (Op.getOpcode()) {
  default:
    break;
  case ISD::INTRINSIC_WO_CHAIN: {
    unsigned IntNo = Op.getConstantOperandVal(0);
    switch (IntNo) {
    default:
      break;
    case Intrinsic::wasm_bitmask: {
      unsigned BitWidth = Known.getBitWidth();
      EVT VT = Op.getOperand(1).getSimpleValueType();
      unsigned PossibleBits = VT.getVectorNumElements();
      APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
      Known.Zero |= ZeroMask;
      break;
    }
    }
    break;
  }
  case WebAssemblyISD::EXTEND_LOW_U:
  case WebAssemblyISD::EXTEND_HIGH_U: {
    // We know the high half, of each destination vector element, will be zero.
    SDValue SrcOp = Op.getOperand(0);
    EVT VT = SrcOp.getSimpleValueType();
    unsigned BitWidth = Known.getBitWidth();
    if (VT == MVT::v8i8 || VT == MVT::v16i8) {
      assert(BitWidth >= 8 && "Unexpected width!");
      APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
      Known.Zero |= Mask;
    } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
      assert(BitWidth >= 16 && "Unexpected width!");
      APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
      Known.Zero |= Mask;
    } else if (VT == MVT::v2i32 || VT == MVT::v4i32) {
      assert(BitWidth >= 32 && "Unexpected width!");
      APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 32);
      Known.Zero |= Mask;
    }
    break;
  }
  // For 128-bit addition if the upper bits are all zero then it's known that
  // the upper bits of the result will have all bits guaranteed zero except the
  // first.
  case WebAssemblyISD::I64_ADD128:
    if (Op.getResNo() == 1) {
      SDValue LHS_HI = Op.getOperand(1);
      SDValue RHS_HI = Op.getOperand(3);
      if (isNullConstant(LHS_HI) && isNullConstant(RHS_HI))
        Known.Zero.setBitsFrom(1);
    }
    break;
  }
}

TargetLoweringBase::LegalizeTypeAction
WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
  if (VT.isFixedLengthVector()) {
    MVT EltVT = VT.getVectorElementType();
    // We have legal vector types with these lane types, so widening the
    // vector would let us use some of the lanes directly without having to
    // extend or truncate values.
    if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
        EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
      return TypeWidenVector;
  }

  return TargetLoweringBase::getPreferredVectorAction(VT);
}

bool WebAssemblyTargetLowering::isFMAFasterThanFMulAndFAdd(
    const MachineFunction &MF, EVT VT) const {
  if (!Subtarget->hasFP16() || !VT.isVector())
    return false;

  EVT ScalarVT = VT.getScalarType();
  if (!ScalarVT.isSimple())
    return false;

  return ScalarVT.getSimpleVT().SimpleTy == MVT::f16;
}

bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
    SDValue Op, const TargetLoweringOpt &TLO) const {
  // ISel process runs DAGCombiner after legalization; this step is called
  // SelectionDAG optimization phase. This post-legalization combining process
  // runs DAGCombiner on each node, and if there was a change to be made,
  // re-runs legalization again on it and its user nodes to make sure
  // everythiing is in a legalized state.
  //
  // The legalization calls lowering routines, and we do our custom lowering for
  // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
  // into zeros. But there is a set of routines in DAGCombiner that turns unused
  // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
  // turns unused vector elements into undefs. But this routine does not work
  // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
  // combination can result in a infinite loop, in which undefs are converted to
  // zeros in legalization and back to undefs in combining.
  //
  // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
  // running for build_vectors.
  if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
    return false;
  return true;
}

//===----------------------------------------------------------------------===//
// WebAssembly Lowering private implementation.
//===----------------------------------------------------------------------===//

//===----------------------------------------------------------------------===//
// Lowering Code
//===----------------------------------------------------------------------===//

static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
  MachineFunction &MF = DAG.getMachineFunction();
  DAG.getContext()->diagnose(
      DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
}

// Test whether the given calling convention is supported.
static bool callingConvSupported(CallingConv::ID CallConv) {
  // We currently support the language-independent target-independent
  // conventions. We don't yet have a way to annotate calls with properties like
  // "cold", and we don't have any call-clobbered registers, so these are mostly
  // all handled the same.
  return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
         CallConv == CallingConv::Cold ||
         CallConv == CallingConv::PreserveMost ||
         CallConv == CallingConv::PreserveAll ||
         CallConv == CallingConv::CXX_FAST_TLS ||
         CallConv == CallingConv::WASM_EmscriptenInvoke ||
         CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail;
}

SDValue
WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                     SmallVectorImpl<SDValue> &InVals) const {
  SelectionDAG &DAG = CLI.DAG;
  SDLoc DL = CLI.DL;
  SDValue Chain = CLI.Chain;
  SDValue Callee = CLI.Callee;
  MachineFunction &MF = DAG.getMachineFunction();
  auto Layout = MF.getDataLayout();

  // A call through a funcref is expressed in IR as a call through the pointer
  // produced by the llvm.wasm.funcref.to_ptr intrinsic. Detect this here and
  // recover the underlying funcref value so the call can be lowered to a
  // table.set + call_indirect through the dedicated __funcref_call_table.
  bool IsFuncrefCall = false;
  if (Callee.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
      Callee.getConstantOperandVal(0) == Intrinsic::wasm_funcref_to_ptr) {
    Callee = Callee.getOperand(1);
    IsFuncrefCall = true;
  }

  CallingConv::ID CallConv = CLI.CallConv;
  if (!callingConvSupported(CallConv))
    fail(DL, DAG,
         "WebAssembly doesn't support language-specific or target-specific "
         "calling conventions yet");
  if (CLI.IsPatchPoint)
    fail(DL, DAG, "WebAssembly doesn't support patch point yet");

  if (CLI.IsTailCall) {
    auto NoTail = [&](const char *Msg) {
      if (CLI.CB && CLI.CB->isMustTailCall())
        fail(DL, DAG, Msg);
      CLI.IsTailCall = false;
    };

    if (!Subtarget->hasTailCall())
      NoTail("WebAssembly 'tail-call' feature not enabled");

    // Varargs calls cannot be tail calls because the buffer is on the stack
    if (CLI.IsVarArg)
      NoTail("WebAssembly does not support varargs tail calls");

    // Do not tail call unless caller and callee return types match
    const Function &F = MF.getFunction();
    const TargetMachine &TM = getTargetMachine();
    Type *RetTy = F.getReturnType();
    SmallVector<MVT, 4> CallerRetTys;
    SmallVector<MVT, 4> CalleeRetTys;
    computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
    computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
    bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
                      std::equal(CallerRetTys.begin(), CallerRetTys.end(),
                                 CalleeRetTys.begin());
    if (!TypesMatch)
      NoTail("WebAssembly tail call requires caller and callee return types to "
             "match");

    // If pointers to local stack values are passed, we cannot tail call
    if (CLI.CB) {
      for (auto &Arg : CLI.CB->args()) {
        Value *Val = Arg.get();
        // Trace the value back through pointer operations
        while (true) {
          Value *Src = Val->stripPointerCastsAndAliases();
          if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
            Src = GEP->getPointerOperand();
          if (Val == Src)
            break;
          Val = Src;
        }
        if (isa<AllocaInst>(Val)) {
          NoTail(
              "WebAssembly does not support tail calling with stack arguments");
          break;
        }
      }
    }

    // A byval argument is copied into this function's stack frame below, and
    // a tail call releases that frame before the callee reads the copy.
    if (llvm::any_of(CLI.Outs, [](const ISD::OutputArg &Out) {
          return Out.Flags.isByVal() && Out.Flags.getByValSize() != 0;
        }))
      NoTail("WebAssembly does not support tail calling with byval arguments");
  }

  SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;

  // The generic code may have added an sret argument. If we're lowering an
  // invoke function, the ABI requires that the function pointer be the first
  // argument, so we may have to swap the arguments.
  if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
      Outs[0].Flags.isSRet()) {
    std::swap(Outs[0], Outs[1]);
    std::swap(OutVals[0], OutVals[1]);
  }

  bool HasSwiftSelfArg = false;
  bool HasSwiftErrorArg = false;
  bool HasSwiftAsyncArg = false;
  unsigned NumFixedArgs = 0;
  for (unsigned I = 0; I < Outs.size(); ++I) {
    const ISD::OutputArg &Out = Outs[I];
    SDValue &OutVal = OutVals[I];
    HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
    HasSwiftErrorArg |= Out.Flags.isSwiftError();
    HasSwiftAsyncArg |= Out.Flags.isSwiftAsync();
    if (Out.Flags.isNest())
      fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
    if (Out.Flags.isInAlloca())
      fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
    if (Out.Flags.isInConsecutiveRegs())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
    if (Out.Flags.isInConsecutiveRegsLast())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
    if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
      auto &MFI = MF.getFrameInfo();
      int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
                                     Out.Flags.getNonZeroByValAlign(),
                                     /*isSS=*/false);
      SDValue SizeNode =
          DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
      SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
      Align Alignment = Out.Flags.getNonZeroByValAlign();
      Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode, Alignment,
                            Alignment,
                            /*isVolatile*/ false, /*AlwaysInline=*/false,
                            /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
                            MachinePointerInfo());
      OutVal = FINode;
    }
    // Count the number of fixed args *after* legalization.
    NumFixedArgs += !Out.Flags.isVarArg();
  }

  bool IsVarArg = CLI.IsVarArg;
  auto PtrVT = getPointerTy(Layout);

  // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
  // (for swifttailcc) swiftasync arguments if there aren't. These additional
  // arguments are also added for callee signature. They are necessary to match
  // callee and caller signature for indirect call.
  if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
    Type *PtrTy = PointerType::getUnqual(*DAG.getContext());
    if (!HasSwiftSelfArg) {
      NumFixedArgs++;
      ISD::ArgFlagsTy Flags;
      Flags.setSwiftSelf();
      ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
      CLI.Outs.push_back(Arg);
      SDValue ArgVal = DAG.getUNDEF(PtrVT);
      CLI.OutVals.push_back(ArgVal);
    }
    if (!HasSwiftErrorArg) {
      NumFixedArgs++;
      ISD::ArgFlagsTy Flags;
      Flags.setSwiftError();
      ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
      CLI.Outs.push_back(Arg);
      SDValue ArgVal = DAG.getUNDEF(PtrVT);
      CLI.OutVals.push_back(ArgVal);
    }
    if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
      NumFixedArgs++;
      ISD::ArgFlagsTy Flags;
      Flags.setSwiftAsync();
      ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
      CLI.Outs.push_back(Arg);
      SDValue ArgVal = DAG.getUNDEF(PtrVT);
      CLI.OutVals.push_back(ArgVal);
    }
  }

  // Analyze operands of the call, assigning locations to each operand.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());

  if (IsVarArg) {
    // Outgoing non-fixed arguments are placed in a buffer. First
    // compute their offsets and the total amount of buffer space needed.
    for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
      const ISD::OutputArg &Out = Outs[I];
      SDValue &Arg = OutVals[I];
      EVT VT = Arg.getValueType();
      assert(VT != MVT::iPTR && "Legalized args should be concrete");
      Type *Ty = VT.getTypeForEVT(*DAG.getContext());
      Align Alignment =
          std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
      unsigned Offset =
          CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
      CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
                                        Offset, VT.getSimpleVT(),
                                        CCValAssign::Full));
    }
  }

  unsigned NumBytes = CCInfo.getAlignedCallFrameSize();

  SDValue FINode;
  if (IsVarArg && NumBytes) {
    // For non-fixed arguments, next emit stores to store the argument values
    // to the stack buffer at the offsets computed above.
    MaybeAlign StackAlign = Layout.getStackAlignment();
    assert(StackAlign && "data layout string is missing stack alignment");
    int FI = MF.getFrameInfo().CreateStackObject(NumBytes, *StackAlign,
                                                 /*isSS=*/false);
    unsigned ValNo = 0;
    SmallVector<SDValue, 8> Chains;
    for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
      assert(ArgLocs[ValNo].getValNo() == ValNo &&
             "ArgLocs should remain in order and only hold varargs args");
      unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
      FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
      SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
                                DAG.getConstant(Offset, DL, PtrVT));
      Chains.push_back(
          DAG.getStore(Chain, DL, Arg, Add,
                       MachinePointerInfo::getFixedStack(MF, FI, Offset)));
    }
    if (!Chains.empty())
      Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
  } else if (IsVarArg) {
    FINode = DAG.getIntPtrConstant(0, DL);
  }

  if (Callee->getOpcode() == ISD::GlobalAddress) {
    // If the callee is a GlobalAddress node (quite common, every direct call
    // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
    // doesn't at MO_GOT which is not needed for direct calls.
    GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);
    Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
                                        getPointerTy(DAG.getDataLayout()),
                                        GA->getOffset());
    Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
                         getPointerTy(DAG.getDataLayout()), Callee);
  }

  // Compute the operands for the CALLn node.
  SmallVector<SDValue, 16> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Callee);

  // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
  // isn't reliable.
  Ops.append(OutVals.begin(),
             IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
  // Add a pointer to the vararg buffer.
  if (IsVarArg)
    Ops.push_back(FINode);

  SmallVector<EVT, 8> InTys;
  for (const auto &In : Ins) {
    assert(!In.Flags.isByVal() && "byval is not valid for return values");
    assert(!In.Flags.isNest() && "nest is not valid for return values");
    if (In.Flags.isInAlloca())
      fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
    if (In.Flags.isInConsecutiveRegs())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
    if (In.Flags.isInConsecutiveRegsLast())
      fail(DL, DAG,
           "WebAssembly hasn't implemented cons regs last return values");
    // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
    // registers.
    InTys.push_back(In.VT);
  }

  // Lastly, if this is a call to a funcref we need to add an instruction
  // table.set to the chain and transform the call.
  if (IsFuncrefCall) {
    // In the absence of function references proposal where a funcref call is
    // lowered to call_ref, using reference types we generate a table.set to set
    // the funcref to a special table used solely for this purpose, followed by
    // a call_indirect. Here we just generate the table set, and return the
    // SDValue of the table.set so that LowerCall can finalize the lowering by
    // generating the call_indirect.
    SDValue Chain = Ops[0];

    MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
        MF.getContext(), Subtarget);
    SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
    SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
    SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
    SDValue TableSet = DAG.getMemIntrinsicNode(
        WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
        MVT::funcref, MachinePointerInfo(), Align(1),
        MachineMemOperand::MOStore);

    Ops[0] = TableSet; // The new chain is the TableSet itself
  }

  if (CLI.IsTailCall) {
    // ret_calls do not return values to the current frame
    SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
    return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
  }

  InTys.push_back(MVT::Other);
  SDVTList InTyList = DAG.getVTList(InTys);
  SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);

  for (size_t I = 0; I < Ins.size(); ++I)
    InVals.push_back(Res.getValue(I));

  // Return the chain
  return Res.getValue(Ins.size());
}

bool WebAssemblyTargetLowering::CanLowerReturn(
    CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext & /*Context*/,
    const Type *RetTy) const {
  // WebAssembly can only handle returning tuples with multivalue enabled
  return WebAssembly::canLowerReturn(Outs.size(), Subtarget);
}

SDValue WebAssemblyTargetLowering::LowerReturn(
    SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
    SelectionDAG &DAG) const {
  assert(WebAssembly::canLowerReturn(Outs.size(), Subtarget) &&
         "MVP WebAssembly can only return up to one value");
  if (!callingConvSupported(CallConv))
    fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

  SmallVector<SDValue, 4> RetOps(1, Chain);
  RetOps.append(OutVals.begin(), OutVals.end());
  Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);

  // Record the number and types of the return values.
  for (const ISD::OutputArg &Out : Outs) {
    assert(!Out.Flags.isByVal() && "byval is not valid for return values");
    assert(!Out.Flags.isNest() && "nest is not valid for return values");
    assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");
    if (Out.Flags.isInAlloca())
      fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
    if (Out.Flags.isInConsecutiveRegs())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
    if (Out.Flags.isInConsecutiveRegsLast())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
  }

  return Chain;
}

SDValue WebAssemblyTargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  if (!callingConvSupported(CallConv))
    fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

  MachineFunction &MF = DAG.getMachineFunction();
  auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();

  // Set up the incoming ARGUMENTS value, which serves to represent the liveness
  // of the incoming values before they're represented by virtual registers.
  MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);

  bool HasSwiftErrorArg = false;
  bool HasSwiftSelfArg = false;
  bool HasSwiftAsyncArg = false;
  for (const ISD::InputArg &In : Ins) {
    HasSwiftSelfArg |= In.Flags.isSwiftSelf();
    HasSwiftErrorArg |= In.Flags.isSwiftError();
    HasSwiftAsyncArg |= In.Flags.isSwiftAsync();
    if (In.Flags.isInAlloca())
      fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
    if (In.Flags.isNest())
      fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
    if (In.Flags.isInConsecutiveRegs())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
    if (In.Flags.isInConsecutiveRegsLast())
      fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
    // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
    // registers.
    InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
                                           DAG.getTargetConstant(InVals.size(),
                                                                 DL, MVT::i32))
                             : DAG.getUNDEF(In.VT));

    // Record the number and types of arguments.
    MFI->addParam(In.VT);
  }

  // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
  // (for swifttailcc) swiftasync arguments if there aren't. These additional
  // arguments are also added for callee signature. They are necessary to match
  // callee and caller signature for indirect call.
  auto PtrVT = getPointerTy(MF.getDataLayout());
  if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
    if (!HasSwiftSelfArg) {
      MFI->addParam(PtrVT);
    }
    if (!HasSwiftErrorArg) {
      MFI->addParam(PtrVT);
    }
    if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
      MFI->addParam(PtrVT);
    }
  }
  // Varargs are copied into a buffer allocated by the caller, and a pointer to
  // the buffer is passed as an argument.
  if (IsVarArg) {
    MVT PtrVT = getPointerTy(MF.getDataLayout());
    Register VarargVreg =
        MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
    MFI->setVarargBufferVreg(VarargVreg);
    Chain = DAG.getCopyToReg(
        Chain, DL, VarargVreg,
        DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
                    DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
    MFI->addParam(PtrVT);
  }

  // Record the number and types of arguments and results.
  SmallVector<MVT, 4> Params;
  SmallVector<MVT, 4> Results;
  computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),
                      MF.getFunction(), DAG.getTarget(), Params, Results);
  for (MVT VT : Results)
    MFI->addResult(VT);
  // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
  // the param logic here with ComputeSignatureVTs
  assert(MFI->getParams().size() == Params.size() &&
         std::equal(MFI->getParams().begin(), MFI->getParams().end(),
                    Params.begin()));

  return Chain;
}

void WebAssemblyTargetLowering::ReplaceNodeResults(
    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
  switch (N->getOpcode()) {
  case ISD::SIGN_EXTEND_INREG:
    // Do not add any results, signifying that N should not be custom lowered
    // after all. This happens because simd128 turns on custom lowering for
    // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
    // illegal type.
    break;
  case ISD::ANY_EXTEND_VECTOR_INREG:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    // Do not add any results, signifying that N should not be custom lowered.
    // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
    break;
  case ISD::FP_ROUND: {
    EVT VT = N->getValueType(0);
    SDValue Src = N->getOperand(0);
    if (VT == MVT::v4f16 && Src.getValueType() == MVT::v4f32) {
      Results.push_back(
          DAG.getNode(WebAssemblyISD::DEMOTE_ZERO, SDLoc(N), MVT::v8f16, Src));
    }
    break;
  }
  case ISD::ADD:
  case ISD::SUB:
    Results.push_back(Replace128Op(N, DAG));
    break;
  default:
    llvm_unreachable(
        "ReplaceNodeResults not implemented for this op for WebAssembly!");
  }
}

//===----------------------------------------------------------------------===//
//  Custom lowering hooks.
//===----------------------------------------------------------------------===//

SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
                                                  SelectionDAG &DAG) const {
  SDLoc DL(Op);
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("unimplemented operation lowering");
    return SDValue();
  case ISD::FrameIndex:
    return LowerFrameIndex(Op, DAG);
  case ISD::GlobalAddress:
    return LowerGlobalAddress(Op, DAG);
  case ISD::GlobalTLSAddress:
    return LowerGlobalTLSAddress(Op, DAG);
  case ISD::ExternalSymbol:
    return LowerExternalSymbol(Op, DAG);
  case ISD::JumpTable:
    return LowerJumpTable(Op, DAG);
  case ISD::BR_JT:
    return LowerBR_JT(Op, DAG);
  case ISD::VASTART:
    return LowerVASTART(Op, DAG);
  case ISD::BlockAddress:
  case ISD::BRIND:
    fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
    return SDValue();
  case ISD::RETURNADDR:
    return LowerRETURNADDR(Op, DAG);
  case ISD::FRAMEADDR:
    return LowerFRAMEADDR(Op, DAG);
  case ISD::CopyToReg:
    return LowerCopyToReg(Op, DAG);
  case ISD::EXTRACT_VECTOR_ELT:
  case ISD::INSERT_VECTOR_ELT:
    return LowerAccessVectorElement(Op, DAG);
  case ISD::INTRINSIC_VOID:
  case ISD::INTRINSIC_WO_CHAIN:
  case ISD::INTRINSIC_W_CHAIN:
    return LowerIntrinsic(Op, DAG);
  case ISD::SIGN_EXTEND_INREG:
    return LowerSIGN_EXTEND_INREG(Op, DAG);
  case ISD::ZERO_EXTEND_VECTOR_INREG:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
  case ISD::ANY_EXTEND_VECTOR_INREG:
    return LowerEXTEND_VECTOR_INREG(Op, DAG);
  case ISD::BUILD_VECTOR:
    return LowerBUILD_VECTOR(Op, DAG);
  case ISD::VECTOR_SHUFFLE:
    return LowerVECTOR_SHUFFLE(Op, DAG);
  case ISD::SETCC:
    return LowerSETCC(Op, DAG);
  case ISD::SHL:
  case ISD::SRA:
  case ISD::SRL:
    return LowerShift(Op, DAG);
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    return LowerFP_TO_INT_SAT(Op, DAG);
  case ISD::FMINNUM:
  case ISD::FMINIMUMNUM:
    return LowerFMIN(Op, DAG);
  case ISD::FMAXNUM:
  case ISD::FMAXIMUMNUM:
    return LowerFMAX(Op, DAG);
  case ISD::LOAD:
    return LowerLoad(Op, DAG);
  case ISD::STORE:
    return LowerStore(Op, DAG);
  case ISD::CTPOP:
  case ISD::CTLZ:
  case ISD::CTTZ:
    return DAG.UnrollVectorOp(Op.getNode());
  case ISD::CLEAR_CACHE:
    // Report this as a diagnostic rather than aborting, like the other
    // unsupported features in this target. Pass the chain through so that
    // codegen can reach the point where the diagnostic is emitted.
    fail(SDLoc(Op), DAG, "llvm.clear_cache is not supported on wasm");
    return Op.getOperand(0);
  case ISD::SMUL_LOHI:
  case ISD::UMUL_LOHI:
    return LowerMUL_LOHI(Op, DAG);
  case ISD::UADDO:
    return LowerUADDO(Op, DAG);
  }
}

static bool IsWebAssemblyGlobal(SDValue Op) {
  if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
    return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());

  return false;
}

static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
                                                  SelectionDAG &DAG) {
  const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
  if (!FI)
    return std::nullopt;

  auto &MF = DAG.getMachineFunction();
  return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());
}

SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
  const SDValue &Value = SN->getValue();
  const SDValue &Base = SN->getBasePtr();
  const SDValue &Offset = SN->getOffset();

  if (IsWebAssemblyGlobal(Base)) {
    if (!Offset->isUndef())
      report_fatal_error("unexpected offset when storing to webassembly global",
                         false);

    SDVTList Tys = DAG.getVTList(MVT::Other);
    SDValue Ops[] = {SN->getChain(), Value, Base};
    return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
                                   SN->getMemoryVT(), SN->getMemOperand());
  }

  if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
    if (!Offset->isUndef())
      report_fatal_error("unexpected offset when storing to webassembly local",
                         false);

    SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
    SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
    SDValue Ops[] = {SN->getChain(), Idx, Value};
    return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
  }

  if (WebAssembly::isWasmVarAddressSpace(SN->getAddressSpace()))
    report_fatal_error(
        "Encountered an unlowerable store to the wasm_var address space",
        false);

  return Op;
}

SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
  const SDValue &Base = LN->getBasePtr();
  const SDValue &Offset = LN->getOffset();

  if (IsWebAssemblyGlobal(Base)) {
    if (!Offset->isUndef())
      report_fatal_error(
          "unexpected offset when loading from webassembly global", false);

    SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
    SDValue Ops[] = {LN->getChain(), Base};
    return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
                                   LN->getMemoryVT(), LN->getMemOperand());
  }

  if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
    if (!Offset->isUndef())
      report_fatal_error(
          "unexpected offset when loading from webassembly local", false);

    SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
    EVT LocalVT = LN->getValueType(0);
    return DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, {LocalVT, MVT::Other},
                       {LN->getChain(), Idx});
  }

  if (WebAssembly::isWasmVarAddressSpace(LN->getAddressSpace()))
    report_fatal_error(
        "Encountered an unlowerable load from the wasm_var address space",
        false);

  return Op;
}

SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
                                                 SelectionDAG &DAG) const {
  assert(Subtarget->hasWideArithmetic());
  assert(Op.getValueType() == MVT::i64);
  SDLoc DL(Op);
  unsigned Opcode;
  switch (Op.getOpcode()) {
  case ISD::UMUL_LOHI:
    Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
    break;
  case ISD::SMUL_LOHI:
    Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
    break;
  default:
    llvm_unreachable("unexpected opcode");
  }
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  SDValue Lo =
      DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64), LHS, RHS);
  SDValue Hi(Lo.getNode(), 1);
  SDValue Ops[] = {Lo, Hi};
  return DAG.getMergeValues(Ops, DL);
}

// Lowers `UADDO` intrinsics to an `i64.add128` instruction when it's enabled.
//
// This enables generating a single wasm instruction for this operation where
// the upper half of both operands are constant zeros. The upper half of the
// result is then whether the overflow happened.
SDValue WebAssemblyTargetLowering::LowerUADDO(SDValue Op,
                                              SelectionDAG &DAG) const {
  assert(Subtarget->hasWideArithmetic());
  assert(Op.getValueType() == MVT::i64);
  assert(Op.getOpcode() == ISD::UADDO);
  SDLoc DL(Op);
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
  SDValue Result =
      DAG.getNode(WebAssemblyISD::I64_ADD128, DL,
                  DAG.getVTList(MVT::i64, MVT::i64), LHS, Zero, RHS, Zero);
  SDValue CarryI64(Result.getNode(), 1);
  SDValue CarryI32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, CarryI64);
  SDValue Ops[] = {Result, CarryI32};
  return DAG.getMergeValues(Ops, DL);
}

SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
                                                SelectionDAG &DAG) const {
  assert(Subtarget->hasWideArithmetic());
  assert(N->getValueType(0) == MVT::i128);
  SDLoc DL(N);
  unsigned Opcode;
  switch (N->getOpcode()) {
  case ISD::ADD:
    Opcode = WebAssemblyISD::I64_ADD128;
    break;
  case ISD::SUB:
    Opcode = WebAssemblyISD::I64_SUB128;
    break;
  default:
    llvm_unreachable("unexpected opcode");
  }
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);

  SDValue C0 = DAG.getConstant(0, DL, MVT::i64);
  SDValue C1 = DAG.getConstant(1, DL, MVT::i64);
  SDValue LHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C0);
  SDValue LHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C1);
  SDValue RHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C0);
  SDValue RHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C1);
  SDValue Result_LO = DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64),
                                  LHS_0, LHS_1, RHS_0, RHS_1);
  SDValue Result_HI(Result_LO.getNode(), 1);
  return DAG.getNode(ISD::BUILD_PAIR, DL, N->getVTList(), Result_LO, Result_HI);
}

SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
                                                  SelectionDAG &DAG) const {
  SDValue Src = Op.getOperand(2);
  if (isa<FrameIndexSDNode>(Src.getNode())) {
    // CopyToReg nodes don't support FrameIndex operands. Other targets select
    // the FI to some LEA-like instruction, but since we don't have that, we
    // need to insert some kind of instruction that can take an FI operand and
    // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
    // local.copy between Op and its FI operand.
    SDValue Chain = Op.getOperand(0);
    SDLoc DL(Op);
    Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
    EVT VT = Src.getValueType();
    SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
                                                   : WebAssembly::COPY_I64,
                                    DL, VT, Src),
                 0);
    return Op.getNode()->getNumValues() == 1
               ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
               : DAG.getCopyToReg(Chain, DL, Reg, Copy,
                                  Op.getNumOperands() == 4 ? Op.getOperand(3)
                                                           : SDValue());
  }
  return SDValue();
}

SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
                                                   SelectionDAG &DAG) const {
  int FI = cast<FrameIndexSDNode>(Op)->getIndex();
  return DAG.getTargetFrameIndex(FI, Op.getValueType());
}

SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
                                                   SelectionDAG &DAG) const {
  SDLoc DL(Op);

  if (!Subtarget->getTargetTriple().isOSEmscripten()) {
    fail(DL, DAG,
         "Non-Emscripten WebAssembly hasn't implemented "
         "__builtin_return_address");
    return SDValue();
  }

  unsigned Depth = Op.getConstantOperandVal(0);
  MakeLibCallOptions CallOptions;
  return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
                     {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
      .first;
}

SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
                                                  SelectionDAG &DAG) const {
  // Non-zero depths are not supported by WebAssembly currently. Use the
  // legalizer's default expansion, which is to return 0 (what this function is
  // documented to do).
  if (Op.getConstantOperandVal(0) > 0)
    return SDValue();

  DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
  EVT VT = Op.getValueType();
  Register FP =
      Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
  return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
}

SDValue
WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  const auto *GA = cast<GlobalAddressSDNode>(Op);

  MachineFunction &MF = DAG.getMachineFunction();
  if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
    report_fatal_error("cannot use thread-local storage without bulk memory",
                       false);

  const GlobalValue *GV = GA->getGlobal();

  // Currently only Emscripten supports dynamic linking with threads. Therefore,
  // on other targets, if we have thread-local storage, only the local-exec
  // model is possible.
  auto model = Subtarget->getTargetTriple().isOSEmscripten()
                   ? GV->getThreadLocalMode()
                   : GlobalValue::LocalExecTLSModel;

  // Unsupported TLS modes
  assert(model != GlobalValue::NotThreadLocal);
  assert(model != GlobalValue::InitialExecTLSModel);

  if (model == GlobalValue::LocalExecTLSModel ||
      model == GlobalValue::LocalDynamicTLSModel ||
      (model == GlobalValue::GeneralDynamicTLSModel &&
       getTargetMachine().shouldAssumeDSOLocal(GV))) {
    // For DSO-local TLS variables we use offset from __tls_base, or
    // __wasm_get_tls_base() if using libcall thread context.

    MVT PtrVT = getPointerTy(DAG.getDataLayout());
    SDValue BaseAddr(WebAssembly::getTLSBase(DAG, DL, Subtarget), 0);

    SDValue TLSOffset = DAG.getTargetGlobalAddress(
        GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
    SDValue SymOffset =
        DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);

    return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
  }

  assert(model == GlobalValue::GeneralDynamicTLSModel);

  EVT VT = Op.getValueType();
  return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                     DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
                                                GA->getOffset(),
                                                WebAssemblyII::MO_GOT_TLS));
}

SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
                                                      SelectionDAG &DAG) const {
  SDLoc DL(Op);
  const auto *GA = cast<GlobalAddressSDNode>(Op);
  EVT VT = Op.getValueType();
  assert(GA->getTargetFlags() == 0 &&
         "Unexpected target flags on generic GlobalAddressSDNode");
  if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))
    fail(DL, DAG, "Invalid address space for WebAssembly target");

  unsigned OperandFlags = 0;
  const GlobalValue *GV = GA->getGlobal();
  // Since WebAssembly tables cannot yet be shared across modules, we don't
  // need special treatment for tables in PIC mode.
  if (isPositionIndependent() &&
      !WebAssembly::isWebAssemblyTableType(GV->getValueType())) {
    if (getTargetMachine().shouldAssumeDSOLocal(GV)) {
      MachineFunction &MF = DAG.getMachineFunction();
      MVT PtrVT = getPointerTy(MF.getDataLayout());
      const char *BaseName;
      if (GV->getValueType()->isFunctionTy()) {
        BaseName = MF.createExternalSymbolName("__table_base");
        OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
      } else {
        BaseName = MF.createExternalSymbolName("__memory_base");
        OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
      }
      SDValue BaseAddr =
          DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
                      DAG.getTargetExternalSymbol(BaseName, PtrVT));

      SDValue SymAddr = DAG.getNode(
          WebAssemblyISD::WrapperREL, DL, VT,
          DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
                                     OperandFlags));

      return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
    }
    OperandFlags = WebAssemblyII::MO_GOT;
  }

  return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                     DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
                                                GA->getOffset(), OperandFlags));
}

SDValue
WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  const auto *ES = cast<ExternalSymbolSDNode>(Op);
  EVT VT = Op.getValueType();
  assert(ES->getTargetFlags() == 0 &&
         "Unexpected target flags on generic ExternalSymbolSDNode");
  return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                     DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
}

SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
                                                  SelectionDAG &DAG) const {
  // There's no need for a Wrapper node because we always incorporate a jump
  // table operand into a BR_TABLE instruction, rather than ever
  // materializing it in a register.
  const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
  return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
                                JT->getTargetFlags());
}

SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op.getOperand(0);
  const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
  SDValue Index = Op.getOperand(2);
  assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");

  SmallVector<SDValue, 8> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Index);

  MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
  const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;

  // Add an operand for each case.
  for (auto *MBB : MBBs)
    Ops.push_back(DAG.getBasicBlock(MBB));

  // Add the first MBB as a dummy default target for now. This will be replaced
  // with the proper default target (and the preceding range check eliminated)
  // if possible by WebAssemblyFixBrTableDefaults.
  Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
  return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
}

SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());

  auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();

  SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
                                    MFI->getVarargBufferVreg(), PtrVT);
  return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
                      MachinePointerInfo(SV));
}

SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
                                                  SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  unsigned IntNo;
  switch (Op.getOpcode()) {
  case ISD::INTRINSIC_VOID:
  case ISD::INTRINSIC_W_CHAIN:
    IntNo = Op.getConstantOperandVal(1);
    break;
  case ISD::INTRINSIC_WO_CHAIN:
    IntNo = Op.getConstantOperandVal(0);
    break;
  default:
    llvm_unreachable("Invalid intrinsic");
  }
  SDLoc DL(Op);

  switch (IntNo) {
  default:
    return SDValue(); // Don't custom lower most intrinsics.

  case Intrinsic::wasm_lsda: {
    auto PtrVT = getPointerTy(MF.getDataLayout());
    const char *SymName = MF.createExternalSymbolName(
        "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
    if (isPositionIndependent()) {
      SDValue Node = DAG.getTargetExternalSymbol(
          SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
      const char *BaseName = MF.createExternalSymbolName("__memory_base");
      SDValue BaseAddr =
          DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
                      DAG.getTargetExternalSymbol(BaseName, PtrVT));
      SDValue SymAddr =
          DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
      return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
    }
    SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
    return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
  }

  case Intrinsic::wasm_shuffle: {
    // Drop in-chain and replace undefs, but otherwise pass through unchanged
    SDValue Ops[18];
    size_t OpIdx = 0;
    Ops[OpIdx++] = Op.getOperand(1);
    Ops[OpIdx++] = Op.getOperand(2);
    while (OpIdx < 18) {
      const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
      if (MaskIdx.isUndef() || MaskIdx.getNode()->getAsZExtVal() >= 32) {
        bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;
        Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32, isTarget);
      } else {
        Ops[OpIdx++] = MaskIdx;
      }
    }
    return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
  }

  case Intrinsic::wasm_funcref_to_ptr: {
    // llvm.wasm.funcref.to_ptr only has a defined lowering when its result
    // feeds directly into an indirect call. Reaching here means the pointer
    // escapes a direct call. We haven't implemented conversion of a funcref
    // into a real function pointer so we crash if we get here.
    fail(DL, DAG,
         "a funcref can only be converted to a pointer to be directly called; "
         "the resulting pointer cannot otherwise be used");
    return DAG.getPOISON(Op.getValueType());
  }

  case Intrinsic::thread_pointer: {
    return SDValue(WebAssembly::getTLSBase(DAG, DL, Subtarget), 0);
  }
  }
}

SDValue
WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
                                                  SelectionDAG &DAG) const {
  SDLoc DL(Op);
  // If sign extension operations are disabled, allow sext_inreg only if operand
  // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
  // extension operations, but allowing sext_inreg in this context lets us have
  // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
  // everywhere would be simpler in this file, but would necessitate large and
  // brittle patterns to undo the expansion and select extract_lane_s
  // instructions.
  assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
  if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();

  const SDValue &Extract = Op.getOperand(0);
  MVT VecT = Extract.getOperand(0).getSimpleValueType();
  if (VecT.getVectorElementType().getSizeInBits() > 32)
    return SDValue();
  MVT ExtractedLaneT =
      cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
  MVT ExtractedVecT =
      MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
  if (ExtractedVecT == VecT)
    return Op;

  // Bitcast vector to appropriate type to ensure ISel pattern coverage
  const SDNode *Index = Extract.getOperand(1).getNode();
  if (!isa<ConstantSDNode>(Index))
    return SDValue();
  unsigned IndexVal = Index->getAsZExtVal();
  unsigned Scale =
      ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
  assert(Scale > 1);
  SDValue NewIndex =
      DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
  SDValue NewExtract = DAG.getNode(
      ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
      DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
                     Op.getOperand(1));
}

static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT,
                             SelectionDAG &DAG) {
  SDValue Source = peekThroughBitcasts(Op);
  if (Source.getOpcode() != ISD::VECTOR_SHUFFLE)
    return SDValue();

  assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||
          UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&
         "expected extend_low");
  auto *Shuffle = cast<ShuffleVectorSDNode>(Source.getNode());

  ArrayRef<int> Mask = Shuffle->getMask();
  // Look for a shuffle which moves from the high half to the low half.
  size_t FirstIdx = Mask.size() / 2;
  for (size_t i = 0; i < Mask.size() / 2; ++i) {
    if (Mask[i] != static_cast<int>(FirstIdx + i)) {
      return SDValue();
    }
  }

  SDLoc DL(Op);
  unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S
                     ? WebAssemblyISD::EXTEND_HIGH_S
                     : WebAssemblyISD::EXTEND_HIGH_U;
  SDValue ShuffleSrc = Shuffle->getOperand(0);
  if (Op.getOpcode() == ISD::BITCAST)
    ShuffleSrc = DAG.getBitcast(Op.getValueType(), ShuffleSrc);

  return DAG.getNode(Opc, DL, VT, ShuffleSrc);
}

SDValue
WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,
                                                    SelectionDAG &DAG) const {
  SDLoc DL(Op);
  EVT VT = Op.getValueType();
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();

  if (SrcVT.getVectorElementType() == MVT::i1 ||
      SrcVT.getVectorElementType() == MVT::i64)
    return SDValue();

  assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
         "Unexpected extension factor.");
  unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();

  if (Scale != 2 && Scale != 4 && Scale != 8)
    return SDValue();

  unsigned Ext;
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("unexpected opcode");
  case ISD::ANY_EXTEND_VECTOR_INREG:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    Ext = WebAssemblyISD::EXTEND_LOW_U;
    break;
  case ISD::SIGN_EXTEND_VECTOR_INREG:
    Ext = WebAssemblyISD::EXTEND_LOW_S;
    break;
  }

  if (Scale == 2) {
    // See if we can use EXTEND_HIGH.
    if (auto ExtendHigh = GetExtendHigh(Op.getOperand(0), Ext, VT, DAG))
      return ExtendHigh;
  }

  SDValue Ret = Src;
  while (Scale != 1) {
    Ret = DAG.getNode(Ext, DL,
                      Ret.getValueType()
                          .widenIntegerVectorElementType(*DAG.getContext())
                          .getHalfNumVectorElementsVT(*DAG.getContext()),
                      Ret);
    Scale /= 2;
  }
  assert(Ret.getValueType() == VT);
  return Ret;
}

static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  if (Op.getValueType() != MVT::v2f64 && Op.getValueType() != MVT::v4f32)
    return SDValue();

  auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
                             unsigned &Index) -> bool {
    switch (Op.getOpcode()) {
    case ISD::SINT_TO_FP:
      Opcode = WebAssemblyISD::CONVERT_LOW_S;
      break;
    case ISD::UINT_TO_FP:
      Opcode = WebAssemblyISD::CONVERT_LOW_U;
      break;
    case ISD::FP_EXTEND:
    case ISD::FP16_TO_FP:
      Opcode = WebAssemblyISD::PROMOTE_LOW;
      break;
    default:
      return false;
    }

    auto ExtractVector = Op.getOperand(0);
    if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
      return false;

    if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
      return false;

    SrcVec = ExtractVector.getOperand(0);
    Index = ExtractVector.getConstantOperandVal(1);
    return true;
  };

  unsigned NumLanes = Op.getValueType() == MVT::v2f64 ? 2 : 4;
  unsigned FirstOpcode = 0, SecondOpcode = 0, ThirdOpcode = 0, FourthOpcode = 0;
  unsigned FirstIndex = 0, SecondIndex = 0, ThirdIndex = 0, FourthIndex = 0;
  SDValue FirstSrcVec, SecondSrcVec, ThirdSrcVec, FourthSrcVec;

  if (!GetConvertedLane(Op.getOperand(0), FirstOpcode, FirstSrcVec,
                        FirstIndex) ||
      !GetConvertedLane(Op.getOperand(1), SecondOpcode, SecondSrcVec,
                        SecondIndex))
    return SDValue();

  // If we're converting to v4f32, check the third and fourth lanes, too.
  if (NumLanes == 4 && (!GetConvertedLane(Op.getOperand(2), ThirdOpcode,
                                          ThirdSrcVec, ThirdIndex) ||
                        !GetConvertedLane(Op.getOperand(3), FourthOpcode,
                                          FourthSrcVec, FourthIndex)))
    return SDValue();

  if (FirstOpcode != SecondOpcode)
    return SDValue();

  // TODO Add an optimization similar to the v2f64 below for shuffling the
  // vectors when the lanes are in the wrong order or come from different src
  // vectors.
  if (NumLanes == 4 &&
      (FirstOpcode != ThirdOpcode || FirstOpcode != FourthOpcode ||
       FirstSrcVec != SecondSrcVec || FirstSrcVec != ThirdSrcVec ||
       FirstSrcVec != FourthSrcVec || FirstIndex != 0 || SecondIndex != 1 ||
       ThirdIndex != 2 || FourthIndex != 3))
    return SDValue();

  MVT ExpectedSrcVT;
  switch (FirstOpcode) {
  case WebAssemblyISD::CONVERT_LOW_S:
  case WebAssemblyISD::CONVERT_LOW_U:
    ExpectedSrcVT = MVT::v4i32;
    break;
  case WebAssemblyISD::PROMOTE_LOW:
    ExpectedSrcVT = NumLanes == 2 ? MVT::v4f32 : MVT::v8i16;
    break;
  }
  if (FirstSrcVec.getValueType() != ExpectedSrcVT)
    return SDValue();

  auto Src = FirstSrcVec;
  if (NumLanes == 2 &&
      (FirstIndex != 0 || SecondIndex != 1 || FirstSrcVec != SecondSrcVec)) {
    // Shuffle the source vector so that the converted lanes are the low lanes.
    Src = DAG.getVectorShuffle(ExpectedSrcVT, DL, FirstSrcVec, SecondSrcVec,
                               {static_cast<int>(FirstIndex),
                                static_cast<int>(SecondIndex) + 4, -1, -1});
  }
  return DAG.getNode(FirstOpcode, DL, NumLanes == 2 ? MVT::v2f64 : MVT::v4f32,
                     Src);
}

SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                                     SelectionDAG &DAG) const {
  MVT VT = Op.getSimpleValueType();
  if (VT == MVT::v8f16) {
    // BUILD_VECTOR can't handle FP16 operands since Wasm doesn't have a scalar
    // FP16 type, so cast them to I16s.
    MVT IVT = VT.changeVectorElementType(MVT::i16);
    SmallVector<SDValue, 8> NewOps;
    for (unsigned I = 0, E = Op.getNumOperands(); I < E; ++I)
      NewOps.push_back(DAG.getBitcast(MVT::i16, Op.getOperand(I)));
    SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
    return DAG.getBitcast(VT, Res);
  }

  if (auto ConvertLow = LowerConvertLow(Op, DAG))
    return ConvertLow;

  SDLoc DL(Op);
  const EVT VecT = Op.getValueType();
  const EVT LaneT = Op.getOperand(0).getValueType();
  const size_t Lanes = Op.getNumOperands();
  bool CanSwizzle = VecT == MVT::v16i8;

  // BUILD_VECTORs are lowered to the instruction that initializes the highest
  // possible number of lanes at once followed by a sequence of replace_lane
  // instructions to individually initialize any remaining lanes.

  // TODO: Tune this. For example, lanewise swizzling is very expensive, so
  // swizzled lanes should be given greater weight.

  // TODO: Investigate looping rather than always extracting/replacing specific
  // lanes to fill gaps.

  auto IsConstant = [](const SDValue &V) {
    return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
  };

  // Returns the source vector and index vector pair if they exist. Checks for:
  //   (extract_vector_elt
  //     $src,
  //     (sign_extend_inreg (extract_vector_elt $indices, $i))
  //   )
  auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
    auto Bail = std::make_pair(SDValue(), SDValue());
    if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
      return Bail;
    const SDValue &SwizzleSrc = Lane->getOperand(0);
    const SDValue &IndexExt = Lane->getOperand(1);
    if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
      return Bail;
    const SDValue &Index = IndexExt->getOperand(0);
    if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
      return Bail;
    const SDValue &SwizzleIndices = Index->getOperand(0);
    if (SwizzleSrc.getValueType() != MVT::v16i8 ||
        SwizzleIndices.getValueType() != MVT::v16i8 ||
        Index->getOperand(1)->getOpcode() != ISD::Constant ||
        Index->getConstantOperandVal(1) != I)
      return Bail;
    return std::make_pair(SwizzleSrc, SwizzleIndices);
  };

  // If the lane is extracted from another vector at a constant index, return
  // that vector. The source vector must not have more lanes than the dest
  // because the shufflevector indices are in terms of the destination lanes and
  // would not be able to address the smaller individual source lanes.
  auto GetShuffleSrc = [&](const SDValue &Lane) {
    if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
      return SDValue();
    if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
      return SDValue();
    if (Lane->getOperand(0).getValueType().getVectorNumElements() >
        VecT.getVectorNumElements())
      return SDValue();
    return Lane->getOperand(0);
  };

  using ValueEntry = std::pair<SDValue, size_t>;
  SmallVector<ValueEntry, 16> SplatValueCounts;

  using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
  SmallVector<SwizzleEntry, 16> SwizzleCounts;

  using ShuffleEntry = std::pair<SDValue, size_t>;
  SmallVector<ShuffleEntry, 16> ShuffleCounts;

  auto AddCount = [](auto &Counts, const auto &Val) {
    auto CountIt =
        llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
    if (CountIt == Counts.end()) {
      Counts.emplace_back(Val, 1);
    } else {
      CountIt->second++;
    }
  };

  auto GetMostCommon = [](auto &Counts) {
    auto CommonIt = llvm::max_element(Counts, llvm::less_second());
    assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
    return *CommonIt;
  };

  size_t NumConstantLanes = 0;

  // Count eligible lanes for each type of vector creation op
  for (size_t I = 0; I < Lanes; ++I) {
    const SDValue &Lane = Op->getOperand(I);
    if (Lane.isUndef())
      continue;

    AddCount(SplatValueCounts, Lane);

    if (IsConstant(Lane))
      NumConstantLanes++;
    if (auto ShuffleSrc = GetShuffleSrc(Lane))
      AddCount(ShuffleCounts, ShuffleSrc);
    if (CanSwizzle) {
      auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
      if (SwizzleSrcs.first)
        AddCount(SwizzleCounts, SwizzleSrcs);
    }
  }

  SDValue SplatValue;
  size_t NumSplatLanes;
  std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);

  SDValue SwizzleSrc;
  SDValue SwizzleIndices;
  size_t NumSwizzleLanes = 0;
  if (SwizzleCounts.size())
    std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
                          NumSwizzleLanes) = GetMostCommon(SwizzleCounts);

  // Shuffles can draw from up to two vectors, so find the two most common
  // sources.
  SDValue ShuffleSrc1, ShuffleSrc2;
  size_t NumShuffleLanes = 0;
  if (ShuffleCounts.size()) {
    std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
    llvm::erase_if(ShuffleCounts,
                   [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
  }
  if (ShuffleCounts.size()) {
    size_t AdditionalShuffleLanes;
    std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
        GetMostCommon(ShuffleCounts);
    NumShuffleLanes += AdditionalShuffleLanes;
  }

  // Predicate returning true if the lane is properly initialized by the
  // original instruction
  std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
  SDValue Result;
  // Prefer swizzles over shuffles over vector consts over splats
  if (NumSwizzleLanes >= NumShuffleLanes &&
      NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
    Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
                         SwizzleIndices);
    auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
    IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
      return Swizzled == GetSwizzleSrcs(I, Lane);
    };
  } else if (NumShuffleLanes >= NumConstantLanes &&
             NumShuffleLanes >= NumSplatLanes) {
    size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
    size_t DestLaneCount = VecT.getVectorNumElements();
    size_t Scale1 = 1;
    size_t Scale2 = 1;
    SDValue Src1 = ShuffleSrc1;
    SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
    if (Src1.getValueType() != VecT) {
      size_t LaneSize =
          Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
      assert(LaneSize > DestLaneSize);
      Scale1 = LaneSize / DestLaneSize;
      Src1 = DAG.getBitcast(VecT, Src1);
    }
    if (Src2.getValueType() != VecT) {
      size_t LaneSize =
          Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
      assert(LaneSize > DestLaneSize);
      Scale2 = LaneSize / DestLaneSize;
      Src2 = DAG.getBitcast(VecT, Src2);
    }

    int Mask[16];
    assert(DestLaneCount <= 16);
    for (size_t I = 0; I < DestLaneCount; ++I) {
      const SDValue &Lane = Op->getOperand(I);
      SDValue Src = GetShuffleSrc(Lane);
      if (Src == ShuffleSrc1) {
        Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
      } else if (Src && Src == ShuffleSrc2) {
        Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
      } else {
        Mask[I] = -1;
      }
    }
    ArrayRef<int> MaskRef(Mask, DestLaneCount);
    Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
    IsLaneConstructed = [&](size_t, const SDValue &Lane) {
      auto Src = GetShuffleSrc(Lane);
      return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
    };
  } else if (NumConstantLanes >= NumSplatLanes) {
    SmallVector<SDValue, 16> ConstLanes;
    for (const SDValue &Lane : Op->op_values()) {
      if (IsConstant(Lane)) {
        // Values may need to be fixed so that they will sign extend to be
        // within the expected range during ISel. Check whether the value is in
        // bounds based on the lane bit width and if it is out of bounds, lop
        // off the extra bits.
        uint64_t LaneBits = 128 / Lanes;
        if (auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode())) {
          ConstLanes.push_back(DAG.getConstant(
              Const->getAPIntValue().trunc(LaneBits).getZExtValue(),
              SDLoc(Lane), LaneT));
        } else {
          ConstLanes.push_back(Lane);
        }
      } else if (LaneT.isFloatingPoint()) {
        ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
      } else {
        ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
      }
    }
    Result = DAG.getBuildVector(VecT, DL, ConstLanes);
    IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
      return IsConstant(Lane);
    };
  } else {
    size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits();
    if (NumSplatLanes == 1 && Op->getOperand(0) == SplatValue &&
        (DestLaneSize == 32 || DestLaneSize == 64)) {
      // Could be selected to load_zero.
      Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecT, SplatValue);
    } else {
      // Use a splat (which might be selected as a load splat)
      Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
    }
    IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
      return Lane == SplatValue;
    };
  }

  assert(Result);
  assert(IsLaneConstructed);

  // Add replace_lane instructions for any unhandled values
  for (size_t I = 0; I < Lanes; ++I) {
    const SDValue &Lane = Op->getOperand(I);
    if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
      Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
                           DAG.getConstant(I, DL, MVT::i32));
  }

  return Result;
}

SDValue
WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
  MVT VecType = Op.getOperand(0).getSimpleValueType();
  assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
  size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;

  // Space for two vector args and sixteen mask indices
  SDValue Ops[18];
  size_t OpIdx = 0;
  Ops[OpIdx++] = Op.getOperand(0);
  Ops[OpIdx++] = Op.getOperand(1);

  // Expand mask indices to byte indices and materialize them as operands
  for (int M : Mask) {
    for (size_t J = 0; J < LaneBytes; ++J) {
      // Lower undefs (represented by -1 in mask) to {0..J}, which use a
      // whole lane of vector input, to allow further reduction at VM. E.g.
      // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
      uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
      Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
    }
  }

  return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
}

SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  // The legalizer does not know how to expand the unsupported comparison modes
  // of i64x2 vectors, so we manually unroll them here.
  assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
  SmallVector<SDValue, 2> LHS, RHS;
  DAG.ExtractVectorElements(Op->getOperand(0), LHS);
  DAG.ExtractVectorElements(Op->getOperand(1), RHS);
  const SDValue &CC = Op->getOperand(2);
  auto MakeLane = [&](unsigned I) {
    return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
                       DAG.getConstant(uint64_t(-1), DL, MVT::i64),
                       DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
  };
  return DAG.getBuildVector(Op->getValueType(0), DL,
                            {MakeLane(0), MakeLane(1)});
}

SDValue
WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
                                                    SelectionDAG &DAG) const {
  if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
      Op.getValueType() == MVT::v8f16) {
    // INSERT_VECTOR_ELT can't handle FP16 operands since Wasm doesn't have a
    // scalar FP16 type, so cast them to I16s.
    SDLoc DL(Op);
    SDValue IntVector = DAG.getBitcast(MVT::v8i16, Op.getOperand(0));
    SDValue IntElement = DAG.getBitcast(MVT::i16, Op.getOperand(1));
    SDValue Inserted = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,
                                   IntVector, IntElement, Op.getOperand(2));
    return DAG.getBitcast(MVT::v8f16, Inserted);
  }

  // Allow constant lane indices, expand variable lane indices
  SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
  if (isa<ConstantSDNode>(IdxNode)) {
    // Ensure the index type is i32 to match the tablegen patterns
    uint64_t Idx = IdxNode->getAsZExtVal();
    SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
    Ops[Op.getNumOperands() - 1] =
        DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32);
    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops);
  }
  // Perform default expansion
  return SDValue();
}

static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
  EVT LaneT = Op.getSimpleValueType().getVectorElementType();
  // 32-bit and 64-bit unrolled shifts will have proper semantics
  if (LaneT.bitsGE(MVT::i32))
    return DAG.UnrollVectorOp(Op.getNode());
  // Otherwise mask the shift value to get proper semantics from 32-bit shift
  SDLoc DL(Op);
  size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
  SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
  unsigned ShiftOpcode = Op.getOpcode();
  SmallVector<SDValue, 16> ShiftedElements;
  DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
  SmallVector<SDValue, 16> ShiftElements;
  DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
  SmallVector<SDValue, 16> UnrolledOps;
  for (size_t i = 0; i < NumLanes; ++i) {
    SDValue MaskedShiftValue =
        DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
    SDValue ShiftedValue = ShiftedElements[i];
    if (ShiftOpcode == ISD::SRA)
      ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
                                 ShiftedValue, DAG.getValueType(LaneT));
    UnrolledOps.push_back(
        DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
  }
  return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
}

SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  // Only manually lower vector shifts
  assert(Op.getSimpleValueType().isVector());

  uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
  auto ShiftVal = Op.getOperand(1);

  // Try to skip bitmask operation since it is implied inside shift instruction
  auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
    if (MaskOp.getOpcode() != ISD::AND)
      return MaskOp;
    SDValue LHS = MaskOp.getOperand(0);
    SDValue RHS = MaskOp.getOperand(1);
    if (MaskOp.getValueType().isVector()) {
      APInt MaskVal;
      if (!ISD::isConstantSplatVector(RHS.getNode(), MaskVal))
        std::swap(LHS, RHS);

      if (ISD::isConstantSplatVector(RHS.getNode(), MaskVal) &&
          MaskVal == MaskBits)
        MaskOp = LHS;
    } else {
      if (!isa<ConstantSDNode>(RHS.getNode()))
        std::swap(LHS, RHS);

      auto ConstantRHS = dyn_cast<ConstantSDNode>(RHS.getNode());
      if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
        MaskOp = LHS;
    }

    return MaskOp;
  };

  // Skip vector and operation
  ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
  ShiftVal = DAG.getSplatValue(ShiftVal);
  if (!ShiftVal)
    return unrollVectorShift(Op, DAG);

  // Skip scalar and operation
  ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
  // Use anyext because none of the high bits can affect the shift
  ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);

  unsigned Opcode;
  switch (Op.getOpcode()) {
  case ISD::SHL:
    Opcode = WebAssemblyISD::VEC_SHL;
    break;
  case ISD::SRA:
    Opcode = WebAssemblyISD::VEC_SHR_S;
    break;
  case ISD::SRL:
    Opcode = WebAssemblyISD::VEC_SHR_U;
    break;
  default:
    llvm_unreachable("unexpected opcode");
  }

  return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
}

SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
                                                      SelectionDAG &DAG) const {
  EVT ResT = Op.getValueType();
  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

  if ((ResT == MVT::i32 || ResT == MVT::i64) &&
      (SatVT == MVT::i32 || SatVT == MVT::i64))
    return Op;

  if (ResT == MVT::v4i32 && SatVT == MVT::i32)
    return Op;

  if (ResT == MVT::v8i16 && SatVT == MVT::i16)
    return Op;

  return SDValue();
}

static bool HasNoSignedZerosOrNaNs(SDValue Op, SelectionDAG &DAG) {
  return (Op->getFlags().hasNoNaNs() ||
          (DAG.isKnownNeverNaN(Op->getOperand(0)) &&
           DAG.isKnownNeverNaN(Op->getOperand(1)))) &&
         (Op->getFlags().hasNoSignedZeros() ||
          DAG.isKnownNeverLogicalZero(Op->getOperand(0)) ||
          DAG.isKnownNeverLogicalZero(Op->getOperand(1)));
}

SDValue WebAssemblyTargetLowering::LowerFMIN(SDValue Op,
                                             SelectionDAG &DAG) const {
  if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
    return DAG.getNode(WebAssemblyISD::RELAXED_FMIN, SDLoc(Op),
                       Op.getValueType(), Op.getOperand(0), Op.getOperand(1));
  }
  return SDValue();
}

SDValue WebAssemblyTargetLowering::LowerFMAX(SDValue Op,
                                             SelectionDAG &DAG) const {
  if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
    return DAG.getNode(WebAssemblyISD::RELAXED_FMAX, SDLoc(Op),
                       Op.getValueType(), Op.getOperand(0), Op.getOperand(1));
  }
  return SDValue();
}

//===----------------------------------------------------------------------===//
//   Custom DAG combine hooks
//===----------------------------------------------------------------------===//
static SDValue
performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
  auto &DAG = DCI.DAG;
  auto Shuffle = cast<ShuffleVectorSDNode>(N);

  // Hoist vector bitcasts that don't change the number of lanes out of unary
  // shuffles, where they are less likely to get in the way of other combines.
  // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
  //  (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
  SDValue Bitcast = N->getOperand(0);
  if (Bitcast.getOpcode() != ISD::BITCAST)
    return SDValue();
  if (!N->getOperand(1).isUndef())
    return SDValue();
  SDValue CastOp = Bitcast.getOperand(0);
  EVT SrcType = CastOp.getValueType();
  EVT DstType = Bitcast.getValueType();
  if (!SrcType.is128BitVector() ||
      SrcType.getVectorNumElements() != DstType.getVectorNumElements())
    return SDValue();
  SDValue NewShuffle = DAG.getVectorShuffle(
      SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
  return DAG.getBitcast(DstType, NewShuffle);
}

/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
/// split up into scalar instructions during legalization, and the vector
/// extending instructions are selected in performVectorExtendCombine below.
static SDValue
performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                               const WebAssemblySubtarget *Subtarget) {
  auto &DAG = DCI.DAG;
  assert(N->getOpcode() == ISD::UINT_TO_FP ||
         N->getOpcode() == ISD::SINT_TO_FP);

  EVT InVT = N->getOperand(0)->getValueType(0);
  EVT ResVT = N->getValueType(0);
  MVT ExtVT;
  if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
    ExtVT = MVT::v4i32;
  else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
    ExtVT = MVT::v2i32;
  else if (Subtarget->hasFP16() && ResVT == MVT::v8f16 && InVT == MVT::v8i8)
    ExtVT = MVT::v8i16;
  else
    return SDValue();

  unsigned Op =
      N->getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
  SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0));
  return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv);
}

static SDValue
performVectorNonNegToFPCombine(SDNode *N,
                               TargetLowering::DAGCombinerInfo &DCI) {
  auto &DAG = DCI.DAG;

  SDNodeFlags Flags = N->getFlags();
  SDValue Op0 = N->getOperand(0);
  EVT VT = N->getValueType(0);

  // Optimize uitofp to sitofp when the sign bit is known to be zero.
  // Depending on the target (runtime) backend, this might be performance
  // neutral (e.g. AArch64) or a significant improvement (e.g. x86_64).
  if (VT.isVector() && (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0))) {
    return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
  }

  return SDValue();
}

static SDValue
performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
  auto &DAG = DCI.DAG;
  assert(N->getOpcode() == ISD::SIGN_EXTEND ||
         N->getOpcode() == ISD::ZERO_EXTEND);

  EVT ResVT = N->getValueType(0);
  bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
  SDLoc DL(N);

  if (ResVT == MVT::v16i32 && N->getOperand(0)->getValueType(0) == MVT::v16i8) {
    // Use a tree of extend low/high to split and extend the input in two
    // layers to avoid doing several shuffles and even more extends.
    unsigned LowOp =
        IsSext ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
    unsigned HighOp =
        IsSext ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
    SDValue Input = N->getOperand(0);
    SDValue LowHalf = DAG.getNode(LowOp, DL, MVT::v8i16, Input);
    SDValue HighHalf = DAG.getNode(HighOp, DL, MVT::v8i16, Input);
    SDValue Subvectors[] = {
        DAG.getNode(LowOp, DL, MVT::v4i32, LowHalf),
        DAG.getNode(HighOp, DL, MVT::v4i32, LowHalf),
        DAG.getNode(LowOp, DL, MVT::v4i32, HighHalf),
        DAG.getNode(HighOp, DL, MVT::v4i32, HighHalf),
    };
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Subvectors);
  }

  // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
  // possible before the extract_subvector can be expanded.
  auto Extract = N->getOperand(0);
  if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
    return SDValue();
  auto Source = Extract.getOperand(0);
  auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
  if (IndexNode == nullptr)
    return SDValue();
  auto Index = IndexNode->getZExtValue();

  // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
  // extracted subvector is the low or high half of its source.
  if (ResVT == MVT::v8i16) {
    if (Extract.getValueType() != MVT::v8i8 ||
        Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
      return SDValue();
  } else if (ResVT == MVT::v4i32) {
    if (Extract.getValueType() != MVT::v4i16 ||
        Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
      return SDValue();
  } else if (ResVT == MVT::v2i64) {
    if (Extract.getValueType() != MVT::v2i32 ||
        Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
      return SDValue();
  } else {
    return SDValue();
  }

  bool IsLow = Index == 0;

  unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
                                : WebAssemblyISD::EXTEND_HIGH_S)
                       : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
                                : WebAssemblyISD::EXTEND_HIGH_U);

  return DAG.getNode(Op, DL, ResVT, Source);
}

static SDValue
performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
  auto &DAG = DCI.DAG;

  auto GetWasmConversionOp = [](unsigned Op) {
    switch (Op) {
    case ISD::FP_TO_SINT_SAT:
      return WebAssemblyISD::TRUNC_SAT_ZERO_S;
    case ISD::FP_TO_UINT_SAT:
      return WebAssemblyISD::TRUNC_SAT_ZERO_U;
    case ISD::FP_ROUND:
      return WebAssemblyISD::DEMOTE_ZERO;
    }
    llvm_unreachable("unexpected op");
  };

  auto IsZeroSplat = [](SDValue SplatVal) {
    auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
    APInt SplatValue, SplatUndef;
    unsigned SplatBitSize;
    bool HasAnyUndefs;
    // Endianness doesn't matter in this context because we are looking for
    // an all-zero value.
    return Splat &&
           Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                                  HasAnyUndefs) &&
           SplatValue == 0;
  };

  if (N->getOpcode() == ISD::CONCAT_VECTORS) {
    // Combine this:
    //
    //   (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
    //
    // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
    //
    // Or this:
    //
    //   (concat_vectors ({v2f32, v4f16} (fp_round ({v2f64, v4f32} $x))),
    //                     ({v2f32, v4f16} (splat 0)))
    //
    // into ({f32x4, f16x8}.demote_zero_{f64x2, f32x4} $x).
    EVT ResVT;
    EVT ExpectedConversionType;
    auto Conversion = N->getOperand(0);
    auto ConversionOp = Conversion.getOpcode();
    switch (ConversionOp) {
    case ISD::FP_TO_SINT_SAT:
    case ISD::FP_TO_UINT_SAT:
      ResVT = MVT::v4i32;
      ExpectedConversionType = MVT::v2i32;
      break;
    case ISD::FP_ROUND:
      if (Conversion.getValueType() == MVT::v2f32) {
        ResVT = MVT::v4f32;
        ExpectedConversionType = MVT::v2f32;
      } else if (Conversion.getValueType() == MVT::v4f16) {
        ResVT = MVT::v8f16;
        ExpectedConversionType = MVT::v4f16;
      } else {
        return SDValue();
      }
      break;
    default:
      return SDValue();
    }

    if (N->getValueType(0) != ResVT)
      return SDValue();

    if (Conversion.getValueType() != ExpectedConversionType)
      return SDValue();

    auto Source = Conversion.getOperand(0);
    if (!((Source.getValueType() == MVT::v2f64 && ResVT == MVT::v4f32) ||
          (Source.getValueType() == MVT::v2f64 && ResVT == MVT::v4i32) ||
          (Source.getValueType() == MVT::v4f32 && ResVT == MVT::v8f16)))
      return SDValue();

    if (!IsZeroSplat(N->getOperand(1)) ||
        N->getOperand(1).getValueType() != ExpectedConversionType)
      return SDValue();

    unsigned Op = GetWasmConversionOp(ConversionOp);
    return DAG.getNode(Op, SDLoc(N), ResVT, Source);
  }

  // Combine this:
  //
  //   (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
  //
  // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
  //
  // Or this:
  //
  //   ({v4f32, v8f16} (fp_round (concat_vectors $x,
  //                               ({v2f64, v4f32} (splat 0)))))
  //
  // into ({f32x4, f16x8}.demote_zero_{f64x2, f32x4} $x).
  EVT ResVT;
  auto ConversionOp = N->getOpcode();
  switch (ConversionOp) {
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    ResVT = MVT::v4i32;
    break;
  case ISD::FP_ROUND:
    ResVT = N->getValueType(0);
    break;
  default:
    llvm_unreachable("unexpected op");
  }

  if (N->getValueType(0) != ResVT)
    return SDValue();

  auto Concat = N->getOperand(0);
  if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
    return SDValue();
  EVT ConcatVT = Concat.getValueType();
  EVT SourceVT = Concat.getOperand(0).getValueType();

  if (!IsZeroSplat(Concat.getOperand(1)))
    return SDValue();

  if (ConversionOp == ISD::FP_ROUND) {
    bool IsF64ToF32 =
        ConcatVT == MVT::v4f64 && SourceVT == MVT::v2f64 && ResVT == MVT::v4f32;
    bool IsF32ToF16 =
        ConcatVT == MVT::v8f32 && SourceVT == MVT::v4f32 && ResVT == MVT::v8f16;
    if (!(IsF64ToF32 || IsF32ToF16))
      return SDValue();
  } else {
    if (ConcatVT != MVT::v4f64 || SourceVT != MVT::v2f64 || ResVT != MVT::v4i32)
      return SDValue();
  }

  unsigned Op = GetWasmConversionOp(ConversionOp);
  return DAG.getNode(Op, SDLoc(N), ResVT, Concat.getOperand(0));
}

// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
                                const SDLoc &DL, unsigned VectorWidth) {
  EVT VT = Vec.getValueType();
  EVT ElVT = VT.getVectorElementType();
  unsigned Factor = VT.getSizeInBits() / VectorWidth;
  EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
                                  VT.getVectorNumElements() / Factor);

  // Extract the relevant VectorWidth bits.  Generate an EXTRACT_SUBVECTOR
  unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
  assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");

  // This is the index of the first element of the VectorWidth-bit chunk
  // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
  IdxVal &= ~(ElemsPerChunk - 1);

  // If the input is a buildvector just emit a smaller one.
  if (Vec.getOpcode() == ISD::BUILD_VECTOR)
    return DAG.getBuildVector(ResultVT, DL,
                              Vec->ops().slice(IdxVal, ElemsPerChunk));

  SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
}

// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
// is the expected destination value type after recursion. In is the initial
// input. Note that the input should have enough leading zero bits to prevent
// NARROW_U from saturating results.
static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,
                                        SelectionDAG &DAG) {
  EVT SrcVT = In.getValueType();

  // No truncation required, we might get here due to recursive calls.
  if (SrcVT == DstVT)
    return In;

  unsigned SrcSizeInBits = SrcVT.getSizeInBits();
  unsigned NumElems = SrcVT.getVectorNumElements();
  if (!isPowerOf2_32(NumElems))
    return SDValue();
  assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
  assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");

  LLVMContext &Ctx = *DAG.getContext();
  EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);

  // Narrow to the largest type possible:
  // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
  EVT InVT = MVT::i16, OutVT = MVT::i8;
  if (SrcVT.getScalarSizeInBits() > 16) {
    InVT = MVT::i32;
    OutVT = MVT::i16;
  }
  unsigned SubSizeInBits = SrcSizeInBits / 2;
  InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
  OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());

  // Split lower/upper subvectors.
  SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
  SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);

  // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
  if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
    Lo = DAG.getBitcast(InVT, Lo);
    Hi = DAG.getBitcast(InVT, Hi);
    SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
    return DAG.getBitcast(DstVT, Res);
  }

  // Recursively narrow lower/upper subvectors, concat result and narrow again.
  EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
  Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
  Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);

  PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
  return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
}

static SDValue performTruncateCombine(SDNode *N,
                                      TargetLowering::DAGCombinerInfo &DCI) {
  auto &DAG = DCI.DAG;

  SDValue In = N->getOperand(0);
  EVT InVT = In.getValueType();
  if (!InVT.isSimple())
    return SDValue();

  EVT OutVT = N->getValueType(0);
  if (!OutVT.isVector())
    return SDValue();

  EVT OutSVT = OutVT.getVectorElementType();
  EVT InSVT = InVT.getVectorElementType();
  // Currently only cover truncate to v16i8 or v8i16.
  if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
        (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
    return SDValue();

  SDLoc DL(N);
  APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(),
                                    OutVT.getScalarSizeInBits());
  In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
  return truncateVectorWithNARROW(OutVT, In, DL, DAG);
}

static SDValue performBitcastCombine(SDNode *N,
                                     TargetLowering::DAGCombinerInfo &DCI) {
  using namespace llvm::SDPatternMatch;
  auto &DAG = DCI.DAG;
  SDLoc DL(N);
  SDValue Src = N->getOperand(0);
  EVT VT = N->getValueType(0);
  EVT SrcVT = Src.getValueType();

  if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
        SrcVT.isFixedLengthVectorOf(MVT::i1)))
    return SDValue();

  unsigned NumElts = SrcVT.getVectorNumElements();
  EVT Width = MVT::getIntegerVT(128 / NumElts);

  // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
  //   ==> bitmask
  if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
    return DAG.getZExtOrTrunc(
        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                    {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
                     DAG.getSExtOrTrunc(N->getOperand(0), DL,
                                        SrcVT.changeVectorElementType(
                                            *DAG.getContext(), Width))}),
        DL, VT);
  }

  // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
  if (NumElts == 32 || NumElts == 64) {
    SDValue Concat, SetCCVector;
    ISD::CondCode SetCond;

    if (!sd_match(N, m_BitCast(m_c_SetCC(SetCond, m_Value(Concat),
                                         m_Value(SetCCVector)))))
      return SDValue();
    if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
      return SDValue();

    // Reconstruct the wide bitmask from each CONCAT_VECTORS operand.
    // Derive the per-chunk mask/integer types from the actual operand type
    // instead of hardcoding v16i1 / i16 for every chunk.
    EVT ConcatOperandVT = Concat.getOperand(0).getValueType();
    unsigned ConcatOperandNumElts = ConcatOperandVT.getVectorNumElements();

    EVT ConcatOperandMaskVT =
        EVT::getVectorVT(*DAG.getContext(), MVT::i1,
                         ElementCount::getFixed(ConcatOperandNumElts));
    EVT ConcatOperandBitmaskVT =
        EVT::getIntegerVT(*DAG.getContext(), ConcatOperandNumElts);
    EVT ReturnVT = N->getValueType(0);
    SDValue ReconstructedBitmask = DAG.getConstant(0, DL, ReturnVT);
    // Example:
    //   v32i16 = concat(v8i16, v8i16, v8i16, v8i16)
    //     -> v8i1 + v8i1 + v8i1 + v8i1
    //     -> i8   + i8   + i8   + i8
    //     -> reconstructed i32 bitmask
    for (size_t I = 0; I < Concat->ops().size(); ++I) {
      SDValue ConcatOperand = Concat.getOperand(I);
      assert(ConcatOperand.getValueType() == ConcatOperandVT &&
             "concat_vectors operands must have the same type");

      SDValue SetCCVectorOperand =
          extractSubVector(SetCCVector, I * ConcatOperandNumElts, DAG, DL, 128);
      if (!SetCCVectorOperand ||
          SetCCVectorOperand.getValueType() != ConcatOperandVT)
        return SDValue();

      // Build the per-chunk mask using the correct chunk type:
      //   v16i8 -> v16i1 -> i16
      //   v8i16 -> v8i1  -> i8
      //   v4i32 -> v4i1  -> i4
      //   v2i64 -> v2i1  -> i2
      SDValue ConcatOperandMask = DAG.getSetCC(
          DL, ConcatOperandMaskVT, ConcatOperand, SetCCVectorOperand, SetCond);
      SDValue ConcatOperandBitmask =
          DAG.getBitcast(ConcatOperandBitmaskVT, ConcatOperandMask);
      SDValue ExtendedConcatOperandBitmask =
          DAG.getZExtOrTrunc(ConcatOperandBitmask, DL, ReturnVT);

      // Shift each chunk's mask to its original lane position before merging it
      // into the result:
      //   Result bit index = I * ConcatOperandNumElts + local lane index
      //
      // Example: four chunks, each containing 8 mask bits:
      //   result = M0 | (M1 << 8) | (M2 << 16) | (M3 << 24)
      //
      SDValue PositionedChunkBitmask = ExtendedConcatOperandBitmask;
      if (I != 0) {
        PositionedChunkBitmask = DAG.getNode(
            ISD::SHL, DL, ReturnVT, ExtendedConcatOperandBitmask,
            DAG.getShiftAmountConstant(I * ConcatOperandNumElts, ReturnVT, DL));
      }

      ReconstructedBitmask = DAG.getNode(
          ISD::OR, DL, ReturnVT, ReconstructedBitmask, PositionedChunkBitmask);
    }

    return ReconstructedBitmask;
  }

  return SDValue();
}

static SDValue performBitmaskCombine(SDNode *N, SelectionDAG &DAG) {
  // bitmask (setcc <X>, 0, setlt) => bitmask X
  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
  using namespace llvm::SDPatternMatch;

  if (N->getConstantOperandVal(0) != Intrinsic::wasm_bitmask)
    return SDValue();

  SDValue LHS;
  if (!sd_match(N->getOperand(1),
                m_c_SpecificSetCC(ISD::SETLT, m_Value(LHS), m_Zero())))
    return SDValue();

  SDLoc DL(N);
  return DAG.getNode(
      ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
      {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32), LHS});
}

static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {
  // any_true (setcc <X>, 0, eq) => (not (all_true X))
  // all_true (setcc <X>, 0, eq) => (not (any_true X))
  // any_true (setcc <X>, 0, ne) => (any_true X)
  // all_true (setcc <X>, 0, ne) => (all_true X)
  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
  using namespace llvm::SDPatternMatch;

  SDValue LHS;
  if (N->getNumOperands() < 2 ||
      !sd_match(N->getOperand(1), m_c_SetCC(m_Value(LHS), m_Zero())))
    return SDValue();
  EVT LT = LHS.getValueType();
  if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
    return SDValue();

  auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
                                 ISD::CondCode SetType,
                                 Intrinsic::WASMIntrinsics InPost) {
    if (N->getConstantOperandVal(0) != InPre)
      return SDValue();

    SDValue LHS;
    if (!sd_match(N->getOperand(1),
                  m_c_SpecificSetCC(SetType, m_Value(LHS), m_Zero())))
      return SDValue();

    SDLoc DL(N);
    SDValue Ret = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                              {DAG.getConstant(InPost, DL, MVT::i32), LHS});
    if (SetType == ISD::SETEQ)
      Ret = DAG.getNode(ISD::XOR, DL, MVT::i32, Ret,
                        DAG.getConstant(1, DL, MVT::i32));
    return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
  };

  if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
                                       Intrinsic::wasm_alltrue))
    return AnyTrueEQ;
  if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
                                       Intrinsic::wasm_anytrue))
    return AllTrueEQ;
  if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
                                       Intrinsic::wasm_anytrue))
    return AnyTrueNE;
  if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
                                       Intrinsic::wasm_alltrue))
    return AllTrueNE;

  return SDValue();
}

struct MaskReduceInfo {
  Intrinsic::ID IID;
  unsigned WideCombineOpcode;
  bool Invert;
};

static SDValue combineSmallMaskReduction(SDNode *N, EVT FromVT,
                                         unsigned NumElts,
                                         const MaskReduceInfo &Info,
                                         SelectionDAG &DAG) {
  EVT VecVT = FromVT.changeVectorElementType(*DAG.getContext(),
                                             MVT::getIntegerVT(128 / NumElts));
  assert(VecVT.getSizeInBits() == 128 &&
         "mask reduction should be widened to a 128-bit vector");

  SDLoc DL(N);
  SDValue Mask = N->getOperand(0)->getOperand(0);
  SDValue Ret = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                            {DAG.getConstant(Info.IID, DL, MVT::i32),
                             DAG.getSExtOrTrunc(Mask, DL, VecVT)});
  if (Info.Invert)
    Ret = DAG.getNode(ISD::XOR, DL, MVT::i32, Ret,
                      DAG.getConstant(1, DL, MVT::i32));
  return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
}

static SDValue combineWideMaskReduction(SDNode *N, SDValue Mask, EVT MaskVT,
                                        unsigned NumElts,
                                        const MaskReduceInfo &Info,
                                        SelectionDAG &DAG) {
  assert((NumElts == 32 || NumElts == 64) &&
         "combineWideMaskReduction is only for wide masks");
  assert(MaskVT.isFixedLengthVector() &&
         MaskVT.getVectorElementType() == MVT::i1);
  SDLoc DL(N);
  unsigned ChunkElts = 16;
  EVT ChunkMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
                                     ElementCount::getFixed(ChunkElts));
  EVT LegalVecVT = ChunkMaskVT.changeVectorElementType(
      *DAG.getContext(), MVT::getIntegerVT(128 / ChunkElts));

  SmallVector<SDValue, 4> ChunkResults;
  // Split the wide mask into v16i1 chunks and reduce each chunk separately.
  // For example:
  //   v32i1:  [0..15] [16..31]
  //              |       |
  //              v       v
  //            chunk0  chunk1
  //
  //   v64i1:  [0..15] [16..31] [32..47] [48..63]
  //              |       |       |       |
  //              v       v       v       v
  //            chunk0  chunk1  chunk2  chunk3
  //
  //   each chunk:
  //     v16i1 -> v16i8 -> wasm_anytrue/alltrue -> i32 0/1
  for (unsigned I = 0; I < NumElts; I += ChunkElts) {
    SDValue ChunkMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ChunkMaskVT,
                                    Mask, DAG.getVectorIdxConstant(I, DL));
    SDValue LegalMask = DAG.getSExtOrTrunc(ChunkMask, DL, LegalVecVT);
    SDValue Reduced =
        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                    DAG.getConstant(Info.IID, DL, MVT::i32), LegalMask);
    ChunkResults.push_back(Reduced);
  }

  SDValue Acc = ChunkResults[0];
  for (unsigned I = 1; I < ChunkResults.size(); ++I)
    Acc =
        DAG.getNode(Info.WideCombineOpcode, DL, MVT::i32, Acc, ChunkResults[I]);

  if (Info.Invert)
    Acc = DAG.getNode(ISD::XOR, DL, MVT::i32, Acc,
                      DAG.getConstant(1, DL, MVT::i32));

  return DAG.getZExtOrTrunc(Acc, DL, N->getValueType(0));
}

static std::optional<MaskReduceInfo> classifyMaskReduction(SDNode *N) {
  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!C)
    return std::nullopt;

  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();

  // setcc (bitcast mask), 0, ne  -> any_true(mask)
  if (C->isZero() && CC == ISD::SETNE)
    return MaskReduceInfo{Intrinsic::wasm_anytrue, ISD::OR, false};

  // setcc (bitcast mask), 0, eq  -> !any_true(mask)
  if (C->isZero() && CC == ISD::SETEQ)
    return MaskReduceInfo{Intrinsic::wasm_anytrue, ISD::OR, true};

  // setcc (bitcast mask), -1, eq -> all_true(mask)
  if (C->isAllOnes() && CC == ISD::SETEQ)
    return MaskReduceInfo{Intrinsic::wasm_alltrue, ISD::AND, false};

  // setcc (bitcast mask), -1, ne -> !all_true(mask)
  if (C->isAllOnes() && CC == ISD::SETNE)
    return MaskReduceInfo{Intrinsic::wasm_alltrue, ISD::AND, true};

  return std::nullopt;
}

/// Try to convert a i128 comparison to a v16i8 comparison before type
/// legalization splits it up into chunks
static SDValue
combineVectorSizedSetCCEquality(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const WebAssemblySubtarget *Subtarget) {

  SDLoc DL(N);
  SDValue X = N->getOperand(0);
  SDValue Y = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT OpVT = X.getValueType();

  SelectionDAG &DAG = DCI.DAG;
  if (DCI.DAG.getMachineFunction().getFunction().hasFnAttribute(
          Attribute::NoImplicitFloat))
    return SDValue();

  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
  // We're looking for an oversized integer equality comparison with SIMD
  if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||
      !Subtarget->hasSIMD128() || !isIntEqualitySetCC(CC))
    return SDValue();

  // Don't perform this combine if constructing the vector will be expensive.
  auto IsVectorBitCastCheap = [](SDValue X) {
    X = peekThroughBitcasts(X);
    return isa<ConstantSDNode>(X) || X.getOpcode() == ISD::LOAD;
  };

  if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
    return SDValue();

  SDValue VecX = DAG.getBitcast(MVT::v16i8, X);
  SDValue VecY = DAG.getBitcast(MVT::v16i8, Y);
  SDValue Cmp = DAG.getSetCC(DL, MVT::v16i8, VecX, VecY, CC);

  SDValue Intr =
      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                  {DAG.getConstant(CC == ISD::SETEQ ? Intrinsic::wasm_alltrue
                                                    : Intrinsic::wasm_anytrue,
                                   DL, MVT::i32),
                   Cmp});

  return DAG.getSetCC(DL, VT, Intr, DAG.getConstant(0, DL, MVT::i32),
                      ISD::SETNE);
}

static SDValue performSETCCCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI,
                                   const WebAssemblySubtarget *Subtarget) {
  if (!DCI.isBeforeLegalize())
    return SDValue();

  EVT VT = N->getValueType(0);
  if (!VT.isScalarInteger())
    return SDValue();

  if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))
    return V;

  SDValue LHS = N->getOperand(0);
  if (LHS->getOpcode() != ISD::BITCAST)
    return SDValue();

  EVT FromVT = LHS->getOperand(0).getValueType();
  if (!FromVT.isFixedLengthVectorOf(MVT::i1))
    return SDValue();

  unsigned NumElts = FromVT.getVectorNumElements();
  auto Info = classifyMaskReduction(N);
  if (!Info)
    return SDValue();

  auto &DAG = DCI.DAG;
  if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16)
    return combineSmallMaskReduction(N, FromVT, NumElts, *Info, DAG);

  if (NumElts == 32 || NumElts == 64)
    return combineWideMaskReduction(N, LHS.getOperand(0), FromVT, NumElts,
                                    *Info, DAG);

  return SDValue();
}

static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  if (VT != MVT::v8i32 && VT != MVT::v16i32)
    return SDValue();

  // Mul with extending inputs.
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  if (LHS.getOpcode() != RHS.getOpcode())
    return SDValue();

  if (LHS.getOpcode() != ISD::SIGN_EXTEND &&
      LHS.getOpcode() != ISD::ZERO_EXTEND)
    return SDValue();

  if (LHS->getOperand(0).getValueType() != RHS->getOperand(0).getValueType())
    return SDValue();

  EVT FromVT = LHS->getOperand(0).getValueType();
  EVT EltTy = FromVT.getVectorElementType();
  if (EltTy != MVT::i8)
    return SDValue();

  // For an input DAG that looks like this
  // %a = input_type
  // %b = input_type
  // %lhs = extend %a to output_type
  // %rhs = extend %b to output_type
  // %mul = mul %lhs, %rhs

  // input_type | output_type | instructions
  // v16i8      | v16i32      | %low = i16x8.extmul_low_i8x16_ %a, %b
  //            |             | %high = i16x8.extmul_high_i8x16_, %a, %b
  //            |             | %low_low = i32x4.ext_low_i16x8_ %low
  //            |             | %low_high = i32x4.ext_high_i16x8_ %low
  //            |             | %high_low = i32x4.ext_low_i16x8_ %high
  //            |             | %high_high = i32x4.ext_high_i16x8_ %high
  //            |             | %res = concat_vector(...)
  // v8i8       | v8i32       | %low = i16x8.extmul_low_i8x16_ %a, %b
  //            |             | %low_low = i32x4.ext_low_i16x8_ %low
  //            |             | %low_high = i32x4.ext_high_i16x8_ %low
  //            |             | %res = concat_vector(%low_low, %low_high)

  SDLoc DL(N);
  unsigned NumElts = VT.getVectorNumElements();
  SDValue ExtendInLHS = LHS->getOperand(0);
  SDValue ExtendInRHS = RHS->getOperand(0);
  bool IsSigned = LHS->getOpcode() == ISD::SIGN_EXTEND;
  unsigned ExtendLowOpc =
      IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
  unsigned ExtendHighOpc =
      IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;

  auto GetExtendLow = [&DAG, &DL, &ExtendLowOpc](EVT VT, SDValue Op) {
    return DAG.getNode(ExtendLowOpc, DL, VT, Op);
  };
  auto GetExtendHigh = [&DAG, &DL, &ExtendHighOpc](EVT VT, SDValue Op) {
    return DAG.getNode(ExtendHighOpc, DL, VT, Op);
  };

  if (NumElts == 16) {
    SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);
    SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);
    SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
    SDValue HighLHS = GetExtendHigh(MVT::v8i16, ExtendInLHS);
    SDValue HighRHS = GetExtendHigh(MVT::v8i16, ExtendInRHS);
    SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v8i16, HighLHS, HighRHS);
    SDValue SubVectors[] = {
        GetExtendLow(MVT::v4i32, MulLow),
        GetExtendHigh(MVT::v4i32, MulLow),
        GetExtendLow(MVT::v4i32, MulHigh),
        GetExtendHigh(MVT::v4i32, MulHigh),
    };
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubVectors);
  } else {
    assert(NumElts == 8);
    SDValue LowLHS = DAG.getNode(LHS->getOpcode(), DL, MVT::v8i16, ExtendInLHS);
    SDValue LowRHS = DAG.getNode(RHS->getOpcode(), DL, MVT::v8i16, ExtendInRHS);
    SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
    SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
    SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
  }
  return SDValue();
}

static SDValue performMulCombine(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI) {
  assert(N->getOpcode() == ISD::MUL);
  EVT VT = N->getValueType(0);
  if (!VT.isVector())
    return SDValue();

  if (auto Res = TryWideExtMulCombine(N, DCI.DAG))
    return Res;

  // We don't natively support v16i8 or v8i8 mul, but we do support v8i16. So,
  // extend them to v8i16.
  if (VT != MVT::v8i8 && VT != MVT::v16i8)
    return SDValue();

  SDLoc DL(N);
  SelectionDAG &DAG = DCI.DAG;
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  EVT MulVT = MVT::v8i16;

  if (VT == MVT::v8i8) {
    SDValue PromotedLHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, LHS,
                                      DAG.getUNDEF(MVT::v8i8));
    SDValue PromotedRHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, RHS,
                                      DAG.getUNDEF(MVT::v8i8));
    SDValue LowLHS =
        DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedLHS);
    SDValue LowRHS =
        DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedRHS);
    SDValue MulLow = DAG.getBitcast(
        MVT::v16i8, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));
    // Take the low byte of each lane.
    SDValue Shuffle = DAG.getVectorShuffle(
        MVT::v16i8, DL, MulLow, DAG.getUNDEF(MVT::v16i8),
        {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});
    return extractSubVector(Shuffle, 0, DAG, DL, 64);
  } else {
    assert(VT == MVT::v16i8 && "Expected v16i8");
    SDValue LowLHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, LHS);
    SDValue LowRHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, RHS);
    SDValue HighLHS =
        DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, LHS);
    SDValue HighRHS =
        DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, RHS);

    SDValue MulLow =
        DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));
    SDValue MulHigh =
        DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, HighLHS, HighRHS));

    // Take the low byte of each lane.
    return DAG.getVectorShuffle(
        VT, DL, MulLow, MulHigh,
        {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
  }
}

SDValue DoubleVectorWidth(SDValue In, unsigned RequiredNumElems,
                          SelectionDAG &DAG) {
  SDLoc DL(In);
  LLVMContext &Ctx = *DAG.getContext();
  EVT InVT = In.getValueType();
  unsigned NumElems = InVT.getVectorNumElements() * 2;
  EVT OutVT = EVT::getVectorVT(Ctx, InVT.getVectorElementType(), NumElems);
  SDValue Concat =
      DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, In, DAG.getPOISON(InVT));
  if (NumElems < RequiredNumElems) {
    return DoubleVectorWidth(Concat, RequiredNumElems, DAG);
  }
  return Concat;
}

SDValue performConvertFPCombine(SDNode *N, SelectionDAG &DAG) {
  EVT OutVT = N->getValueType(0);
  if (!OutVT.isVector())
    return SDValue();

  EVT OutElTy = OutVT.getVectorElementType();
  if (OutElTy != MVT::i8 && OutElTy != MVT::i16)
    return SDValue();

  unsigned NumElems = OutVT.getVectorNumElements();
  if (!isPowerOf2_32(NumElems))
    return SDValue();

  EVT FPVT = N->getOperand(0)->getValueType(0);
  if (FPVT.getVectorElementType() != MVT::f32)
    return SDValue();

  SDLoc DL(N);

  // First, convert to i32.
  LLVMContext &Ctx = *DAG.getContext();
  EVT IntVT = EVT::getVectorVT(Ctx, MVT::i32, NumElems);
  SDValue ToInt = DAG.getNode(N->getOpcode(), DL, IntVT, N->getOperand(0));
  APInt Mask = APInt::getLowBitsSet(IntVT.getScalarSizeInBits(),
                                    OutVT.getScalarSizeInBits());
  // Mask out the top MSBs.
  SDValue Masked =
      DAG.getNode(ISD::AND, DL, IntVT, ToInt, DAG.getConstant(Mask, DL, IntVT));

  if (OutVT.getSizeInBits() < 128) {
    // Create a wide enough vector that we can use narrow.
    EVT NarrowedVT = OutElTy == MVT::i8 ? MVT::v16i8 : MVT::v8i16;
    unsigned NumRequiredElems = NarrowedVT.getVectorNumElements();
    SDValue WideVector = DoubleVectorWidth(Masked, NumRequiredElems, DAG);
    SDValue Trunc = truncateVectorWithNARROW(NarrowedVT, WideVector, DL, DAG);
    return DAG.getBitcast(
        OutVT, extractSubVector(Trunc, 0, DAG, DL, OutVT.getSizeInBits()));
  } else {
    return truncateVectorWithNARROW(OutVT, Masked, DL, DAG);
  }
  return SDValue();
}

// Wide vector shift operations such as v8i32 with sign-extended
// operands cause Type Legalizer crashes because the target-specific
// extension nodes cannot be directly mapped to the 256-bit size.
//
// To resolve the crash and optimize performance, we intercept the
// illegal v8i32 shift in DAGCombine. We convert the shift amounts
// into multipliers and manually split the vector into two v4i32 halves.
//
// Before: t1: v8i32 = shl (sign_extend v8i16), const_vec
// After : t2: v4i32 = mul (ext_low_s v8i16), (ext_low_s narrow_vec)
//         t3: v4i32 = mul (ext_high_s v8i16), (ext_high_s narrow_vec)
//         t4: v8i32 = concat_vectors t2, t3
static SDValue performShiftCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI) {
  SelectionDAG &DAG = DCI.DAG;
  assert(N->getOpcode() == ISD::SHL);
  EVT VT = N->getValueType(0);
  if (VT != MVT::v8i32)
    return SDValue();

  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  unsigned ExtOpc = LHS.getOpcode();
  if (ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND)
    return SDValue();

  if (RHS.getOpcode() != ISD::BUILD_VECTOR)
    return SDValue();

  SDLoc DL(N);
  SDValue ExtendIn = LHS.getOperand(0);
  EVT FromVT = ExtendIn.getValueType();
  if (FromVT != MVT::v8i16)
    return SDValue();

  unsigned NumElts = VT.getVectorNumElements();
  unsigned BitWidth = FromVT.getScalarSizeInBits();
  bool IsSigned = (ExtOpc == ISD::SIGN_EXTEND);
  unsigned MaxValidShift = IsSigned ? (BitWidth - 1) : BitWidth;
  SmallVector<SDValue, 16> MulConsts;
  for (unsigned I = 0; I < NumElts; ++I) {
    auto *C = dyn_cast<ConstantSDNode>(RHS.getOperand(I));
    if (!C)
      return SDValue();

    const APInt &ShiftAmt = C->getAPIntValue();
    if (ShiftAmt.uge(MaxValidShift))
      return SDValue();

    APInt MulAmt = APInt::getOneBitSet(BitWidth, ShiftAmt.getZExtValue());
    MulConsts.push_back(DAG.getConstant(MulAmt, DL, FromVT.getScalarType(),
                                        /*isTarget=*/false, /*isOpaque=*/true));
  }

  SDValue NarrowConst = DAG.getBuildVector(FromVT, DL, MulConsts);
  unsigned ExtLowOpc =
      IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
  unsigned ExtHighOpc =
      IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;

  EVT HalfVT = MVT::v4i32;
  SDValue LHSLo = DAG.getNode(ExtLowOpc, DL, HalfVT, ExtendIn);
  SDValue LHSHi = DAG.getNode(ExtHighOpc, DL, HalfVT, ExtendIn);
  SDValue RHSLo = DAG.getNode(ExtLowOpc, DL, HalfVT, NarrowConst);
  SDValue RHSHi = DAG.getNode(ExtHighOpc, DL, HalfVT, NarrowConst);
  SDValue MulLo = DAG.getNode(ISD::MUL, DL, HalfVT, LHSLo, RHSLo);
  SDValue MulHi = DAG.getNode(ISD::MUL, DL, HalfVT, LHSHi, RHSHi);
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, MulLo, MulHi);
}

static SDValue performMinMaxF128Combine(SDNode *N, SelectionDAG &DAG) {
  if (N->getValueType(0) != MVT::f128)
    return SDValue();

  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  switch (N->getOpcode()) {
  // wasi-libc and emscripten do not currently define fminimuml and fmaximuml.
  case ISD::FMINIMUM:
  case ISD::FMAXIMUM:
    return TLI.expandFMINIMUM_FMAXIMUM(N, DAG);

  // wasi-libc and emscripten do not currently define fminimum_numl and
  // fmaximum_numl.
  case ISD::FMINIMUMNUM:
  case ISD::FMAXIMUMNUM:
    return TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N, DAG);

  default:
    return SDValue();
  }
}

SDValue
WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
  switch (N->getOpcode()) {
  default:
    return SDValue();
  case ISD::BITCAST:
    return performBitcastCombine(N, DCI);
  case ISD::SETCC:
    return performSETCCCombine(N, DCI, Subtarget);
  case ISD::VECTOR_SHUFFLE:
    return performVECTOR_SHUFFLECombine(N, DCI);
  case ISD::SIGN_EXTEND:
  case ISD::ZERO_EXTEND:
    return performVectorExtendCombine(N, DCI);
  case ISD::UINT_TO_FP:
    if (auto ExtCombine = performVectorExtendToFPCombine(N, DCI, Subtarget))
      return ExtCombine;
    return performVectorNonNegToFPCombine(N, DCI);
  case ISD::SINT_TO_FP:
    return performVectorExtendToFPCombine(N, DCI, Subtarget);
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
  case ISD::FP_ROUND:
  case ISD::CONCAT_VECTORS:
    return performVectorTruncZeroCombine(N, DCI);
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT:
    return performConvertFPCombine(N, DCI.DAG);
  case ISD::TRUNCATE:
    return performTruncateCombine(N, DCI);
  case ISD::INTRINSIC_WO_CHAIN: {
    if (SDValue V = performBitmaskCombine(N, DCI.DAG))
      return V;
    return performAnyAllCombine(N, DCI.DAG);
  }
  case ISD::MUL:
    return performMulCombine(N, DCI);
  case ISD::SHL:
    return performShiftCombine(N, DCI);
  case ISD::FMINIMUM:
  case ISD::FMAXIMUM:
  case ISD::FMINIMUMNUM:
  case ISD::FMAXIMUMNUM:
    return performMinMaxF128Combine(N, DCI.DAG);
  }
}