//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../ExprConstShared.h"
#include "Boolean.h"
#include "Char.h"
#include "EvalEmitter.h"
#include "InterpBuiltinBitCast.h"
#include "InterpHelpers.h"
#include "PrimType.h"
#include "Program.h"
#include "clang/AST/InferAlloc.h"
#include "clang/AST/OSLog.h"
#include "clang/AST/RecordLayout.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/TargetBuiltins.h"
#include "clang/Basic/TargetInfo.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/AllocToken.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/SipHash.h"

namespace clang {
namespace interp {

[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
  switch (ID) {
  case Builtin::BIas_const:
  case Builtin::BIforward:
  case Builtin::BIforward_like:
  case Builtin::BImove:
  case Builtin::BImove_if_noexcept:
  case Builtin::BIaddressof:
  case Builtin::BI__addressof:
  case Builtin::BI__builtin_addressof:
  case Builtin::BI__builtin_launder:
    return true;
  default:
    return false;
  }
  return false;
}

static void discard(InterpStack &Stk, PrimType T) {
  TYPE_SWITCH(T, { Stk.discard<T>(); });
}

static bool popToUInt64(const InterpState &S, const Expr *E, uint64_t &Out) {
  INT_TYPE_SWITCH(*S.getContext().classify(E->getType()), {
    const auto &Val = S.Stk.pop<T>();
    if (!Val.isNumber())
      return false;
    Out = static_cast<uint64_t>(Val);
    return true;
  });
}

static bool popToAPSInt(InterpStack &Stk, PrimType T, APSInt &Out) {
  INT_TYPE_SWITCH(T, {
    const auto &Val = Stk.pop<T>();
    if (!Val.isNumber())
      return false;
    Out = Val.toAPSInt();
    return true;
  });
}

static bool popToAPSInt(InterpState &S, const Expr *E, APSInt &Out) {
  return popToAPSInt(S.Stk, *S.getContext().classify(E->getType()), Out);
}
static bool popToAPSInt(InterpState &S, QualType T, APSInt &Out) {
  return popToAPSInt(S.Stk, *S.getContext().classify(T), Out);
}

/// Check for common reasons a pointer can't be read from, which
/// are usually not diagnosed in a builtin function.
static bool isReadable(const Pointer &P) {
  if (P.isDummy())
    return false;
  if (!P.isReadablePointerType())
    return false;
  if (!P.isLive())
    return false;
  if (P.isOnePastEnd())
    return false;
  return true;
}

/// Pushes \p Val on the stack as the type given by \p QT.
static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
  assert(QT->isSignedIntegerOrEnumerationType() ||
         QT->isUnsignedIntegerOrEnumerationType());
  OptPrimType T = *S.getContext().classify(QT);
  assert(T);

  if (T == PT_IntAPS) {
    unsigned BitWidth = S.getASTContext().getIntWidth(QT);
    auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
    Result.copy(Val.extOrTrunc(BitWidth));
    S.Stk.push<IntegralAP<true>>(Result);
    return;
  }

  if (T == PT_IntAP) {
    unsigned BitWidth = S.getASTContext().getIntWidth(QT);
    auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
    Result.copy(Val.extOrTrunc(BitWidth));
    S.Stk.push<IntegralAP<false>>(Result);
    return;
  }

  if (isSignedType(*T)) {
    int64_t V = Val.getSExtValue();
    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
  } else {
    assert(QT->isUnsignedIntegerOrEnumerationType());
    uint64_t V = Val.getZExtValue();
    INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V)); });
  }
}

template <typename T>
static void pushInteger(InterpState &S, T Val, QualType QT) {
  if constexpr (std::is_same_v<T, APInt>)
    pushInteger(S, APSInt(Val, !std::is_signed_v<T>), QT);
  else if constexpr (std::is_same_v<T, APSInt>)
    pushInteger(S, Val, QT);
  else
    pushInteger(S,
                APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
                             std::is_signed_v<T>),
                       !std::is_signed_v<T>),
                QT);
}

static void assignIntegral(InterpState &S, const Pointer &Dest, PrimType ValueT,
                           const APSInt &Value) {

  if (ValueT == PT_IntAPS) {
    Dest.deref<IntegralAP<true>>() =
        S.allocAP<IntegralAP<true>>(Value.getBitWidth());
    Dest.deref<IntegralAP<true>>().copy(Value);
  } else if (ValueT == PT_IntAP) {
    Dest.deref<IntegralAP<false>>() =
        S.allocAP<IntegralAP<false>>(Value.getBitWidth());
    Dest.deref<IntegralAP<false>>().copy(Value);
  } else if (ValueT == PT_Bool) {
    Dest.deref<Boolean>() = Boolean::from(!Value.isZero());
  } else {
    INT_TYPE_SWITCH_NO_BOOL(
        ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
  }
}

static QualType getElemType(const Pointer &P) {
  if (P.isStringPointer()) {
    return P.asStringPointer()
        .getLiteral()
        ->getType()
        ->getAsArrayTypeUnsafe()
        ->getElementType();
  }

  if (P.isOpaquePointer() || P.isIntegralPointer())
    return P.getType();

  const Descriptor *Desc = P.getFieldDesc();
  QualType T = Desc->getType();
  if (Desc->isPrimitive())
    return T;
  if (T->isPointerType())
    return T->castAs<PointerType>()->getPointeeType();
  if (Desc->isArray())
    return Desc->getElemQualType();
  if (const auto *AT = T->getAsArrayTypeUnsafe())
    return AT->getElementType();
  return T;
}

static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
                                        unsigned ID) {
  if (!S.diagnosing())
    return;

  auto Loc = S.Current->getSource(OpPC);
  if (S.getLangOpts().CPlusPlus11)
    S.CCEDiag(Loc, diag::note_constexpr_invalid_function)
        << /*isConstexpr=*/0 << /*isConstructor=*/0
        << S.getASTContext().BuiltinInfo.getQuotedName(ID);
  else
    S.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
}

static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
  assert(Val.getFieldDesc()->isPrimitiveArray() &&
         Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
         "Not a boolean vector");
  unsigned NumElems = Val.getNumElems();

  // Each element is one bit, so create an integer with NumElts bits.
  llvm::APSInt Result(NumElems, 0);
  for (unsigned I = 0; I != NumElems; ++I) {
    if (Val.elem<bool>(I))
      Result.setBit(I);
  }

  return Result;
}

// Strict double -> float conversion used for X86 PD2PS/cvtsd2ss intrinsics.
// Reject NaN/Inf/Subnormal inputs and any lossy/inexact conversions.
static bool convertDoubleToFloatStrict(const APFloat &Src, Floating &Dst,
                                       InterpState &S, const Expr *DiagExpr) {
  if (Src.isInfinity()) {
    if (S.diagnosing())
      S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic) << 0;
    return false;
  }
  if (Src.isNaN()) {
    if (S.diagnosing())
      S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic) << 1;
    return false;
  }
  APFloat Val = Src;
  bool LosesInfo = false;
  APFloat::opStatus Status = Val.convert(
      APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, &LosesInfo);
  if (LosesInfo || Val.isDenormal()) {
    if (S.diagnosing())
      S.CCEDiag(DiagExpr, diag::note_constexpr_float_arithmetic_strict);
    return false;
  }
  if (Status != APFloat::opOK) {
    if (S.diagnosing())
      S.CCEDiag(DiagExpr, diag::note_invalid_subexpr_in_const_expr);
    return false;
  }
  Dst.copy(Val);
  return true;
}

static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
                                                  const InterpFrame *Frame,
                                                  const CallExpr *Call) {
  unsigned Depth = S.Current->getDepth();
  auto isStdCall = [](const FunctionDecl *F) -> bool {
    return F && F->isInStdNamespace() && F->getIdentifier() &&
           F->getIdentifier()->isStr("is_constant_evaluated");
  };
  const InterpFrame *Caller = Frame->Caller;
  // The current frame is the one for __builtin_is_constant_evaluated.
  // The one above that, potentially the one for std::is_constant_evaluated().
  if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
      S.getEvalStatus().Diag &&
      (Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
    if (Caller && isStdCall(Frame->getCallee())) {
      const Expr *E = Caller->getExpr(Caller->getRetPC());
      S.report(E->getExprLoc(),
               diag::warn_is_constant_evaluated_always_true_constexpr)
          << "std::is_constant_evaluated" << E->getSourceRange();
    } else {
      S.report(Call->getExprLoc(),
               diag::warn_is_constant_evaluated_always_true_constexpr)
          << "__builtin_is_constant_evaluated" << Call->getSourceRange();
    }
  }

  S.Stk.push<Boolean>(Boolean::from(S.inConstantContext()));
  return true;
}

// __builtin_assume
// __assume (MS extension)
static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call) {
  // Nothing to be done here since the argument is NOT evaluated.
  assert(Call->getNumArgs() == 1);
  return true;
}

static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call, unsigned ID) {
  uint64_t Limit = ~static_cast<uint64_t>(0);
  if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
      ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp) {
    if (!popToUInt64(S, Call->getArg(2), Limit))
      return false;
  }

  const Pointer &B = S.Stk.pop<Pointer>();
  const Pointer &A = S.Stk.pop<Pointer>();
  if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
      ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
    diagnoseNonConstexprBuiltin(S, OpPC, ID);

  if (Limit == 0) {
    pushInteger(S, 0, Call->getType());
    return true;
  }

  if (!CheckLive(S, OpPC, A, AK_Read) || !CheckLive(S, OpPC, B, AK_Read))
    return false;

  if (!A.isReadablePointerType() || !B.isReadablePointerType())
    return false;

  if (A.isDummy() || B.isDummy() || A.isUnknownSizeArray() ||
      B.isUnknownSizeArray())
    return false;

  bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
                ID == Builtin::BI__builtin_wcscmp ||
                ID == Builtin::BI__builtin_wcsncmp;

  QualType ElemTy = getElemType(A);
  // Different element types shouldn't happen, but with casts they can.
  if (!S.getASTContext().hasSameUnqualifiedType(ElemTy, getElemType(B)))
    return false;

  PrimType ElemT = *S.getContext().classify(ElemTy);

  auto returnResult = [&](int V) -> bool {
    pushInteger(S, V, Call->getType());
    return true;
  };

  unsigned IndexA = A.getIndex();
  unsigned IndexB = B.getIndex();
  unsigned NumElemsA = A.getNumElems();
  unsigned NumElemsB = B.getNumElems();
  uint64_t Steps = 0;
  for (;; ++IndexA, ++IndexB, ++Steps) {

    if (Steps >= Limit)
      break;

    // Diagnose this as a read of one-past-the-end.
    if (IndexA >= NumElemsA || IndexB >= NumElemsB) {
      S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)
          << AK_Read << S.Current->getRange(OpPC);
      return false;
    }

    if (IsWide) {
      INT_TYPE_SWITCH(ElemT, {
        T CA = A.loadElem<T>(IndexA);
        T CB = B.loadElem<T>(IndexB);
        if (CA > CB)
          return returnResult(1);
        if (CA < CB)
          return returnResult(-1);
        if (CA.isZero() || CB.isZero())
          return returnResult(0);
      });
      continue;
    }

    uint8_t CA = A.loadElem<uint8_t>(IndexA);
    uint8_t CB = B.loadElem<uint8_t>(IndexB);

    if (CA > CB)
      return returnResult(1);
    if (CA < CB)
      return returnResult(-1);
    if (CA == 0 || CB == 0)
      return returnResult(0);
  }

  return returnResult(0);
}

static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call, unsigned ID) {
  const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();

  if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
    diagnoseNonConstexprBuiltin(S, OpPC, ID);

  if (StrPtr.isConstexprUnknown())
    return false;

  if (!CheckArray(S, OpPC, StrPtr))
    return false;

  if (!CheckLive(S, OpPC, StrPtr, AK_Read))
    return false;

  // For string literal pointers, this is pretty simple.
  if (StrPtr.isStringPointer()) {
    if (StrPtr.isOnePastEnd())
      return CheckRange(S, OpPC, StrPtr, AK_Read);

    const auto *Lit = StrPtr.asStringPointer().getLiteral();
    int64_t Off = StrPtr.getByteOffset();
    if (Off < 0)
      return false;

    UnsignedOrNone ZeroIndex = Lit->findZeroCodeUnit(Off);
    if (!ZeroIndex)
      return false;
    pushInteger(S, *ZeroIndex, Call->getType());
    return true;
  }

  if (!StrPtr.isBlockPointer())
    return false;

  if (!CheckDummy(S, OpPC, StrPtr, AK_Read))
    return false;

  if (!StrPtr.getFieldDesc()->isPrimitiveArray())
    return false;

  assert(StrPtr.getFieldDesc()->isPrimitiveArray());
  PrimType ElemT = StrPtr.getFieldDesc()->getPrimType();
  unsigned ElemSize = StrPtr.getFieldDesc()->getElemDataSize();
  if (ElemSize != 1 && ElemSize != 2 && ElemSize != 4)
    return Invalid(S, OpPC);

  if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
    const ASTContext &AC = S.getASTContext();
    unsigned WCharSize = AC.getTypeSizeInChars(AC.getWCharType()).getQuantity();
    if (StrPtr.getFieldDesc()->getElemDataSize() != WCharSize)
      return false;
  }

  size_t Len = 0;
  for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
    PtrView ElemPtr = StrPtr.view().atIndex(I);

    if (!CheckRange(S, OpPC, ElemPtr, AK_Read))
      return false;

    uint32_t Val;
    FIXED_SIZE_INT_TYPE_SWITCH(
        ElemT, { Val = static_cast<uint32_t>(ElemPtr.deref<T>()); });
    if (Val == 0)
      break;
  }

  pushInteger(S, Len, Call->getType());

  return true;
}

static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
                                const InterpFrame *Frame, const CallExpr *Call,
                                bool Signaling) {
  const Pointer &Arg = S.Stk.pop<Pointer>();

  if (!CheckLoad(S, OpPC, Arg))
    return false;

  // Convert the given string to an integer using StringRef's API.
  llvm::APInt Fill;
  if (Arg.isBlockPointer()) {
    if (!Arg.getFieldDesc()->isPrimitiveArray())
      return Invalid(S, OpPC);

    std::string Str;
    unsigned ArgLength = Arg.getNumElems();
    bool FoundZero = false;
    for (unsigned I = 0; I != ArgLength; ++I) {
      if (!Arg.isElementInitialized(I))
        return false;

      if (Arg.loadElem<int8_t>(I) == 0) {
        FoundZero = true;
        break;
      }
      Str += Arg.elem<char>(I);
    }

    // If we didn't find a NUL byte, diagnose as a one-past-the-end read.
    if (!FoundZero)
      return CheckRange(S, OpPC, Arg.atIndex(ArgLength), AK_Read);

    // Treat empty strings as if they were zero.
    if (Str.empty())
      Fill = llvm::APInt(32, 0);
    else if (StringRef(Str).getAsInteger(0, Fill))
      return false;
  } else if (Arg.isStringPointer()) {
    if (!Arg.asStringPointer().getLiteral()->isOrdinary())
      return false;
    StringRef Str = Arg.asStringPointer().getLiteral()->getString();
    // Treat empty strings as if they were zero.
    if (Str.empty())
      Fill = llvm::APInt(32, 0);
    else if (StringRef(Str).getAsInteger(0, Fill))
      return false;
  } else {
    return false;
  }

  const llvm::fltSemantics &TargetSemantics =
      S.getASTContext().getFloatTypeSemantics(
          Call->getDirectCallee()->getReturnType());

  Floating Result = S.allocFloat(TargetSemantics);
  if (S.getASTContext().getTargetInfo().isNan2008()) {
    if (Signaling)
      Result.copy(
          llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
    else
      Result.copy(
          llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
  } else {
    // Prior to IEEE 754-2008, architectures were allowed to choose whether
    // the first bit of their significand was set for qNaN or sNaN. MIPS chose
    // a different encoding to what became a standard in 2008, and for pre-
    // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
    // sNaN. This is now known as "legacy NaN" encoding.
    if (Signaling)
      Result.copy(
          llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
    else
      Result.copy(
          llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
  }

  S.Stk.push<Floating>(Result);
  return true;
}

static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
                                const InterpFrame *Frame,
                                const CallExpr *Call) {
  const llvm::fltSemantics &TargetSemantics =
      S.getASTContext().getFloatTypeSemantics(
          Call->getDirectCallee()->getReturnType());

  Floating Result = S.allocFloat(TargetSemantics);
  Result.copy(APFloat::getInf(TargetSemantics));
  S.Stk.push<Floating>(Result);
  return true;
}

static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame) {
  const Floating &Arg2 = S.Stk.pop<Floating>();
  const Floating &Arg1 = S.Stk.pop<Floating>();
  Floating Result = S.allocFloat(Arg1.getSemantics());

  APFloat Copy = Arg1.getAPFloat();
  Copy.copySign(Arg2.getAPFloat());
  Result.copy(Copy);
  S.Stk.push<Floating>(Result);

  return true;
}

static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame, bool IsNumBuiltin) {
  const Floating &RHS = S.Stk.pop<Floating>();
  const Floating &LHS = S.Stk.pop<Floating>();
  Floating Result = S.allocFloat(LHS.getSemantics());

  if (IsNumBuiltin)
    Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
  else
    Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
  S.Stk.push<Floating>(Result);
  return true;
}

static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame, bool IsNumBuiltin) {
  const Floating &RHS = S.Stk.pop<Floating>();
  const Floating &LHS = S.Stk.pop<Floating>();
  Floating Result = S.allocFloat(LHS.getSemantics());

  if (IsNumBuiltin)
    Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
  else
    Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
  S.Stk.push<Floating>(Result);
  return true;
}

/// Defined as __builtin_isnan(...), to accommodate the fact that it can
/// take a float, double, long double, etc.
/// But for us, that's all a Floating anyway.
static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame,
                                  const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isNan(), Call->getType());
  return true;
}

static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isSignaling(), Call->getType());
  return true;
}

static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool CheckSign,
                                  const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();
  APFloat F = Arg.getAPFloat();
  bool IsInf = F.isInfinity();

  if (CheckSign)
    pushInteger(S, IsInf ? (F.isNegative() ? -1 : 1) : 0, Call->getType());
  else
    pushInteger(S, IsInf, Call->getType());
  return true;
}

static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isFinite(), Call->getType());
  return true;
}

static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isNormal(), Call->getType());
  return true;
}

static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isDenormal(), Call->getType());
  return true;
}

static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isZero(), Call->getType());
  return true;
}

static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call) {
  const Floating &Arg = S.Stk.pop<Floating>();

  pushInteger(S, Arg.isNegative(), Call->getType());
  return true;
}

static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
                                       const CallExpr *Call, unsigned ID) {
  const Floating &RHS = S.Stk.pop<Floating>();
  const Floating &LHS = S.Stk.pop<Floating>();

  pushInteger(
      S,
      [&] {
        switch (ID) {
        case Builtin::BI__builtin_isgreater:
          return LHS > RHS;
        case Builtin::BI__builtin_isgreaterequal:
          return LHS >= RHS;
        case Builtin::BI__builtin_isless:
          return LHS < RHS;
        case Builtin::BI__builtin_islessequal:
          return LHS <= RHS;
        case Builtin::BI__builtin_islessgreater: {
          ComparisonCategoryResult Cmp = LHS.compare(RHS);
          return Cmp == ComparisonCategoryResult::Less ||
                 Cmp == ComparisonCategoryResult::Greater;
        }
        case Builtin::BI__builtin_isunordered:
          return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
        default:
          llvm_unreachable("Unexpected builtin ID: Should be a floating point "
                           "comparison function");
        }
      }(),
      Call->getType());
  return true;
}

/// First parameter to __builtin_isfpclass is the floating value, the
/// second one is an integral value.
static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame,
                                      const CallExpr *Call) {
  APSInt FPClassArg;
  if (!popToAPSInt(S, Call->getArg(1), FPClassArg))
    return false;
  const Floating &F = S.Stk.pop<Floating>();

  int32_t Result = static_cast<int32_t>(
      (F.classify() & std::move(FPClassArg)).getZExtValue());
  pushInteger(S, Result, Call->getType());

  return true;
}

/// Five int values followed by one floating value.
/// __builtin_fpclassify(int, int, int, int, int, float)
static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
                                       const InterpFrame *Frame,
                                       const CallExpr *Call) {
  const Floating &Val = S.Stk.pop<Floating>();

  PrimType IntT = *S.getContext().classify(Call->getArg(0));
  APSInt Values[5];
  for (unsigned I = 0; I != 5; ++I) {
    if (!popToAPSInt(S.Stk, IntT, Values[4 - I]))
      return false;
  }

  unsigned Index;
  switch (Val.getCategory()) {
  case APFloat::fcNaN:
    Index = 0;
    break;
  case APFloat::fcInfinity:
    Index = 1;
    break;
  case APFloat::fcNormal:
    Index = Val.isDenormal() ? 3 : 2;
    break;
  case APFloat::fcZero:
    Index = 4;
    break;
  }

  // The last argument is first on the stack.
  assert(Index <= 4);

  pushInteger(S, Values[Index], Call->getType());
  return true;
}

static inline Floating abs(InterpState &S, const Floating &In) {
  if (!In.isNegative())
    return In;

  Floating Output = S.allocFloat(In.getSemantics());
  APFloat New = In.getAPFloat();
  New.changeSign();
  Output.copy(New);
  return Output;
}

// The C standard says "fabs raises no floating-point exceptions,
// even if x is a signaling NaN. The returned value is independent of
// the current rounding direction mode."  Therefore constant folding can
// proceed without regard to the floating point settings.
// Reference, WG14 N2478 F.10.4.3
static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame) {
  const Floating &Val = S.Stk.pop<Floating>();
  S.Stk.push<Floating>(abs(S, Val));
  return true;
}

static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
                                const InterpFrame *Frame,
                                const CallExpr *Call) {
  APSInt Val;
  if (!popToAPSInt(S, Call->getArg(0), Val))
    return false;
  if (Val ==
      APSInt(APInt::getSignedMinValue(Val.getBitWidth()), /*IsUnsigned=*/false))
    return false;
  if (Val.isNegative())
    Val.negate();
  pushInteger(S, Val, Call->getType());
  return true;
}

static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
                                     const InterpFrame *Frame,
                                     const CallExpr *Call) {
  APSInt Val;
  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
    const Pointer &Arg = S.Stk.pop<Pointer>();
    Val = convertBoolVectorToInt(Arg);
  } else {
    if (!popToAPSInt(S, Call->getArg(0), Val))
      return false;
  }
  pushInteger(S, Val.popcount(), Call->getType());
  return true;
}

static bool interp__builtin_ia32_crc32(InterpState &S, CodePtr OpPC,
                                       const InterpFrame *Frame,
                                       const CallExpr *Call,
                                       unsigned DataBytes) {
  uint64_t DataVal;
  if (!popToUInt64(S, Call->getArg(1), DataVal))
    return false;
  uint64_t CRCVal;
  if (!popToUInt64(S, Call->getArg(0), CRCVal))
    return false;

  // CRC32C polynomial (iSCSI polynomial, bit-reversed)
  static const uint32_t CRC32C_POLY = 0x82F63B78;

  // Process each byte
  uint32_t Result = static_cast<uint32_t>(CRCVal);
  for (unsigned I = 0; I != DataBytes; ++I) {
    uint8_t Byte = static_cast<uint8_t>((DataVal >> (I * 8)) & 0xFF);
    Result ^= Byte;
    for (int J = 0; J != 8; ++J) {
      Result = (Result >> 1) ^ ((Result & 1) ? CRC32C_POLY : 0);
    }
  }

  pushInteger(S, Result, Call->getType());
  return true;
}

static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
                                          const InterpFrame *Frame,
                                          const CallExpr *Call) {
  // This is an unevaluated call, so there are no arguments on the stack.
  assert(Call->getNumArgs() == 1);
  const Expr *Arg = Call->getArg(0);

  GCCTypeClass ResultClass =
      EvaluateBuiltinClassifyType(Arg->getType(), S.getLangOpts());
  int32_t ReturnVal = static_cast<int32_t>(ResultClass);
  pushInteger(S, ReturnVal, Call->getType());
  return true;
}

// __builtin_expect(long, long)
// __builtin_expect_with_probability(long, long, double)
static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call) {
  // The return value is simply the value of the first parameter.
  // We ignore the probability.
  unsigned NumArgs = Call->getNumArgs();
  assert(NumArgs == 2 || NumArgs == 3);

  PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
  if (NumArgs == 3)
    S.Stk.discard<Floating>();
  discard(S.Stk, ArgT);
  // Top of the stack is now the first paramter. Leave it there as the return
  // value.

  return true;
}

static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame,
                                      const CallExpr *Call) {
#ifndef NDEBUG
  assert(Call->getArg(0)->isLValue());
  PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
  assert(PtrT == PT_Ptr &&
         "Unsupported pointer type passed to __builtin_addressof()");
#endif
  return true;
}

static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
                                 const InterpFrame *Frame,
                                 const CallExpr *Call) {
  return Call->getDirectCallee()->isConstexpr();
}

static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
                                                 const InterpFrame *Frame,
                                                 const CallExpr *Call) {
  APSInt Arg;
  if (!popToAPSInt(S, Call->getArg(0), Arg))
    return false;

  int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
      Arg.getZExtValue());
  pushInteger(S, Result, Call->getType());
  return true;
}

// Two integral values followed by a pointer (lhs, rhs, resultOut)
static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
                                       const CallExpr *Call,
                                       unsigned BuiltinOp) {
  const Pointer &ResultPtr = S.Stk.pop<Pointer>();
  if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
    return false;

  PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
  PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
  APSInt RHS;
  if (!popToAPSInt(S.Stk, RHST, RHS))
    return false;
  APSInt LHS;
  if (!popToAPSInt(S.Stk, LHST, LHS))
    return false;
  QualType ResultType = Call->getArg(2)->getType()->getPointeeType();
  PrimType ResultT = *S.getContext().classify(ResultType);
  bool Overflow;

  APSInt Result;
  if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
      BuiltinOp == Builtin::BI__builtin_sub_overflow ||
      BuiltinOp == Builtin::BI__builtin_mul_overflow) {
    bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
                    ResultType->isSignedIntegerOrEnumerationType();
    bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
                     ResultType->isSignedIntegerOrEnumerationType();
    uint64_t LHSSize = LHS.getBitWidth();
    uint64_t RHSSize = RHS.getBitWidth();
    uint64_t ResultSize = S.getASTContext().getIntWidth(ResultType);
    uint64_t MaxBits = std::max(std::max(LHSSize, RHSSize), ResultSize);

    // Add an additional bit if the signedness isn't uniformly agreed to. We
    // could do this ONLY if there is a signed and an unsigned that both have
    // MaxBits, but the code to check that is pretty nasty.  The issue will be
    // caught in the shrink-to-result later anyway.
    if (IsSigned && !AllSigned)
      ++MaxBits;

    LHS = APSInt(LHS.extOrTrunc(MaxBits), !IsSigned);
    RHS = APSInt(RHS.extOrTrunc(MaxBits), !IsSigned);
    Result = APSInt(MaxBits, !IsSigned);
  }

  // Find largest int.
  switch (BuiltinOp) {
  default:
    llvm_unreachable("Invalid value for BuiltinOp");
  case Builtin::BI__builtin_add_overflow:
  case Builtin::BI__builtin_sadd_overflow:
  case Builtin::BI__builtin_saddl_overflow:
  case Builtin::BI__builtin_saddll_overflow:
  case Builtin::BI__builtin_uadd_overflow:
  case Builtin::BI__builtin_uaddl_overflow:
  case Builtin::BI__builtin_uaddll_overflow:
    Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
                            : LHS.uadd_ov(RHS, Overflow);
    break;
  case Builtin::BI__builtin_sub_overflow:
  case Builtin::BI__builtin_ssub_overflow:
  case Builtin::BI__builtin_ssubl_overflow:
  case Builtin::BI__builtin_ssubll_overflow:
  case Builtin::BI__builtin_usub_overflow:
  case Builtin::BI__builtin_usubl_overflow:
  case Builtin::BI__builtin_usubll_overflow:
    Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
                            : LHS.usub_ov(RHS, Overflow);
    break;
  case Builtin::BI__builtin_mul_overflow:
  case Builtin::BI__builtin_smul_overflow:
  case Builtin::BI__builtin_smull_overflow:
  case Builtin::BI__builtin_smulll_overflow:
  case Builtin::BI__builtin_umul_overflow:
  case Builtin::BI__builtin_umull_overflow:
  case Builtin::BI__builtin_umulll_overflow:
    Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
                            : LHS.umul_ov(RHS, Overflow);
    break;
  }

  // In the case where multiple sizes are allowed, truncate and see if
  // the values are the same.
  if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
      BuiltinOp == Builtin::BI__builtin_sub_overflow ||
      BuiltinOp == Builtin::BI__builtin_mul_overflow) {
    // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
    // since it will give us the behavior of a TruncOrSelf in the case where
    // its parameter <= its size.  We previously set Result to be at least the
    // integer width of the result, so getIntWidth(ResultType) <=
    // Result.BitWidth
    APSInt Temp = Result.extOrTrunc(S.getASTContext().getIntWidth(ResultType));
    Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());

    if (!APSInt::isSameValue(Temp, Result))
      Overflow = true;
    Result = std::move(Temp);
  }

  // Write Result to ResultPtr and put Overflow on the stack.
  assignIntegral(S, ResultPtr, ResultT, Result);
  if (ResultPtr.canBeInitialized())
    ResultPtr.initialize();

  assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
  S.Stk.push<Boolean>(Overflow);
  return true;
}

/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call, unsigned BuiltinOp) {
  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
  PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
  PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
  APSInt CarryIn;
  if (!popToAPSInt(S.Stk, LHST, CarryIn))
    return false;
  APSInt RHS;
  if (!popToAPSInt(S.Stk, RHST, RHS))
    return false;
  APSInt LHS;
  if (!popToAPSInt(S.Stk, LHST, LHS))
    return false;

  if (!isReadable(CarryOutPtr))
    return false;

  APSInt CarryOut;

  APSInt Result;
  // Copy the number of bits and sign.
  Result = LHS;
  CarryOut = LHS;

  bool FirstOverflowed = false;
  bool SecondOverflowed = false;
  switch (BuiltinOp) {
  default:
    llvm_unreachable("Invalid value for BuiltinOp");
  case Builtin::BI__builtin_addcb:
  case Builtin::BI__builtin_addcs:
  case Builtin::BI__builtin_addc:
  case Builtin::BI__builtin_addcl:
  case Builtin::BI__builtin_addcll:
    Result =
        LHS.uadd_ov(RHS, FirstOverflowed).uadd_ov(CarryIn, SecondOverflowed);
    break;
  case Builtin::BI__builtin_subcb:
  case Builtin::BI__builtin_subcs:
  case Builtin::BI__builtin_subc:
  case Builtin::BI__builtin_subcl:
  case Builtin::BI__builtin_subcll:
    Result =
        LHS.usub_ov(RHS, FirstOverflowed).usub_ov(CarryIn, SecondOverflowed);
    break;
  }
  // It is possible for both overflows to happen but CGBuiltin uses an OR so
  // this is consistent.
  CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);

  QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
  PrimType CarryOutT = *S.getContext().classify(CarryOutType);
  assignIntegral(S, CarryOutPtr, CarryOutT, CarryOut);
  if (CarryOutPtr.canBeInitialized())
    CarryOutPtr.initialize();

  assert(S.getASTContext().hasSimilarType(Call->getType(),
                                          Call->getArg(0)->getType()));
  pushInteger(S, Result, Call->getType());
  return true;
}

static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
                                const InterpFrame *Frame, const CallExpr *Call,
                                unsigned BuiltinOp) {

  std::optional<APSInt> Fallback;
  if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2) {
    APSInt FallbackVal;
    if (!popToAPSInt(S, Call->getArg(1), FallbackVal))
      return false;
    Fallback = FallbackVal;
  }

  APSInt Val;
  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
    const Pointer &Arg = S.Stk.pop<Pointer>();
    Val = convertBoolVectorToInt(Arg);
  } else {
    if (!popToAPSInt(S, Call->getArg(0), Val))
      return false;
  }

  // When the argument is 0, the result of GCC builtins is undefined, whereas
  // for Microsoft intrinsics, the result is the bit-width of the argument.
  bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
                         BuiltinOp != Builtin::BI__lzcnt &&
                         BuiltinOp != Builtin::BI__lzcnt64;

  if (Val == 0) {
    if (Fallback) {
      pushInteger(S, *Fallback, Call->getType());
      return true;
    }

    if (ZeroIsUndefined)
      return false;
  }

  pushInteger(S, Val.countl_zero(), Call->getType());
  return true;
}

static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
                                const InterpFrame *Frame, const CallExpr *Call,
                                unsigned BuiltinID) {
  std::optional<APSInt> Fallback;
  if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2) {
    APSInt FallbackVal;
    if (!popToAPSInt(S, Call->getArg(1), FallbackVal))
      return false;
    Fallback = FallbackVal;
  }

  APSInt Val;
  if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
    const Pointer &Arg = S.Stk.pop<Pointer>();
    Val = convertBoolVectorToInt(Arg);
  } else {
    if (!popToAPSInt(S, Call->getArg(0), Val))
      return false;
  }

  if (Val == 0) {
    if (Fallback) {
      pushInteger(S, *Fallback, Call->getType());
      return true;
    }
    return false;
  }

  pushInteger(S, Val.countr_zero(), Call->getType());
  return true;
}

static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame,
                                  const CallExpr *Call) {
  APSInt Val;
  if (!popToAPSInt(S, Call->getArg(0), Val))
    return false;
  if (Val.getBitWidth() == 8 || Val.getBitWidth() == 1)
    pushInteger(S, Val, Call->getType());
  else
    pushInteger(S, Val.byteSwap(), Call->getType());
  return true;
}

/// bool __atomic_always_lock_free(size_t, void const volatile*)
/// bool __atomic_is_lock_free(size_t, void const volatile*)
static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
                                             const InterpFrame *Frame,
                                             const CallExpr *Call,
                                             unsigned BuiltinOp) {
  auto returnBool = [&S](bool Value) -> bool {
    S.Stk.push<Boolean>(Value);
    return true;
  };

  const Pointer &Ptr = S.Stk.pop<Pointer>();
  uint64_t SizeVal;
  if (!popToUInt64(S, Call->getArg(0), SizeVal))
    return false;

  // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
  // of two less than or equal to the maximum inline atomic width, we know it
  // is lock-free.  If the size isn't a power of two, or greater than the
  // maximum alignment where we promote atomics, we know it is not lock-free
  // (at least not in the sense of atomic_is_lock_free).  Otherwise,
  // the answer can only be determined at runtime; for example, 16-byte
  // atomics have lock-free implementations on some, but not all,
  // x86-64 processors.

  // Check power-of-two.
  CharUnits Size = CharUnits::fromQuantity(SizeVal);
  if (Size.isPowerOfTwo()) {
    // Check against inlining width.
    unsigned InlineWidthBits =
        S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
    if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {

      // OK, we will inline appropriately-aligned operations of this size,
      // and _Atomic(T) is appropriately-aligned.
      if (Size == CharUnits::One())
        return returnBool(true);

      // Same for null pointers.
      assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
      if (Ptr.isZero())
        return returnBool(true);

      if (Ptr.isIntegralPointer()) {
        uint64_t IntVal = Ptr.getIntegerRepresentation();
        if (APSInt(APInt(64, IntVal, false), true).isAligned(Size.getAsAlign()))
          return returnBool(true);
      }

      const Expr *PtrArg = Call->getArg(1);
      // Otherwise, check if the type's alignment against Size.
      if (const auto *ICE = dyn_cast<ImplicitCastExpr>(PtrArg)) {
        // Drop the potential implicit-cast to 'const volatile void*', getting
        // the underlying type.
        if (ICE->getCastKind() == CK_BitCast)
          PtrArg = ICE->getSubExpr();
      }

      if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
        QualType PointeeType = PtrTy->getPointeeType();
        if (!PointeeType->isIncompleteType() &&
            S.getASTContext().getTypeAlignInChars(PointeeType) >= Size) {
          // OK, we will inline operations on this object.
          return returnBool(true);
        }
      }
    }
  }

  if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
    return returnBool(false);

  return Invalid(S, OpPC);
}

/// bool __c11_atomic_is_lock_free(size_t)
static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
                                                    CodePtr OpPC,
                                                    const InterpFrame *Frame,
                                                    const CallExpr *Call) {
  uint64_t SizeVal;
  if (!popToUInt64(S, Call->getArg(0), SizeVal))
    return false;

  CharUnits Size = CharUnits::fromQuantity(SizeVal);
  if (Size.isPowerOfTwo()) {
    // Check against inlining width.
    unsigned InlineWidthBits =
        S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
    if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {
      S.Stk.push<Boolean>(true);
      return true;
    }
  }

  return false; // returnBool(false);
}

/// __builtin_complex(Float A, float B);
static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call) {
  const Floating &Arg2 = S.Stk.pop<Floating>();
  const Floating &Arg1 = S.Stk.pop<Floating>();
  Pointer &Result = S.Stk.peek<Pointer>();

  Result.elem<Floating>(0) = Arg1;
  Result.elem<Floating>(1) = Arg2;
  Result.initializeAllElements();

  return true;
}

/// __builtin_is_aligned()
/// __builtin_align_up()
/// __builtin_align_down()
/// The first parameter is either an integer or a pointer.
/// The second parameter is the requested alignment as an integer.
static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
                                               const InterpFrame *Frame,
                                               const CallExpr *Call,
                                               unsigned BuiltinOp) {
  APSInt Alignment;
  if (!popToAPSInt(S, Call->getArg(1), Alignment))
    return false;

  if (Alignment < 0 || !Alignment.isPowerOf2()) {
    S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;
    return false;
  }
  unsigned SrcWidth = S.getASTContext().getIntWidth(Call->getArg(0)->getType());
  APSInt MaxValue(APInt::getOneBitSet(SrcWidth, SrcWidth - 1));
  if (APSInt::compareValues(Alignment, MaxValue) > 0) {
    S.FFDiag(Call, diag::note_constexpr_alignment_too_big)
        << MaxValue << Call->getArg(0)->getType() << Alignment;
    return false;
  }

  // The first parameter is either an integer or a pointer.
  PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));

  if (isIntegerType(FirstArgT)) {
    APSInt Src;
    if (!popToAPSInt(S.Stk, FirstArgT, Src))
      return false;
    APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;
    if (BuiltinOp == Builtin::BI__builtin_align_up) {
      APSInt AlignedVal =
          APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
      pushInteger(S, AlignedVal, Call->getType());
    } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
      APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
      pushInteger(S, AlignedVal, Call->getType());
    } else {
      assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
      S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);
    }
    return true;
  }
  assert(FirstArgT == PT_Ptr);
  const Pointer &Ptr = S.Stk.pop<Pointer>();
  if (!Ptr.isBlockPointer() && !Ptr.isOpaquePointer()) {
    S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
        << Alignment;
    return false;
  }

  const VarDecl *PtrDecl = Ptr.getRootVarDecl();
  // We need a pointer for a declaration here.
  if (!PtrDecl) {
    if (BuiltinOp == Builtin::BI__builtin_is_aligned)
      S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
          << Alignment;
    else
      S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust)
          << Alignment;
    return false;
  }

  unsigned PtrOffset;
  if (Ptr.isBlockPointer()) {
    // For one-past-end pointers, we can't call getIndex() since it asserts.
    // Use getNumElems() instead which gives the correct index for past-end.
    PtrOffset = Ptr.isElementPastEnd() ? Ptr.getNumElems() : Ptr.getIndex();
  } else {
    if (std::optional<size_t> PtrOff =
            Ptr.computeLayoutOffset(S.getASTContext()))
      PtrOffset = *PtrOff;
    else
      return false;
  }

  CharUnits BaseAlignment = S.getASTContext().getDeclAlign(PtrDecl);
  CharUnits PtrAlign =
      BaseAlignment.alignmentAtOffset(CharUnits::fromQuantity(PtrOffset));

  if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
    if (PtrAlign.getQuantity() >= Alignment) {
      S.Stk.push<Boolean>(true);
      return true;
    }
    // If the alignment is not known to be sufficient, some cases could still
    // be aligned at run time. However, if the requested alignment is less or
    // equal to the base alignment and the offset is not aligned, we know that
    // the run-time value can never be aligned.
    if (BaseAlignment.getQuantity() >= Alignment &&
        PtrAlign.getQuantity() < Alignment) {
      S.Stk.push<Boolean>(false);
      return true;
    }

    S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
        << Alignment;
    return false;
  }

  assert(BuiltinOp == Builtin::BI__builtin_align_down ||
         BuiltinOp == Builtin::BI__builtin_align_up);

  // For align_up/align_down, we can return the same value if the alignment
  // is known to be greater or equal to the requested value.
  if (PtrAlign.getQuantity() >= Alignment) {
    S.Stk.push<Pointer>(Ptr);
    return true;
  }

  // The alignment could be greater than the minimum at run-time, so we cannot
  // infer much about the resulting pointer value. One case is possible:
  // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
  // can infer the correct index if the requested alignment is smaller than
  // the base alignment so we can perform the computation on the offset.
  if (BaseAlignment.getQuantity() >= Alignment) {
    assert(Alignment.getBitWidth() <= 64 &&
           "Cannot handle > 64-bit address-space");
    uint64_t Alignment64 = Alignment.getZExtValue();
    CharUnits NewOffset =
        CharUnits::fromQuantity(BuiltinOp == Builtin::BI__builtin_align_down
                                    ? llvm::alignDown(PtrOffset, Alignment64)
                                    : llvm::alignTo(PtrOffset, Alignment64));

    if (Ptr.isBlockPointer()) {
      S.Stk.push<Pointer>(Ptr.atIndex(NewOffset.getQuantity()));
      return true;
    }

    assert(Ptr.isOpaquePointer());

    APSInt APOffset =
        APSInt(APInt(64, NewOffset.getQuantity(), /*IsSigned=*/true),
               /*IsUnsigned=*/false);
    return arrayElemPtrOpaque(S, OpPC, Ptr, std::move(APOffset),
                              /*AllocReplace=*/true);
  }

  // Otherwise, we cannot constant-evaluate the result.
  S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust) << Alignment;
  return false;
}

/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
                                           const InterpFrame *Frame,
                                           const CallExpr *Call) {
  assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);

  std::optional<APSInt> ExtraOffset;
  if (Call->getNumArgs() == 3) {
    APSInt ExtraOffsetVal;
    if (!popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)), ExtraOffsetVal))
      return false;
    ExtraOffset = ExtraOffsetVal;
  }

  APSInt Alignment;
  if (!popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)), Alignment))
    return false;
  const Pointer &Ptr = S.Stk.pop<Pointer>();

  const ASTContext &ASTCtx = S.getASTContext();
  CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());

  // If there is a base object, then it must have the correct alignment.
  if (Ptr.isBlockPointer() || Ptr.isOpaquePointer()) {
    CharUnits BaseAlignment;
    if (const auto *VD = Ptr.getRootVarDecl())
      BaseAlignment = ASTCtx.getDeclAlign(VD);
    else if (const auto *E = Ptr.getRootExpr())
      BaseAlignment = GetAlignOfExpr(ASTCtx, E, UETT_AlignOf);

    if (BaseAlignment < Align) {
      S.CCEDiag(Call->getArg(0),
                diag::note_constexpr_baa_insufficient_alignment)
          << 0 << BaseAlignment.getQuantity() << Align.getQuantity();
      return false;
    }
  }

  std::optional<size_t> LayoutOffset = Ptr.computeLayoutOffset(ASTCtx);
  if (!LayoutOffset)
    return false;

  CharUnits AVOffset = CharUnits::fromQuantity(*LayoutOffset);
  if (ExtraOffset)
    AVOffset -= CharUnits::fromQuantity(ExtraOffset->getZExtValue());
  if (AVOffset.alignTo(Align) != AVOffset) {
    if (Ptr.isBlockPointer() || Ptr.isOpaquePointer())
      S.CCEDiag(Call->getArg(0),
                diag::note_constexpr_baa_insufficient_alignment)
          << 1 << AVOffset.getQuantity() << Align.getQuantity();
    else
      S.CCEDiag(Call->getArg(0),
                diag::note_constexpr_baa_value_insufficient_alignment)
          << AVOffset.getQuantity() << Align.getQuantity();
    return false;
  }

  S.Stk.push<Pointer>(Ptr);
  return true;
}

/// (CarryIn, LHS, RHS, Result)
static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
                                                    CodePtr OpPC,
                                                    const InterpFrame *Frame,
                                                    const CallExpr *Call,
                                                    bool IsAdd) {
  if (Call->getNumArgs() != 4 || !Call->getArg(0)->getType()->isIntegerType() ||
      !Call->getArg(1)->getType()->isIntegerType() ||
      !Call->getArg(2)->getType()->isIntegerType())
    return false;

  const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();

  APSInt RHS;
  if (!popToAPSInt(S, Call->getArg(2), RHS))
    return false;
  APSInt LHS;
  if (!popToAPSInt(S, Call->getArg(1), LHS))
    return false;
  APSInt CarryIn;
  if (!popToAPSInt(S, Call->getArg(0), CarryIn))
    return false;

  unsigned BitWidth = LHS.getBitWidth();
  unsigned CarryInBit = CarryIn.ugt(0) ? 1 : 0;
  APInt ExResult =
      IsAdd ? (LHS.zext(BitWidth + 1) + (RHS.zext(BitWidth + 1) + CarryInBit))
            : (LHS.zext(BitWidth + 1) - (RHS.zext(BitWidth + 1) + CarryInBit));

  APInt Result = ExResult.extractBits(BitWidth, 0);
  APSInt CarryOut =
      APSInt(ExResult.extractBits(1, BitWidth), /*IsUnsigned=*/true);

  QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
  PrimType CarryOutT = *S.getContext().classify(CarryOutType);
  assignIntegral(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));

  pushInteger(S, CarryOut, Call->getType());

  return true;
}

static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
                                                      CodePtr OpPC,
                                                      const InterpFrame *Frame,
                                                      const CallExpr *Call) {
  analyze_os_log::OSLogBufferLayout Layout;
  analyze_os_log::computeOSLogBufferLayout(S.getASTContext(), Call, Layout);
  pushInteger(S, Layout.size().getQuantity(), Call->getType());
  return true;
}

static bool
interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
                                             const InterpFrame *Frame,
                                             const CallExpr *Call) {
  const auto &Ptr = S.Stk.pop<Pointer>();
  if (!Ptr.isStringPointer())
    return false;

  uint64_t Result = getPointerAuthStableSipHash(
      cast<StringLiteral>(Ptr.getRootExpr())->getString());
  pushInteger(S, Result, Call->getType());
  return true;
}

static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
                                              const InterpFrame *Frame,
                                              const CallExpr *Call) {
  const ASTContext &ASTCtx = S.getASTContext();
  uint64_t BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
  auto Mode =
      ASTCtx.getLangOpts().AllocTokenMode.value_or(llvm::DefaultAllocTokenMode);
  auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
  uint64_t MaxTokens =
      MaxTokensOpt.value_or(0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));

  // We do not read any of the arguments; discard them.
  for (int I = Call->getNumArgs() - 1; I >= 0; --I)
    discard(S.Stk, S.getContext().classify(Call->getArg(I)).value_or(PT_Ptr));

  // Note: Type inference from a surrounding cast is not supported in
  // constexpr evaluation.
  QualType AllocType = infer_alloc::inferPossibleType(Call, ASTCtx, nullptr);
  if (AllocType.isNull()) {
    S.CCEDiag(Call,
              diag::note_constexpr_infer_alloc_token_type_inference_failed);
    return false;
  }

  auto ATMD = infer_alloc::getAllocTokenMetadata(AllocType, ASTCtx);
  if (!ATMD) {
    S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_no_metadata);
    return false;
  }

  auto MaybeToken = llvm::getAllocToken(Mode, *ATMD, MaxTokens);
  if (!MaybeToken) {
    S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_stateful_mode);
    return false;
  }

  pushInteger(S, llvm::APInt(BitWidth, *MaybeToken), ASTCtx.getSizeType());
  return true;
}

static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const CallExpr *Call) {
  // A call to __operator_new is only valid within std::allocate<>::allocate.
  // Walk up the call stack to find the appropriate caller and get the
  // element type from it.
  auto [NewCall, ElemType] = S.getStdAllocatorCaller("allocate");

  if (ElemType.isNull()) {
    S.FFDiag(Call, S.getLangOpts().CPlusPlus20
                       ? diag::note_constexpr_new_untyped
                       : diag::note_constexpr_new);
    return false;
  }
  assert(NewCall);

  if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
    S.FFDiag(Call, diag::note_constexpr_new_not_complete_object_type)
        << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
    return false;
  }

  // We only care about the first parameter (the size), so discard all the
  // others.
  {
    unsigned NumArgs = Call->getNumArgs();
    assert(NumArgs >= 1);

    // The std::nothrow_t arg never gets put on the stack.
    if (Call->getArg(NumArgs - 1)->getType()->isNothrowT())
      --NumArgs;
    auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
    // First arg is needed.
    Args = Args.drop_front();

    // Discard the rest.
    for (const Expr *Arg : Args)
      discard(S.Stk, *S.getContext().classify(Arg));
  }

  APSInt Bytes;
  if (!popToAPSInt(S, Call->getArg(0), Bytes))
    return false;
  CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(ElemType);
  assert(!ElemSize.isZero());
  // Divide the number of bytes by sizeof(ElemType), so we get the number of
  // elements we should allocate.
  APInt NumElems, Remainder;
  APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
  APInt::udivrem(Bytes, ElemSizeAP, NumElems, Remainder);
  if (Remainder != 0) {
    // This likely indicates a bug in the implementation of 'std::allocator'.
    S.FFDiag(Call, diag::note_constexpr_operator_new_bad_size)
        << Bytes << APSInt(ElemSizeAP, true) << ElemType;
    return false;
  }

  // NB: The same check we're using in CheckArraySize()
  if (NumElems.getActiveBits() >
          ConstantArrayType::getMaxSizeBits(S.getASTContext()) ||
      NumElems.ugt(Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
    // FIXME: NoThrow check?
    const SourceInfo &Loc = S.Current->getSource(OpPC);
    S.FFDiag(Loc, diag::note_constexpr_new_too_large)
        << NumElems.getZExtValue();
    return false;
  }

  if (!CheckArraySize(S, OpPC, NumElems.getZExtValue()))
    return false;

  bool IsArray = NumElems.ugt(1);
  OptPrimType ElemT = S.getContext().classify(ElemType);
  DynamicAllocator &Allocator = S.getAllocator();
  if (ElemT) {
    Block *B =
        Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
                           S.Ctx.getEvalID(), DynamicAllocator::Form::Operator);
    assert(B);
    S.Stk.push<Pointer>(Pointer(B).atIndex(0));
    return true;
  }

  assert(!ElemT);

  // Composite arrays
  if (IsArray) {
    const Descriptor *Desc =
        S.P.createDescriptor(NewCall, ElemType.getTypePtr());
    Block *B =
        Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
                           DynamicAllocator::Form::Operator);
    assert(B);
    S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
    return true;
  }

  // Records. Still allocate them as single-element arrays.
  QualType AllocType = S.getASTContext().getConstantArrayType(
      ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);

  const Descriptor *Desc =
      S.P.createDescriptor(NewCall, AllocType.getTypePtr());
  Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
                                DynamicAllocator::Form::Operator);
  assert(B);
  S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
  return true;
}

static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
                                            const InterpFrame *Frame,
                                            const CallExpr *Call) {
  const Expr *Source = nullptr;
  const Block *BlockToDelete = nullptr;

  unsigned NumArgs = Call->getNumArgs();
  assert(NumArgs >= 1);

  // Args are pushed in source order. The trailing sized/aligned delete
  // operands are above the pointer on the stack.
  for (unsigned I = NumArgs - 1; I != 0; --I)
    discard(S.Stk, *S.getContext().classify(Call->getArg(I)));

  if (S.checkingPotentialConstantExpression()) {
    S.Stk.discard<Pointer>();
    return false;
  }

  // This is permitted only within a call to std::allocator<T>::deallocate.
  if (!S.getStdAllocatorCaller("deallocate")) {
    S.FFDiag(Call);
    S.Stk.discard<Pointer>();
    return true;
  }

  {
    const Pointer &Ptr = S.Stk.pop<Pointer>();

    if (Ptr.isZero()) {
      S.CCEDiag(Call, diag::note_constexpr_deallocate_null);
      return true;
    }

    Source = Ptr.getRootExpr();
    BlockToDelete = Ptr.block();

    if (!BlockToDelete->isDynamic()) {
      S.FFDiag(Call, diag::note_constexpr_delete_not_heap_alloc)
          << Ptr.toDiagnosticString(S.getASTContext());
      if (const auto *D = Ptr.getFieldDesc()->asDecl())
        S.Note(D->getLocation(), diag::note_declared_at);
    }
  }
  assert(BlockToDelete);

  DynamicAllocator &Allocator = S.getAllocator();
  const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
  std::optional<DynamicAllocator::Form> AllocForm =
      Allocator.getAllocationForm(Source);

  if (!Allocator.deallocate(Source, BlockToDelete)) {
    // Nothing has been deallocated, this must be a double-delete.
    const SourceInfo &Loc = S.Current->getSource(OpPC);
    S.FFDiag(Loc, diag::note_constexpr_double_delete);
    return false;
  }
  assert(AllocForm);

  return CheckNewDeleteForms(
      S, OpPC, *AllocForm, DynamicAllocator::Form::Operator, BlockDesc, Source);
}

static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
                                             const InterpFrame *Frame,
                                             const CallExpr *Call) {
  const Floating &Arg0 = S.Stk.pop<Floating>();
  S.Stk.push<Floating>(Arg0);
  return true;
}

static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call, unsigned ID) {
  const Pointer &Arg = S.Stk.pop<Pointer>();
  assert(Arg.getFieldDesc()->isPrimitiveArray());

  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
  assert(Call->getType() == ElemType);
  PrimType ElemT = *S.getContext().classify(ElemType);
  unsigned NumElems = Arg.getNumElems();

  if (!isIntegerType(ElemT))
    return false;

  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
    T Result = Arg.elem<T>(0);
    unsigned BitWidth = Result.bitWidth();
    for (unsigned I = 1; I != NumElems; ++I) {
      T Elem = Arg.elem<T>(I);
      T PrevResult = Result;

      if (ID == Builtin::BI__builtin_reduce_add) {
        if (T::add(Result, Elem, BitWidth, &Result)) {
          unsigned OverflowBits = BitWidth + 1;
          (void)handleOverflow(S, OpPC,
                               (PrevResult.toAPSInt(OverflowBits) +
                                Elem.toAPSInt(OverflowBits)));
          return false;
        }
      } else if (ID == Builtin::BI__builtin_reduce_mul) {
        if (T::mul(Result, Elem, BitWidth, &Result)) {
          unsigned OverflowBits = BitWidth * 2;
          (void)handleOverflow(S, OpPC,
                               (PrevResult.toAPSInt(OverflowBits) *
                                Elem.toAPSInt(OverflowBits)));
          return false;
        }

      } else if (ID == Builtin::BI__builtin_reduce_and) {
        (void)T::bitAnd(Result, Elem, BitWidth, &Result);
      } else if (ID == Builtin::BI__builtin_reduce_or) {
        (void)T::bitOr(Result, Elem, BitWidth, &Result);
      } else if (ID == Builtin::BI__builtin_reduce_xor) {
        (void)T::bitXor(Result, Elem, BitWidth, &Result);
      } else if (ID == Builtin::BI__builtin_reduce_min) {
        if (Elem < Result)
          Result = Elem;
      } else if (ID == Builtin::BI__builtin_reduce_max) {
        if (Elem > Result)
          Result = Elem;
      } else {
        llvm_unreachable("Unhandled vector reduce builtin");
      }
    }
    pushInteger(S, Result.toAPSInt(), Call->getType());
  });

  return true;
}

static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
                                            const InterpFrame *Frame,
                                            const CallExpr *Call,
                                            unsigned BuiltinID) {
  assert(Call->getNumArgs() == 1);
  QualType Ty = Call->getArg(0)->getType();
  if (Ty->isIntegerType()) {
    APSInt Val;
    if (!popToAPSInt(S, Call->getArg(0), Val))
      return false;
    pushInteger(S, Val.abs(), Call->getType());
    return true;
  }

  if (Ty->isFloatingType()) {
    Floating Val = S.Stk.pop<Floating>();
    Floating Result = abs(S, Val);
    S.Stk.push<Floating>(Result);
    return true;
  }

  // Otherwise, the argument must be a vector.
  assert(Call->getArg(0)->getType()->isVectorType());
  const Pointer &Arg = S.Stk.pop<Pointer>();
  assert(Arg.getFieldDesc()->isPrimitiveArray());
  const Pointer &Dst = S.Stk.peek<Pointer>();
  assert(Dst.getFieldDesc()->isPrimitiveArray());
  assert(Arg.getFieldDesc()->getNumElems() ==
         Dst.getFieldDesc()->getNumElems());

  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
  PrimType ElemT = *S.getContext().classify(ElemType);
  unsigned NumElems = Arg.getNumElems();
  // we can either have a vector of integer or a vector of floating point
  for (unsigned I = 0; I != NumElems; ++I) {
    if (ElemType->isIntegerType()) {
      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
        Dst.elem<T>(I) = T::from(static_cast<T>(
            APSInt(Arg.elem<T>(I).toAPSInt().abs(),
                   ElemType->isUnsignedIntegerOrEnumerationType())));
      });
    } else {
      Floating Val = Arg.elem<Floating>(I);
      Dst.elem<Floating>(I) = abs(S, Val);
    }
  }
  Dst.initializeAllElements();

  return true;
}

/// Can be called with an integer or vector as the first and only parameter.
static bool interp__builtin_elementwise_countzeroes(InterpState &S,
                                                    CodePtr OpPC,
                                                    const InterpFrame *Frame,
                                                    const CallExpr *Call,
                                                    unsigned BuiltinID) {
  bool HasZeroArg = Call->getNumArgs() == 2;
  bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
  assert(Call->getNumArgs() == 1 || HasZeroArg);
  if (Call->getArg(0)->getType()->isIntegerType()) {
    PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
    APSInt Val;
    if (!popToAPSInt(S.Stk, ArgT, Val))
      return false;
    std::optional<APSInt> ZeroVal;
    if (HasZeroArg) {
      ZeroVal = Val;
      if (!popToAPSInt(S.Stk, ArgT, Val))
        return false;
    }

    if (Val.isZero()) {
      if (ZeroVal) {
        pushInteger(S, *ZeroVal, Call->getType());
        return true;
      }
      // If we haven't been provided the second argument, the result is
      // undefined
      S.FFDiag(S.Current->getSource(OpPC),
               diag::note_constexpr_countzeroes_zero)
          << /*IsTrailing=*/IsCTTZ;
      return false;
    }

    if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
      pushInteger(S, Val.countLeadingZeros(), Call->getType());
    } else {
      pushInteger(S, Val.countTrailingZeros(), Call->getType());
    }
    return true;
  }
  // Otherwise, the argument must be a vector.
  const ASTContext &ASTCtx = S.getASTContext();
  Pointer ZeroArg;
  if (HasZeroArg) {
    assert(Call->getArg(1)->getType()->isVectorType() &&
           ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
                                         Call->getArg(1)->getType()));
    (void)ASTCtx;
    ZeroArg = S.Stk.pop<Pointer>();
    assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
  }
  assert(Call->getArg(0)->getType()->isVectorType());
  const Pointer &Arg = S.Stk.pop<Pointer>();
  assert(Arg.getFieldDesc()->isPrimitiveArray());
  const Pointer &Dst = S.Stk.peek<Pointer>();
  assert(Dst.getFieldDesc()->isPrimitiveArray());
  assert(Arg.getFieldDesc()->getNumElems() ==
         Dst.getFieldDesc()->getNumElems());

  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
  PrimType ElemT = *S.getContext().classify(ElemType);
  unsigned NumElems = Arg.getNumElems();

  // FIXME: Reading from uninitialized vector elements?
  for (unsigned I = 0; I != NumElems; ++I) {
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
      if (EltVal.isZero()) {
        if (HasZeroArg) {
          Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
        } else {
          // If we haven't been provided the second argument, the result is
          // undefined
          S.FFDiag(S.Current->getSource(OpPC),
                   diag::note_constexpr_countzeroes_zero)
              << /*IsTrailing=*/IsCTTZ;
          return false;
        }
      } else if (IsCTTZ) {
        Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
      } else {
        Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
      }
      Dst.atIndex(I).initialize();
    });
  }

  return true;
}

static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call, unsigned ID) {
  assert(Call->getNumArgs() == 3);
  const ASTContext &ASTCtx = S.getASTContext();
  uint64_t Size;
  if (!popToUInt64(S, Call->getArg(2), Size))
    return false;
  Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
  Pointer DestPtr = S.Stk.pop<Pointer>().expand();

  if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
    diagnoseNonConstexprBuiltin(S, OpPC, ID);

  bool Move =
      (ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
       ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
  bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
               ID == Builtin::BI__builtin_wmemcpy ||
               ID == Builtin::BI__builtin_wmemmove;

  // If the size is zero, we treat this as always being a valid no-op.
  if (Size == 0) {
    S.Stk.push<Pointer>(DestPtr);
    return true;
  }

  if (SrcPtr.isZero() || DestPtr.isZero()) {
    Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
        << /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
        << DiagPtr.toDiagnosticString(ASTCtx);
    return false;
  }

  // Diagnose integral src/dest pointers specially.
  if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
    std::string DiagVal = "(void *)";
    DiagVal += SrcPtr.isIntegralPointer()
                   ? std::to_string(SrcPtr.getIntegerRepresentation())
                   : std::to_string(DestPtr.getIntegerRepresentation());
    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
        << Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
    return false;
  }

  if (!isReadable(DestPtr) || !isReadable(SrcPtr))
    return false;

  if (DestPtr.getType()->isIncompleteType()) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcpy_incomplete_type)
        << Move << DestPtr.getType();
    return false;
  }
  if (SrcPtr.getType()->isIncompleteType()) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcpy_incomplete_type)
        << Move << SrcPtr.getType();
    return false;
  }

  QualType DestElemType = getElemType(DestPtr);
  if (DestElemType->isIncompleteType()) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcpy_incomplete_type)
        << Move << DestElemType;
    return false;
  }

  size_t RemainingDestElems;
  if (DestPtr.inArray()) {
    RemainingDestElems = DestPtr.isUnknownSizeArray()
                             ? 0
                             : (DestPtr.getNumElems() - DestPtr.getIndex());
  } else {
    RemainingDestElems = 1;
  }
  unsigned DestElemSize = ASTCtx.getTypeSizeInChars(DestElemType).getQuantity();

  if (WChar) {
    uint64_t WCharSize =
        ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
    Size *= WCharSize;
  }

  if (Size % DestElemSize != 0) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcpy_unsupported)
        << Move << WChar << 0 << DestElemType << Size << DestElemSize;
    return false;
  }

  QualType SrcElemType = getElemType(SrcPtr);
  size_t RemainingSrcElems;
  if (SrcPtr.inArray()) {
    RemainingSrcElems = SrcPtr.isUnknownSizeArray()
                            ? 0
                            : (SrcPtr.getNumElems() - SrcPtr.getIndex());
  } else {
    RemainingSrcElems = 1;
  }
  unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(SrcElemType).getQuantity();

  if (!ASTCtx.hasSameUnqualifiedType(DestElemType, SrcElemType)) {
    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_type_pun)
        << Move << SrcElemType << DestElemType;
    return false;
  }

  if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {
    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)
        << Move << DestElemType;
    return false;
  }

  // Check if we have enough elements to read from and write to.
  size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
  size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
  if (Size > RemainingDestBytes || Size > RemainingSrcBytes) {
    APInt N = APInt(64, Size / DestElemSize);
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcpy_unsupported)
        << Move << WChar << (Size > RemainingSrcBytes ? 1 : 2) << DestElemType
        << toString(N, 10, /*Signed=*/false);
    return false;
  }

  // Check for overlapping memory regions.
  if (!Move && Pointer::pointToSameBlock(SrcPtr, DestPtr)) {
    // Remove base casts.
    Pointer SrcP = SrcPtr.stripBaseCasts();
    Pointer DestP = DestPtr.stripBaseCasts();

    unsigned SrcIndex = SrcP.expand().getIndex() * SrcElemSize;
    unsigned DstIndex = DestP.expand().getIndex() * DestElemSize;

    if ((SrcIndex <= DstIndex && (SrcIndex + Size) > DstIndex) ||
        (DstIndex <= SrcIndex && (DstIndex + Size) > SrcIndex)) {
      S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_overlap)
          << /*IsWChar=*/false;
      return false;
    }
  }

  assert(Size % DestElemSize == 0);
  if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Bytes(Size).toBits()))
    return false;

  S.Stk.push<Pointer>(DestPtr);
  return true;
}

/// Determine if T is a character type for which we guarantee that
/// sizeof(T) == 1.
static bool isOneByteCharacterType(QualType T) {
  return T->isCharType() || T->isChar8Type();
}

static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame,
                                   const CallExpr *Call, unsigned ID) {
  assert(Call->getNumArgs() == 3);
  uint64_t Size;
  if (!popToUInt64(S, Call->getArg(2), Size))
    return false;
  const Pointer &PtrB = S.Stk.pop<Pointer>();
  const Pointer &PtrA = S.Stk.pop<Pointer>();

  if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
      ID == Builtin::BIwmemcmp)
    diagnoseNonConstexprBuiltin(S, OpPC, ID);

  if (Size == 0) {
    pushInteger(S, 0, Call->getType());
    return true;
  }
  bool IsWide =
      (ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);

  const ASTContext &ASTCtx = S.getASTContext();
  QualType ElemTypeA = getElemType(PtrA);
  QualType ElemTypeB = getElemType(PtrB);
  // FIXME: This is an arbitrary limitation the current constant interpreter
  // had. We could remove this.
  if (!IsWide && (!isOneByteCharacterType(ElemTypeA) ||
                  !isOneByteCharacterType(ElemTypeB))) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memcmp_unsupported)
        << ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
        << PtrB.getType();
    return false;
  }

  if (!PtrA.isReadablePointerType() || !PtrB.isReadablePointerType())
    return false;

  if (!CheckLoad(S, OpPC, PtrA, AK_Read) || !CheckLoad(S, OpPC, PtrB, AK_Read))
    return false;

  // Now, read both pointers to a buffer and compare those.
  BitcastBuffer BufferA(
      Bits(ASTCtx.getTypeSize(ElemTypeA) * PtrA.getNumElems()));
  readPointerToBuffer(S.getContext(), PtrA, BufferA, /*ReturnOnUninit=*/false);

  // FIXME: The swapping here is UNDOING something we do when reading the
  // data into the buffer.
  if (ASTCtx.getTargetInfo().isBigEndian())
    swapBytes(BufferA.Data.get(), BufferA.byteSize().getQuantity());

  BitcastBuffer BufferB(
      Bits(ASTCtx.getTypeSize(ElemTypeB) * PtrB.getNumElems()));
  readPointerToBuffer(S.getContext(), PtrB, BufferB, /*ReturnOnUninit=*/false);
  // FIXME: The swapping here is UNDOING something we do when reading the
  // data into the buffer.
  if (ASTCtx.getTargetInfo().isBigEndian())
    swapBytes(BufferB.Data.get(), BufferB.byteSize().getQuantity());

  size_t MinBufferSize = std::min(BufferA.byteSize().getQuantity(),
                                  BufferB.byteSize().getQuantity());

  unsigned ElemSize = 1;
  if (IsWide)
    ElemSize = ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
  // The Size given for the wide variants is in wide-char units. Convert it
  // to bytes.
  size_t ByteSize = Size * ElemSize;
  size_t CmpSize = std::min(MinBufferSize, ByteSize);

  for (size_t I = 0; I != CmpSize; I += ElemSize) {
    if (IsWide) {
      FIXED_SIZE_INT_TYPE_SWITCH(
          *S.getContext().classify(ASTCtx.getWCharType()), {
            T A = T::bitcastFromMemory(BufferA.atByte(I), T::bitWidth());
            T B = T::bitcastFromMemory(BufferB.atByte(I), T::bitWidth());
            if (A < B) {
              pushInteger(S, -1, Call->getType());
              return true;
            }
            if (A > B) {
              pushInteger(S, 1, Call->getType());
              return true;
            }
          });
    } else {
      auto A = BufferA.deref<std::byte>(Bytes(I));
      auto B = BufferB.deref<std::byte>(Bytes(I));

      if (A < B) {
        pushInteger(S, -1, Call->getType());
        return true;
      }
      if (A > B) {
        pushInteger(S, 1, Call->getType());
        return true;
      }
    }
  }

  // We compared CmpSize bytes above. If the limiting factor was the Size
  // passed, we're done and the result is equality (0).
  if (ByteSize <= CmpSize) {
    pushInteger(S, 0, Call->getType());
    return true;
  }

  // However, if we read all the available bytes but were instructed to read
  // even more, diagnose this as a "read of dereferenced one-past-the-end
  // pointer". This is what would happen if we called CheckLoad() on every array
  // element.
  S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)
      << AK_Read << S.Current->getRange(OpPC);
  return false;
}

// __builtin_memchr(ptr, int, int)
// __builtin_strchr(ptr, int)
static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
                                   const CallExpr *Call, unsigned ID) {
  if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
      ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
    diagnoseNonConstexprBuiltin(S, OpPC, ID);

  std::optional<APSInt> MaxLength;
  if (Call->getNumArgs() == 3) {
    APSInt MaxLengthVal;
    if (!popToAPSInt(S, Call->getArg(2), MaxLengthVal))
      return false;
    MaxLength = MaxLengthVal;
  }

  APSInt Desired;
  if (!popToAPSInt(S, Call->getArg(1), Desired))
    return false;
  const Pointer &Ptr = S.Stk.pop<Pointer>();

  if (MaxLength && MaxLength->isZero()) {
    S.Stk.push<Pointer>();
    return true;
  }

  if (Ptr.isDummy()) {
    if (Ptr.getType()->isIncompleteType())
      S.FFDiag(S.Current->getSource(OpPC),
               diag::note_constexpr_ltor_incomplete_type)
          << Ptr.getType();
    return false;
  }

  // Null is only okay if the given size is 0.
  if (Ptr.isZero()) {
    S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_null)
        << AK_Read;
    return false;
  }

  if (!Ptr.isReadablePointerType())
    return false;

  QualType ElemTy = getElemType(Ptr);
  bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;

  // Give up on byte-oriented matching against multibyte elements.
  if (IsRawByte && !isOneByteCharacterType(ElemTy)) {
    S.FFDiag(S.Current->getSource(OpPC),
             diag::note_constexpr_memchr_unsupported)
        << S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
    return false;
  }

  if (!isReadable(Ptr))
    return false;

  if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
    int64_t DesiredTrunc;
    if (S.getASTContext().CharTy->isSignedIntegerType())
      DesiredTrunc =
          Desired.trunc(S.getASTContext().getCharWidth()).getSExtValue();
    else
      DesiredTrunc =
          Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
    // strchr compares directly to the passed integer, and therefore
    // always fails if given an int that is not a char.
    if (Desired != DesiredTrunc) {
      S.Stk.push<Pointer>();
      return true;
    }
  }

  uint64_t DesiredVal;
  if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
      ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
    // wcschr and wmemchr are given a wchar_t to look for. Just use it.
    DesiredVal = Desired.getZExtValue();
  } else {
    DesiredVal = Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
  }

  bool StopAtZero =
      (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
       ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);

  PrimType ElemT =
      IsRawByte ? PT_Sint8 : *S.getContext().classify(getElemType(Ptr));

  size_t Index = Ptr.getIndex();
  size_t Step = 0;
  for (;;) {
    const Pointer &ElemPtr =
        (Index + Step) > 0 ? Ptr.atIndex(Index + Step) : Ptr;

    if (!CheckLoad(S, OpPC, ElemPtr))
      return false;

    uint64_t V;
    INT_TYPE_SWITCH_NO_BOOL(
        ElemT, { V = static_cast<uint64_t>(ElemPtr.load<T>().toUnsigned()); });

    if (V == DesiredVal) {
      S.Stk.push<Pointer>(ElemPtr);
      return true;
    }

    if (StopAtZero && V == 0)
      break;

    ++Step;
    if (MaxLength && Step == MaxLength->getZExtValue())
      break;
  }

  S.Stk.push<Pointer>();
  return true;
}

static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
                                        const InterpFrame *Frame,
                                        const CallExpr *Call, bool IsDynamic) {
  const ASTContext &ASTCtx = S.getASTContext();
  // From the GCC docs:
  // Kind is an integer constant from 0 to 3. If the least significant bit is
  // clear, objects are whole variables. If it is set, a closest surrounding
  // subobject is considered the object a pointer points to. The second bit
  // determines if maximum or minimum of remaining bytes is computed.
  uint64_t Kind;
  if (!popToUInt64(S, Call->getArg(1), Kind))
    return false;
  assert(Kind <= 3 && "unexpected kind");
  Pointer Ptr = S.Stk.pop<Pointer>();

  if (auto Result = evaluateBuiltinObjectSize(ASTCtx, Kind, Ptr,
                                              Call->getArg(0), IsDynamic)) {
    pushInteger(S, *Result, Call->getType());
    return true;
  }

  if (Call->getArg(0)->HasSideEffects(ASTCtx)) {
    // "If there are any side effects in them, it returns (size_t) -1
    // for type 0 or 1 and (size_t) 0 for type 2 or 3."
    pushInteger(S, Kind <= 1 ? (size_t)-1 : (size_t)0, Call->getType());
    return true;
  }

  switch (S.EvalMode) {
  case EvaluationMode::ConstantExpression:
  case EvaluationMode::ConstantFold:
  case EvaluationMode::IgnoreSideEffects:
    // Leave it to IR generation.
    return Invalid(S, OpPC);
  case EvaluationMode::ConstantExpressionUnevaluated:
    // Reduce it to a constant now.
    pushInteger(S, ((Kind & 2u) ? (size_t)0 : (size_t)-1), Call->getType());
    return true;
  }

  return false;
}

static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
                                               const CallExpr *Call) {

  if (!S.inConstantContext())
    return false;

  const Pointer &Ptr = S.Stk.pop<Pointer>();

  auto Error = [&](int Diag) {
    bool CalledFromStd = false;
    const auto *Callee = S.Current->getCallee();
    if (Callee && Callee->isInStdNamespace()) {
      const IdentifierInfo *Identifier = Callee->getIdentifier();
      CalledFromStd = Identifier && Identifier->isStr("is_within_lifetime");
    }
    S.CCEDiag(CalledFromStd
                  ? S.Current->Caller->getSource(S.Current->getRetPC())
                  : S.Current->getSource(OpPC),
              diag::err_invalid_is_within_lifetime)
        << (CalledFromStd ? "std::is_within_lifetime"
                          : "__builtin_is_within_lifetime")
        << Diag;
    return false;
  };

  if (Ptr.isZero())
    return Error(0);
  if (Ptr.isOnePastEnd())
    return Error(1);

  bool Result = Ptr.getLifetime() != Lifetime::Ended;
  if (!Ptr.isActive()) {
    Result = false;
  } else {
    if (!CheckLive(S, OpPC, Ptr, AK_Read))
      return false;
    if (!CheckMutable(S, OpPC, Ptr))
      return false;
    if (!CheckDummy(S, OpPC, Ptr, AK_Read))
      return false;
  }

  // Check if we're currently running an initializer.
  if (S.initializingBlock(Ptr.block()))
    return Error(2);
  if (S.EvaluatingDecl && Ptr.getRootVarDecl() == S.EvaluatingDecl)
    return Error(2);

  pushInteger(S, Result, Call->getType());
  return true;
}

static bool interp__builtin_elementwise_int_unaryop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APSInt &)> Fn) {
  assert(Call->getNumArgs() == 1);

  // Single integer case.
  if (!Call->getArg(0)->getType()->isVectorType()) {
    assert(Call->getType()->isIntegerType());
    APSInt Src;
    if (!popToAPSInt(S, Call->getArg(0), Src))
      return false;
    APInt Result = Fn(Src);
    pushInteger(S, APSInt(std::move(Result), !Src.isSigned()), Call->getType());
    return true;
  }

  // Vector case.
  const Pointer &Arg = S.Stk.pop<Pointer>();
  assert(Arg.getFieldDesc()->isPrimitiveArray());
  const Pointer &Dst = S.Stk.peek<Pointer>();
  assert(Dst.getFieldDesc()->isPrimitiveArray());
  assert(Arg.getFieldDesc()->getNumElems() ==
         Dst.getFieldDesc()->getNumElems());

  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
  PrimType ElemT = *S.getContext().classify(ElemType);
  unsigned NumElems = Arg.getNumElems();
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  for (unsigned I = 0; I != NumElems; ++I) {
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      APSInt Src = Arg.elem<T>(I).toAPSInt();
      APInt Result = Fn(Src);
      Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
    });
  }
  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_elementwise_fp_binop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<std::optional<APFloat>(
        const APFloat &, const APFloat &, std::optional<APSInt> RoundingMode)>
        Fn,
    bool IsScalar = false) {
  assert((Call->getNumArgs() == 2) || (Call->getNumArgs() == 3));
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  assert(VT->getElementType()->isFloatingType());
  unsigned NumElems = VT->getNumElements();

  // Vector case.
  assert(Call->getArg(0)->getType()->isVectorType() &&
         Call->getArg(1)->getType()->isVectorType());
  assert(VT->getElementType() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
  assert(VT->getNumElements() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());

  std::optional<APSInt> RoundingMode = std::nullopt;
  if (Call->getNumArgs() == 3) {
    APSInt RoundingModeVal;
    if (!popToAPSInt(S, Call->getArg(2), RoundingModeVal))
      return false;
    RoundingMode = RoundingModeVal;
  }

  const Pointer &BPtr = S.Stk.pop<Pointer>();
  const Pointer &APtr = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  for (unsigned ElemIdx = 0; ElemIdx != NumElems; ++ElemIdx) {
    using T = PrimConv<PT_Float>::T;
    if (IsScalar && ElemIdx > 0) {
      Dst.elem<T>(ElemIdx) = APtr.elem<T>(ElemIdx);
      continue;
    }
    APFloat ElemA = APtr.elem<T>(ElemIdx).getAPFloat();
    APFloat ElemB = BPtr.elem<T>(ElemIdx).getAPFloat();
    std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
    if (!Result)
      return false;
    Dst.elem<T>(ElemIdx) = static_cast<T>(*Result);
  }

  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_scalar_fp_round_mask_binop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<std::optional<APFloat>(const APFloat &, const APFloat &,
                                              std::optional<APSInt>)>
        Fn) {
  assert(Call->getNumArgs() == 5);
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  unsigned NumElems = VT->getNumElements();

  APSInt RoundingMode;
  if (!popToAPSInt(S, Call->getArg(4), RoundingMode))
    return false;
  uint64_t MaskVal;
  if (!popToUInt64(S, Call->getArg(3), MaskVal))
    return false;
  const Pointer &SrcPtr = S.Stk.pop<Pointer>();
  const Pointer &BPtr = S.Stk.pop<Pointer>();
  const Pointer &APtr = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  using T = PrimConv<PT_Float>::T;

  if (MaskVal & 1) {
    APFloat ElemA = APtr.elem<T>(0).getAPFloat();
    APFloat ElemB = BPtr.elem<T>(0).getAPFloat();
    std::optional<APFloat> Result = Fn(ElemA, ElemB, RoundingMode);
    if (!Result)
      return false;
    Dst.elem<T>(0) = static_cast<T>(*Result);
  } else {
    Dst.elem<T>(0) = SrcPtr.elem<T>(0);
  }

  for (unsigned I = 1; I < NumElems; ++I)
    Dst.elem<T>(I) = APtr.elem<T>(I);

  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_elementwise_int_binop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
  assert(Call->getNumArgs() == 2);

  // Single integer case.
  if (!Call->getArg(0)->getType()->isVectorType()) {
    assert(!Call->getArg(1)->getType()->isVectorType());
    APSInt RHS;
    if (!popToAPSInt(S, Call->getArg(1), RHS))
      return false;
    APSInt LHS;
    if (!popToAPSInt(S, Call->getArg(0), LHS))
      return false;
    APInt Result = Fn(LHS, RHS);
    pushInteger(S, APSInt(std::move(Result), !LHS.isSigned()), Call->getType());
    return true;
  }

  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  assert(VT->getElementType()->isIntegralOrEnumerationType());
  PrimType ElemT = *S.getContext().classify(VT->getElementType());
  unsigned NumElems = VT->getNumElements();
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  // Vector + Scalar case.
  if (!Call->getArg(1)->getType()->isVectorType()) {
    assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());

    APSInt RHS;
    if (!popToAPSInt(S, Call->getArg(1), RHS))
      return false;
    const Pointer &LHS = S.Stk.pop<Pointer>();
    const Pointer &Dst = S.Stk.peek<Pointer>();

    for (unsigned I = 0; I != NumElems; ++I) {
      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
        Dst.elem<T>(I) = static_cast<T>(
            APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
      });
    }
    Dst.initializeAllElements();
    return true;
  }

  // Vector case.
  assert(Call->getArg(0)->getType()->isVectorType() &&
         Call->getArg(1)->getType()->isVectorType());
  assert(VT->getElementType() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
  assert(VT->getNumElements() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
  assert(VT->getElementType()->isIntegralOrEnumerationType());

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  for (unsigned I = 0; I != NumElems; ++I) {
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
      APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
      Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
    });
  }
  Dst.initializeAllElements();

  return true;
}

static bool
interp__builtin_ia32_pack(InterpState &S, CodePtr, const CallExpr *E,
                          llvm::function_ref<APInt(const APSInt &)> PackFn) {
  const auto *VT0 = E->getArg(0)->getType()->castAs<VectorType>();
  [[maybe_unused]] const auto *VT1 =
      E->getArg(1)->getType()->castAs<VectorType>();
  assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
  assert(VT0->getElementType() == VT1->getElementType() &&
         VT0->getNumElements() == VT1->getNumElements() &&
         "pack builtin VT0 and VT1 ElementType must be same");

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const ASTContext &ASTCtx = S.getASTContext();
  unsigned SrcBits = ASTCtx.getIntWidth(VT0->getElementType());
  unsigned LHSVecLen = VT0->getNumElements();
  unsigned SrcPerLane = 128 / SrcBits;
  unsigned Lanes = LHSVecLen * SrcBits / 128;

  PrimType SrcT = *S.getContext().classify(VT0->getElementType());
  PrimType DstT = *S.getContext().classify(getElemType(Dst));
  bool IsUnsigend = getElemType(Dst)->isUnsignedIntegerType();

  for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
    unsigned BaseSrc = Lane * SrcPerLane;
    unsigned BaseDst = Lane * (2 * SrcPerLane);

    for (unsigned I = 0; I != SrcPerLane; ++I) {
      INT_TYPE_SWITCH_NO_BOOL(SrcT, {
        APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
        APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();

        assignIntegral(S, Dst.atIndex(BaseDst + I), DstT,
                       APSInt(PackFn(A), IsUnsigend));
        assignIntegral(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
                       APSInt(PackFn(B), IsUnsigend));
      });
    }
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
                                               const CallExpr *Call,
                                               unsigned BuiltinID) {
  assert(Call->getNumArgs() == 2);

  QualType Arg0Type = Call->getArg(0)->getType();

  // TODO: Support floating-point types.
  if (!(Arg0Type->isIntegerType() ||
        (Arg0Type->isVectorType() &&
         Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
    return false;

  if (!Arg0Type->isVectorType()) {
    assert(!Call->getArg(1)->getType()->isVectorType());
    APSInt RHS;
    if (!popToAPSInt(S, Call->getArg(1), RHS))
      return false;
    APSInt LHS;
    if (!popToAPSInt(S, Arg0Type, LHS))
      return false;
    APInt Result;
    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
      Result = std::max(LHS, RHS);
    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
      Result = std::min(LHS, RHS);
    } else {
      llvm_unreachable("Wrong builtin ID");
    }

    pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());
    return true;
  }

  // Vector case.
  assert(Call->getArg(0)->getType()->isVectorType() &&
         Call->getArg(1)->getType()->isVectorType());
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  assert(VT->getElementType() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
  assert(VT->getNumElements() ==
         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
  assert(VT->getElementType()->isIntegralOrEnumerationType());

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  PrimType ElemT = *S.getContext().classify(VT->getElementType());
  unsigned NumElems = VT->getNumElements();
  for (unsigned I = 0; I != NumElems; ++I) {
    APSInt Elem1;
    APSInt Elem2;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      Elem1 = LHS.elem<T>(I).toAPSInt();
      Elem2 = RHS.elem<T>(I).toAPSInt();
    });

    APSInt Result;
    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
      Result = APSInt(std::max(Elem1, Elem2),
                      Call->getType()->isUnsignedIntegerOrEnumerationType());
    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
      Result = APSInt(std::min(Elem1, Elem2),
                      Call->getType()->isUnsignedIntegerOrEnumerationType());
    } else {
      llvm_unreachable("Wrong builtin ID");
    }

    INT_TYPE_SWITCH_NO_BOOL(ElemT,
                            { Dst.elem<T>(I) = static_cast<T>(Result); });
  }
  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_ia32_pmul(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
                             const APSInt &)>
        Fn) {
  assert(Call->getArg(0)->getType()->isVectorType() &&
         Call->getArg(1)->getType()->isVectorType());
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VT->getElementType());
  unsigned NumElems = VT->getNumElements();
  const auto *DestVT = Call->getType()->castAs<VectorType>();
  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  unsigned DstElem = 0;
  for (unsigned I = 0; I != NumElems; I += 2) {
    APSInt Result;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
      APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
      APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
      APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
      Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
    });

    INT_TYPE_SWITCH_NO_BOOL(DestElemT,
                            { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
    ++DstElem;
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_psadbw(InterpState &S, CodePtr OpPC,
                                        const CallExpr *Call) {
  assert(Call->getNumArgs() == 2);

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const auto *SrcVT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType SrcElemT = *S.getContext().classify(SrcVT->getElementType());
  unsigned SourceLen = SrcVT->getNumElements();
  assert((SourceLen % 8) == 0);

  const auto *DestVT = Call->getType()->castAs<VectorType>();
  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
  bool DestUnsigned =
      DestVT->getElementType()->isUnsignedIntegerOrEnumerationType();

  unsigned DstElem = 0;
  for (unsigned Lane = 0; Lane != SourceLen; Lane += 8) {
    APInt Sum(64, 0);
    for (unsigned I = 0; I != 8; ++I) {
      INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
        APSInt L = LHS.elem<T>(Lane + I).toAPSInt();
        APSInt R = RHS.elem<T>(Lane + I).toAPSInt();
        Sum += llvm::APIntOps::abdu(L.extOrTrunc(8), R.extOrTrunc(8)).zext(64);
      });
    }

    INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
      Dst.elem<T>(DstElem) = static_cast<T>(APSInt(Sum, DestUnsigned));
    });
    ++DstElem;
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_dbpsadbw(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call) {
  assert(Call->getNumArgs() == 3);
  uint64_t Imm;
  if (!popToUInt64(S, Call->getArg(2), Imm))
    return false;

  const Pointer &Src2 = S.Stk.pop<Pointer>();
  const Pointer &Src1 = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const auto *SrcVT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType SrcElemT = *S.getContext().classify(SrcVT->getElementType());
  unsigned SourceLen = SrcVT->getNumElements();

  const auto *DestVT = Call->getType()->castAs<VectorType>();
  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes

  // Phase 1: Shuffle Src2 using all four 2-bit fields of imm8.
  // Within each 128-bit lane, for group j (0..3), select a 4-byte block
  // from Src2 based on bits [2*j+1:2*j] of imm8.
  SmallVector<uint8_t, 64> Shuffled(SourceLen);
  for (unsigned I = 0; I < SourceLen; I += LaneSize) {
    for (unsigned J = 0; J < 4; ++J) {
      unsigned Part = (Imm >> (2 * J)) & 3;
      for (unsigned K = 0; K < 4; ++K) {
        INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
          Shuffled[I + 4 * J + K] =
              static_cast<uint8_t>(Src2.elem<T>(I + 4 * Part + K));
        });
      }
    }
  }

  // Phase 2: Sliding SAD computation.
  // For every group of 4 output u16 values, compute absolute differences
  // using overlapping windows into Src1 and the shuffled array.
  unsigned Size = SourceLen / 2; // number of output u16 elements
  for (unsigned I = 0; I < Size; I += 4) {
    unsigned Sad[4] = {0, 0, 0, 0};
    for (unsigned J = 0; J < 4; ++J) {
      uint8_t A1, A2;
      INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
        A1 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J));
        A2 = static_cast<uint8_t>(Src1.elem<T>(2 * I + J + 4));
      });
      uint8_t B0 = Shuffled[2 * I + J];
      uint8_t B1 = Shuffled[2 * I + J + 1];
      uint8_t B2 = Shuffled[2 * I + J + 2];
      uint8_t B3 = Shuffled[2 * I + J + 3];
      Sad[0] += (A1 > B0) ? (A1 - B0) : (B0 - A1);
      Sad[1] += (A1 > B1) ? (A1 - B1) : (B1 - A1);
      Sad[2] += (A2 > B2) ? (A2 - B2) : (B2 - A2);
      Sad[3] += (A2 > B3) ? (A2 - B3) : (B3 - A2);
    }
    for (unsigned R = 0; R < 4; ++R) {
      INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
        Dst.elem<T>(I + R) =
            static_cast<T>(APSInt(APInt(16, Sad[R]), DestUnsigned));
      });
    }
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_mpsadbw(InterpState &S, CodePtr OpPC,
                                         const CallExpr *Call) {
  assert(Call->getNumArgs() == 3);
  uint64_t Imm;
  if (!popToUInt64(S, Call->getArg(2), Imm))
    return false;

  const Pointer &Src2 = S.Stk.pop<Pointer>();
  const Pointer &Src1 = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const auto *SrcVT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType SrcElemT = *S.getContext().classify(SrcVT->getElementType());
  unsigned SourceLen = SrcVT->getNumElements();
  assert((SourceLen == 16 || SourceLen == 32) &&
         "MPSADBW operates on 128-bit or 256-bit vectors");

  const auto *DestVT = Call->getType()->castAs<VectorType>();
  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  constexpr unsigned LaneSize = 16; // 128-bit lane = 16 bytes
  unsigned NumLanes = SourceLen / LaneSize;

  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
    unsigned Ctrl = (Imm >> (3 * Lane)) & 0x7;
    unsigned AOff = ((Ctrl >> 2) & 1) * 4;
    unsigned BOff = (Ctrl & 3) * 4;
    for (unsigned J = 0; J != 8; ++J) {
      uint16_t Sad = 0;
      for (unsigned K = 0; K != 4; ++K) {
        uint8_t A, B;
        INT_TYPE_SWITCH_NO_BOOL(SrcElemT, {
          A = static_cast<uint8_t>(
              Src1.elem<T>(Lane * LaneSize + AOff + J + K));
          B = static_cast<uint8_t>(Src2.elem<T>(Lane * LaneSize + BOff + K));
        });
        Sad += (A > B) ? (A - B) : (B - A);
      }
      INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
        Dst.elem<T>(Lane * 8 + J) =
            static_cast<T>(APSInt(APInt(16, Sad), DestUnsigned));
      });
    }
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp_builtin_horizontal_int_binop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VT->getElementType());
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  unsigned NumElts = VT->getNumElements();
  unsigned EltBits = S.getASTContext().getIntWidth(VT->getElementType());
  unsigned EltsPerLane = 128 / EltBits;
  unsigned Lanes = NumElts * EltBits / 128;
  unsigned DestIndex = 0;

  for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
    unsigned LaneStart = Lane * EltsPerLane;
    for (unsigned I = 0; I < EltsPerLane; I += 2) {
      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
        APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
        APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
        APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
        Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
      });
    }

    for (unsigned I = 0; I < EltsPerLane; I += 2) {
      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
        APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
        APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
        APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
        Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
      });
    }
  }
  Dst.initializeAllElements();
  return true;
}

static bool interp_builtin_horizontal_fp_binop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APFloat(const APFloat &, const APFloat &,
                               llvm::RoundingMode)>
        Fn) {
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
  llvm::RoundingMode RM = getRoundingMode(FPO);
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();

  unsigned NumElts = VT->getNumElements();
  unsigned EltBits = S.getASTContext().getTypeSize(VT->getElementType());
  unsigned NumLanes = NumElts * EltBits / 128;
  unsigned NumElemsPerLane = NumElts / NumLanes;
  unsigned HalfElemsPerLane = NumElemsPerLane / 2;

  for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
    using T = PrimConv<PT_Float>::T;
    for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
      APFloat Elem1 = LHS.elem<T>(L + (2 * E) + 0).getAPFloat();
      APFloat Elem2 = LHS.elem<T>(L + (2 * E) + 1).getAPFloat();
      Dst.elem<T>(L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
    }
    for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
      APFloat Elem1 = RHS.elem<T>(L + (2 * E) + 0).getAPFloat();
      APFloat Elem2 = RHS.elem<T>(L + (2 * E) + 1).getAPFloat();
      Dst.elem<T>(L + E + HalfElemsPerLane) =
          static_cast<T>(Fn(Elem1, Elem2, RM));
    }
  }
  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
                                        const CallExpr *Call) {
  // Addsub: alternates between subtraction and addition
  // Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
  llvm::RoundingMode RM = getRoundingMode(FPO);
  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  unsigned NumElems = VT->getNumElements();

  using T = PrimConv<PT_Float>::T;
  for (unsigned I = 0; I != NumElems; ++I) {
    APFloat LElem = LHS.elem<T>(I).getAPFloat();
    APFloat RElem = RHS.elem<T>(I).getAPFloat();
    if (I % 2 == 0) {
      // Even indices: subtract
      LElem.subtract(RElem, RM);
    } else {
      // Odd indices: add
      LElem.add(RElem, RM);
    }
    Dst.elem<T>(I) = static_cast<T>(LElem);
  }
  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_pclmulqdq(InterpState &S, CodePtr OpPC,
                                           const CallExpr *Call) {
  // PCLMULQDQ: carry-less multiplication of selected 64-bit halves
  // imm8 bit 0: selects lower (0) or upper (1) 64 bits of first operand
  // imm8 bit 4: selects lower (0) or upper (1) 64 bits of second operand
  assert(Call->getArg(0)->getType()->isVectorType() &&
         Call->getArg(1)->getType()->isVectorType());

  // Extract imm8 argument
  APSInt Imm8;
  if (!popToAPSInt(S, Call->getArg(2), Imm8))
    return false;
  bool SelectUpperA = (Imm8 & 0x01) != 0;
  bool SelectUpperB = (Imm8 & 0x10) != 0;

  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VT->getElementType());
  unsigned NumElems = VT->getNumElements();
  const auto *DestVT = Call->getType()->castAs<VectorType>();
  PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  // Process each 128-bit lane (2 elements at a time)
  for (unsigned Lane = 0; Lane < NumElems; Lane += 2) {
    APSInt A0, A1, B0, B1;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      A0 = LHS.elem<T>(Lane + 0).toAPSInt();
      A1 = LHS.elem<T>(Lane + 1).toAPSInt();
      B0 = RHS.elem<T>(Lane + 0).toAPSInt();
      B1 = RHS.elem<T>(Lane + 1).toAPSInt();
    });

    // Select the appropriate 64-bit values based on imm8
    APInt A = SelectUpperA ? A1 : A0;
    APInt B = SelectUpperB ? B1 : B0;

    // Extend both operands to 128 bits for carry-less multiplication
    APInt A128 = A.zext(128);
    APInt B128 = B.zext(128);

    // Use APIntOps::clmul for carry-less multiplication
    APInt Result = llvm::APIntOps::clmul(A128, B128);

    // Split the 128-bit result into two 64-bit halves
    APSInt ResultLow(Result.extractBits(64, 0), DestUnsigned);
    APSInt ResultHigh(Result.extractBits(64, 64), DestUnsigned);

    INT_TYPE_SWITCH_NO_BOOL(DestElemT, {
      Dst.elem<T>(Lane + 0) = static_cast<T>(ResultLow);
      Dst.elem<T>(Lane + 1) = static_cast<T>(ResultHigh);
    });
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_elementwise_triop_fp(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APFloat(const APFloat &, const APFloat &,
                               const APFloat &, llvm::RoundingMode)>
        Fn) {
  assert(Call->getNumArgs() == 3);

  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
  llvm::RoundingMode RM = getRoundingMode(FPO);
  QualType Arg1Type = Call->getArg(0)->getType();
  QualType Arg2Type = Call->getArg(1)->getType();
  QualType Arg3Type = Call->getArg(2)->getType();

  // Non-vector floating point types.
  if (!Arg1Type->isVectorType()) {
    assert(!Arg2Type->isVectorType());
    assert(!Arg3Type->isVectorType());
    (void)Arg2Type;
    (void)Arg3Type;

    const Floating &Z = S.Stk.pop<Floating>();
    const Floating &Y = S.Stk.pop<Floating>();
    const Floating &X = S.Stk.pop<Floating>();
    APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
    Floating Result = S.allocFloat(X.getSemantics());
    Result.copy(F);
    S.Stk.push<Floating>(Result);
    return true;
  }

  // Vector type.
  assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
         Arg3Type->isVectorType());

  const VectorType *VecTy = Arg1Type->castAs<VectorType>();
  QualType ElemQT = VecTy->getElementType();
  unsigned NumElems = VecTy->getNumElements();

  assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
         ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
  assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
         NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
  assert(ElemQT->isRealFloatingType());
  (void)ElemQT;

  const Pointer &VZ = S.Stk.pop<Pointer>();
  const Pointer &VY = S.Stk.pop<Pointer>();
  const Pointer &VX = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  for (unsigned I = 0; I != NumElems; ++I) {
    using T = PrimConv<PT_Float>::T;
    APFloat X = VX.elem<T>(I).getAPFloat();
    APFloat Y = VY.elem<T>(I).getAPFloat();
    APFloat Z = VZ.elem<T>(I).getAPFloat();
    APFloat F = Fn(X, Y, Z, RM);
    Dst.elem<Floating>(I) = Floating(F);
  }
  Dst.initializeAllElements();
  return true;
}

/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
static bool interp__builtin_ia32_select(InterpState &S, CodePtr OpPC,
                                        const CallExpr *Call) {
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();
  APSInt Mask;
  if (!popToAPSInt(S, Call->getArg(0), Mask))
    return false;
  const Pointer &Dst = S.Stk.peek<Pointer>();

  assert(LHS.getNumElems() == RHS.getNumElems());
  assert(LHS.getNumElems() == Dst.getNumElems());
  unsigned NumElems = LHS.getNumElems();
  PrimType ElemT = LHS.getFieldDesc()->getPrimType();
  PrimType DstElemT = Dst.getFieldDesc()->getPrimType();

  for (unsigned I = 0; I != NumElems; ++I) {
    if (ElemT == PT_Float) {
      assert(DstElemT == PT_Float);
      Dst.elem<Floating>(I) =
          Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
    } else {
      APSInt Elem;
      INT_TYPE_SWITCH(ElemT, {
        Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
      });
      INT_TYPE_SWITCH_NO_BOOL(DstElemT,
                              { Dst.elem<T>(I) = static_cast<T>(Elem); });
    }
  }
  Dst.initializeAllElements();

  return true;
}

/// Scalar variant of AVX512 predicated select:
/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
/// All other elements are taken from RHS.
static bool interp__builtin_ia32_select_scalar(InterpState &S,
                                               const CallExpr *Call) {
  unsigned N =
      Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements();

  const Pointer &W = S.Stk.pop<Pointer>();
  const Pointer &A = S.Stk.pop<Pointer>();
  APSInt U;
  if (!popToAPSInt(S, Call->getArg(0), U))
    return false;
  const Pointer &Dst = S.Stk.peek<Pointer>();

  bool TakeA0 = U.getZExtValue() & 1ULL;

  for (unsigned I = TakeA0; I != N; ++I)
    Dst.elem<Floating>(I) = W.elem<Floating>(I);
  if (TakeA0)
    Dst.elem<Floating>(0) = A.elem<Floating>(0);

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_test_op(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();

  assert(LHS.getNumElems() == RHS.getNumElems());

  unsigned SourceLen = LHS.getNumElems();
  QualType ElemQT = getElemType(LHS);
  OptPrimType ElemPT = S.getContext().classify(ElemQT);
  unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);

  APInt AWide(LaneWidth * SourceLen, 0);
  APInt BWide(LaneWidth * SourceLen, 0);

  for (unsigned I = 0; I != SourceLen; ++I) {
    APInt ALane;
    APInt BLane;

    if (ElemQT->isIntegerType()) { // Get value.
      INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
        ALane = LHS.elem<T>(I).toAPSInt();
        BLane = RHS.elem<T>(I).toAPSInt();
      });
    } else if (ElemQT->isFloatingType()) { // Get only sign bit.
      using T = PrimConv<PT_Float>::T;
      ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
      BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
    } else { // Must be integer or floating type.
      return false;
    }
    AWide.insertBits(ALane, I * LaneWidth);
    BWide.insertBits(BLane, I * LaneWidth);
  }
  pushInteger(S, Fn(AWide, BWide), Call->getType());
  return true;
}

static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
                                           const CallExpr *Call) {
  assert(Call->getNumArgs() == 1);

  const Pointer &Source = S.Stk.pop<Pointer>();

  unsigned SourceLen = Source.getNumElems();
  QualType ElemQT = getElemType(Source);
  OptPrimType ElemT = S.getContext().classify(ElemQT);
  unsigned ResultLen =
      S.getASTContext().getTypeSize(Call->getType()); // Always 32-bit integer.
  APInt Result(ResultLen, 0);

  for (unsigned I = 0; I != SourceLen; ++I) {
    APInt Elem;
    if (ElemQT->isIntegerType()) {
      INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
    } else if (ElemQT->isRealFloatingType()) {
      using T = PrimConv<PT_Float>::T;
      Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
    } else {
      return false;
    }
    Result.setBitVal(I, Elem.isNegative());
  }
  pushInteger(S, Result, Call->getType());
  return true;
}

static bool interp__builtin_elementwise_triop(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
        Fn) {
  assert(Call->getNumArgs() == 3);

  QualType Arg0Type = Call->getArg(0)->getType();
  QualType Arg2Type = Call->getArg(2)->getType();
  // Non-vector integer types.
  if (!Arg0Type->isVectorType()) {
    APSInt Op2;
    if (!popToAPSInt(S, Arg2Type, Op2))
      return false;
    APSInt Op1;
    if (!popToAPSInt(S, Call->getArg(1), Op1))
      return false;
    APSInt Op0;
    if (!popToAPSInt(S, Arg0Type, Op0))
      return false;
    APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
    pushInteger(S, Result, Call->getType());
    return true;
  }

  const auto *VecT = Arg0Type->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VecT->getElementType());
  unsigned NumElems = VecT->getNumElements();
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();

  // Vector + Vector + Scalar case.
  if (!Arg2Type->isVectorType()) {
    APSInt Op2;
    if (!popToAPSInt(S, Arg2Type, Op2))
      return false;

    const Pointer &Op1 = S.Stk.pop<Pointer>();
    const Pointer &Op0 = S.Stk.pop<Pointer>();
    const Pointer &Dst = S.Stk.peek<Pointer>();
    for (unsigned I = 0; I != NumElems; ++I) {
      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
        Dst.elem<T>(I) = static_cast<T>(APSInt(
            Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
            DestUnsigned));
      });
    }
    Dst.initializeAllElements();

    return true;
  }

  // Vector type.
  const Pointer &Op2 = S.Stk.pop<Pointer>();
  const Pointer &Op1 = S.Stk.pop<Pointer>();
  const Pointer &Op0 = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();
  for (unsigned I = 0; I != NumElems; ++I) {
    APSInt Val0, Val1, Val2;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      Val0 = Op0.elem<T>(I).toAPSInt();
      Val1 = Op1.elem<T>(I).toAPSInt();
      Val2 = Op2.elem<T>(I).toAPSInt();
    });
    APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
    INT_TYPE_SWITCH_NO_BOOL(ElemT,
                            { Dst.elem<T>(I) = static_cast<T>(Result); });
  }
  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_ia32_extract_vector(InterpState &S, CodePtr OpPC,
                                                const CallExpr *Call,
                                                unsigned ID) {
  assert(Call->getNumArgs() == 2);

  APSInt ImmAPS;
  if (!popToAPSInt(S, Call->getArg(1), ImmAPS))
    return false;
  uint64_t Index = ImmAPS.getZExtValue();

  const Pointer &Src = S.Stk.pop<Pointer>();
  if (!Src.getFieldDesc()->isPrimitiveArray())
    return false;

  const Pointer &Dst = S.Stk.peek<Pointer>();
  if (!Dst.getFieldDesc()->isPrimitiveArray())
    return false;

  unsigned SrcElems = Src.getNumElems();
  unsigned DstElems = Dst.getNumElems();

  unsigned NumLanes = SrcElems / DstElems;
  unsigned Lane = static_cast<unsigned>(Index % NumLanes);
  unsigned ExtractPos = Lane * DstElems;

  PrimType ElemT = Src.getFieldDesc()->getPrimType();

  TYPE_SWITCH(ElemT, {
    for (unsigned I = 0; I != DstElems; ++I) {
      Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
    }
  });

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_extract_vector_masked(InterpState &S,
                                                       CodePtr OpPC,
                                                       const CallExpr *Call,
                                                       unsigned ID) {
  assert(Call->getNumArgs() == 4);

  APSInt MaskAPS;
  if (!popToAPSInt(S, Call->getArg(3), MaskAPS))
    return false;
  const Pointer &Merge = S.Stk.pop<Pointer>();
  APSInt ImmAPS;
  if (!popToAPSInt(S, Call->getArg(1), ImmAPS))
    return false;
  const Pointer &Src = S.Stk.pop<Pointer>();

  if (!Src.getFieldDesc()->isPrimitiveArray() ||
      !Merge.getFieldDesc()->isPrimitiveArray())
    return false;

  const Pointer &Dst = S.Stk.peek<Pointer>();
  if (!Dst.getFieldDesc()->isPrimitiveArray())
    return false;

  unsigned SrcElems = Src.getNumElems();
  unsigned DstElems = Dst.getNumElems();

  unsigned NumLanes = SrcElems / DstElems;
  unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
  unsigned Base = Lane * DstElems;

  PrimType ElemT = Src.getFieldDesc()->getPrimType();

  TYPE_SWITCH(ElemT, {
    for (unsigned I = 0; I != DstElems; ++I) {
      if (MaskAPS[I])
        Dst.elem<T>(I) = Src.elem<T>(Base + I);
      else
        Dst.elem<T>(I) = Merge.elem<T>(I);
    }
  });

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_insert_subvector(InterpState &S, CodePtr OpPC,
                                                  const CallExpr *Call,
                                                  unsigned ID) {
  assert(Call->getNumArgs() == 3);

  APSInt ImmAPS;
  if (!popToAPSInt(S, Call->getArg(2), ImmAPS))
    return false;
  uint64_t Index = ImmAPS.getZExtValue();

  const Pointer &SubVec = S.Stk.pop<Pointer>();
  if (!SubVec.getFieldDesc()->isPrimitiveArray())
    return false;

  const Pointer &BaseVec = S.Stk.pop<Pointer>();
  if (!BaseVec.getFieldDesc()->isPrimitiveArray())
    return false;

  const Pointer &Dst = S.Stk.peek<Pointer>();

  unsigned BaseElements = BaseVec.getNumElems();
  unsigned SubElements = SubVec.getNumElems();

  assert(SubElements != 0 && BaseElements != 0 &&
         (BaseElements % SubElements) == 0);

  unsigned NumLanes = BaseElements / SubElements;
  unsigned Lane = static_cast<unsigned>(Index % NumLanes);
  unsigned InsertPos = Lane * SubElements;

  PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();

  TYPE_SWITCH(ElemT, {
    for (unsigned I = 0; I != BaseElements; ++I)
      Dst.elem<T>(I) = BaseVec.elem<T>(I);
    for (unsigned I = 0; I != SubElements; ++I)
      Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
  });

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
                                            const CallExpr *Call) {
  assert(Call->getNumArgs() == 1);

  const Pointer &Source = S.Stk.pop<Pointer>();
  const Pointer &Dest = S.Stk.peek<Pointer>();

  unsigned SourceLen = Source.getNumElems();
  QualType ElemQT = getElemType(Source);
  OptPrimType ElemT = S.getContext().classify(ElemQT);
  unsigned ElemBitWidth = S.getASTContext().getTypeSize(ElemQT);

  bool DestUnsigned = Call->getCallReturnType(S.getASTContext())
                          ->castAs<VectorType>()
                          ->getElementType()
                          ->isUnsignedIntegerOrEnumerationType();

  INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
    APSInt MinIndex(ElemBitWidth, DestUnsigned);
    APSInt MinVal = Source.elem<T>(0).toAPSInt();

    for (unsigned I = 1; I != SourceLen; ++I) {
      APSInt Val = Source.elem<T>(I).toAPSInt();
      if (MinVal.ugt(Val)) {
        MinVal = Val;
        MinIndex = I;
      }
    }

    Dest.elem<T>(0) = static_cast<T>(MinVal);
    Dest.elem<T>(1) = static_cast<T>(MinIndex);
    for (unsigned I = 2; I != SourceLen; ++I) {
      Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
    }
  });
  Dest.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call, bool MaskZ) {
  assert(Call->getNumArgs() == 5);

  APSInt UVal;
  if (!popToAPSInt(S, Call->getArg(4), UVal))
    return false;
  APInt U = UVal; // Lane mask
  APSInt ImmVal;
  if (!popToAPSInt(S, Call->getArg(3), ImmVal))
    return false;
  APInt Imm = ImmVal; // Ternary truth table
  const Pointer &C = S.Stk.pop<Pointer>();
  const Pointer &B = S.Stk.pop<Pointer>();
  const Pointer &A = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  unsigned DstLen = A.getNumElems();
  QualType ElemQT = getElemType(A);
  OptPrimType ElemT = S.getContext().classify(ElemQT);
  unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);
  bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();

  INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
    for (unsigned I = 0; I != DstLen; ++I) {
      APInt ALane = A.elem<T>(I).toAPSInt();
      APInt BLane = B.elem<T>(I).toAPSInt();
      APInt CLane = C.elem<T>(I).toAPSInt();
      APInt RLane(LaneWidth, 0);
      if (U[I]) { // If lane not masked, compute ternary logic.
        for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
          unsigned ABit = ALane[Bit];
          unsigned BBit = BLane[Bit];
          unsigned CBit = CLane[Bit];
          unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
          RLane.setBitVal(Bit, Imm[Idx]);
        }
        Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
      } else if (MaskZ) { // If zero masked, zero the lane.
        Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
      } else { // Just masked, put in A lane.
        Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
      }
    }
  });
  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_vec_ext(InterpState &S, CodePtr OpPC,
                                         const CallExpr *Call, unsigned ID) {
  assert(Call->getNumArgs() == 2);

  APSInt ImmAPS;
  if (!popToAPSInt(S, Call->getArg(1), ImmAPS))
    return false;
  const Pointer &Vec = S.Stk.pop<Pointer>();
  if (!Vec.getFieldDesc()->isPrimitiveArray())
    return false;

  unsigned NumElems = Vec.getNumElems();
  unsigned Index =
      static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));

  PrimType ElemT = Vec.getFieldDesc()->getPrimType();
  // FIXME(#161685): Replace float+int split with a numeric-only type switch
  if (ElemT == PT_Float) {
    S.Stk.push<Floating>(Vec.elem<Floating>(Index));
    return true;
  }
  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
    APSInt V = Vec.elem<T>(Index).toAPSInt();
    pushInteger(S, V, Call->getType());
  });

  return true;
}

static bool interp__builtin_ia32_vec_set(InterpState &S, CodePtr OpPC,
                                         const CallExpr *Call, unsigned ID) {
  assert(Call->getNumArgs() == 3);

  APSInt ImmAPS;
  if (!popToAPSInt(S, Call->getArg(2), ImmAPS))
    return false;
  APSInt ValAPS;
  if (!popToAPSInt(S, Call->getArg(1), ValAPS))
    return false;

  const Pointer &Base = S.Stk.pop<Pointer>();
  if (!Base.getFieldDesc()->isPrimitiveArray())
    return false;

  const Pointer &Dst = S.Stk.peek<Pointer>();

  unsigned NumElems = Base.getNumElems();
  unsigned Index =
      static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));

  PrimType ElemT = Base.getFieldDesc()->getPrimType();
  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
    for (unsigned I = 0; I != NumElems; ++I)
      Dst.elem<T>(I) = Base.elem<T>(I);
    Dst.elem<T>(Index) = static_cast<T>(ValAPS);
  });

  Dst.initializeAllElements();
  return true;
}

static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
                        bool IsUnsigned) {
  switch (Imm & 0x7) {
  case 0x00: // _MM_CMPINT_EQ
    return (A == B);
  case 0x01: // _MM_CMPINT_LT
    return IsUnsigned ? A.ult(B) : A.slt(B);
  case 0x02: // _MM_CMPINT_LE
    return IsUnsigned ? A.ule(B) : A.sle(B);
  case 0x03: // _MM_CMPINT_FALSE
    return false;
  case 0x04: // _MM_CMPINT_NE
    return (A != B);
  case 0x05: // _MM_CMPINT_NLT
    return IsUnsigned ? A.ugt(B) : A.sgt(B);
  case 0x06: // _MM_CMPINT_NLE
    return IsUnsigned ? A.uge(B) : A.sge(B);
  case 0x07: // _MM_CMPINT_TRUE
    return true;
  default:
    llvm_unreachable("Invalid Op");
  }
}

static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call, unsigned ID,
                                          bool IsUnsigned) {
  assert(Call->getNumArgs() == 4);

  APSInt Mask;
  if (!popToAPSInt(S, Call->getArg(3), Mask))
    return false;
  APSInt Opcode;
  if (!popToAPSInt(S, Call->getArg(2), Opcode))
    return false;
  unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
  const Pointer &RHS = S.Stk.pop<Pointer>();
  const Pointer &LHS = S.Stk.pop<Pointer>();

  assert(LHS.getNumElems() == RHS.getNumElems());

  APInt RetMask = APInt::getZero(LHS.getNumElems());
  unsigned VectorLen = LHS.getNumElems();
  PrimType ElemT = LHS.getFieldDesc()->getPrimType();

  for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
    APSInt A, B;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      A = LHS.elem<T>(ElemNum).toAPSInt();
      B = RHS.elem<T>(ElemNum).toAPSInt();
    });
    RetMask.setBitVal(ElemNum,
                      Mask[ElemNum] && evalICmpImm(CmpOp, A, B, IsUnsigned));
  }
  pushInteger(S, RetMask, Call->getType());
  return true;
}

static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
                                            const CallExpr *Call) {
  assert(Call->getNumArgs() == 1);

  QualType Arg0Type = Call->getArg(0)->getType();
  const auto *VecT = Arg0Type->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VecT->getElementType());
  unsigned NumElems = VecT->getNumElements();
  bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
  const Pointer &Src = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  for (unsigned I = 0; I != NumElems; ++I) {
    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
      APSInt ElemI = Src.elem<T>(I).toAPSInt();
      APInt ConflictMask(ElemI.getBitWidth(), 0);
      for (unsigned J = 0; J != I; ++J) {
        APSInt ElemJ = Src.elem<T>(J).toAPSInt();
        ConflictMask.setBitVal(J, ElemI == ElemJ);
      }
      Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
    });
  }
  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
                                              const CallExpr *Call,
                                              unsigned ID) {
  assert(Call->getNumArgs() == 1);

  const Pointer &Vec = S.Stk.pop<Pointer>();
  unsigned RetWidth = S.getASTContext().getIntWidth(Call->getType());
  APInt RetMask(RetWidth, 0);

  unsigned VectorLen = Vec.getNumElems();
  PrimType ElemT = Vec.getFieldDesc()->getPrimType();

  for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
    APSInt A;
    INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
    unsigned MSB = A[A.getBitWidth() - 1];
    RetMask.setBitVal(ElemNum, MSB);
  }
  pushInteger(S, RetMask, Call->getType());
  return true;
}

static bool interp__builtin_ia32_cvt_mask2vec(InterpState &S, CodePtr OpPC,
                                              const CallExpr *Call,
                                              unsigned ID) {
  assert(Call->getNumArgs() == 1);

  APSInt Mask;
  if (!popToAPSInt(S, Call->getArg(0), Mask))
    return false;

  const Pointer &Vec = S.Stk.peek<Pointer>();
  unsigned NumElems = Vec.getNumElems();
  PrimType ElemT = Vec.getFieldDesc()->getPrimType();

  for (unsigned I = 0; I != NumElems; ++I) {
    bool BitSet = Mask[I];

    INT_TYPE_SWITCH_NO_BOOL(
        ElemT, { Vec.elem<T>(I) = BitSet ? T::from(-1) : T::from(0); });
  }

  Vec.initializeAllElements();

  return true;
}

static bool interp__builtin_ia32_cvtsd2ss(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call,
                                          bool HasRoundingMask) {
  APSInt Rounding, MaskInt;
  Pointer Src, B, A;

  if (HasRoundingMask) {
    assert(Call->getNumArgs() == 5);
    if (!popToAPSInt(S, Call->getArg(4), Rounding))
      return false;
    if (!popToAPSInt(S, Call->getArg(3), MaskInt))
      return false;
    Src = S.Stk.pop<Pointer>();
    B = S.Stk.pop<Pointer>();
    A = S.Stk.pop<Pointer>();
    if (!CheckLoad(S, OpPC, A) || !CheckLoad(S, OpPC, B) ||
        !CheckLoad(S, OpPC, Src))
      return false;
  } else {
    assert(Call->getNumArgs() == 2);
    B = S.Stk.pop<Pointer>();
    A = S.Stk.pop<Pointer>();
    if (!CheckLoad(S, OpPC, A) || !CheckLoad(S, OpPC, B))
      return false;
  }

  const auto *DstVTy = Call->getType()->castAs<VectorType>();
  unsigned NumElems = DstVTy->getNumElements();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  // Copy all elements except lane 0 (overwritten below) from A to Dst.
  for (unsigned I = 1; I != NumElems; ++I)
    Dst.elem<Floating>(I) = A.elem<Floating>(I);

  // Convert element 0 from double to float, or use Src if masked off.
  if (!HasRoundingMask || (MaskInt.getZExtValue() & 0x1)) {
    assert(S.getASTContext().FloatTy == DstVTy->getElementType() &&
           "cvtsd2ss requires float element type in destination vector");

    Floating Conv = S.allocFloat(
        S.getASTContext().getFloatTypeSemantics(DstVTy->getElementType()));
    APFloat SrcVal = B.elem<Floating>(0).getAPFloat();
    if (!convertDoubleToFloatStrict(SrcVal, Conv, S, Call))
      return false;
    Dst.elem<Floating>(0) = Conv;
  } else {
    Dst.elem<Floating>(0) = Src.elem<Floating>(0);
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_cvtpd2ps(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call, bool IsMasked,
                                          bool HasRounding) {
  APSInt MaskVal;
  Pointer PassThrough;
  Pointer Src;
  APSInt Rounding;

  if (IsMasked) {
    // Pop in reverse order.
    if (HasRounding) {
      if (!popToAPSInt(S, Call->getArg(3), Rounding))
        return false;
      if (!popToAPSInt(S, Call->getArg(2), MaskVal))
        return false;
      PassThrough = S.Stk.pop<Pointer>();
      Src = S.Stk.pop<Pointer>();
    } else {
      if (!popToAPSInt(S, Call->getArg(2), MaskVal))
        return false;
      PassThrough = S.Stk.pop<Pointer>();
      Src = S.Stk.pop<Pointer>();
    }

    if (!CheckLoad(S, OpPC, PassThrough))
      return false;
  } else {
    // Pop source only.
    Src = S.Stk.pop<Pointer>();
  }

  if (!CheckLoad(S, OpPC, Src))
    return false;

  const auto *RetVTy = Call->getType()->castAs<VectorType>();
  unsigned RetElems = RetVTy->getNumElements();
  unsigned SrcElems = Src.getNumElems();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  // Initialize destination with passthrough or zeros.
  for (unsigned I = 0; I != RetElems; ++I)
    if (IsMasked)
      Dst.elem<Floating>(I) = PassThrough.elem<Floating>(I);
    else
      Dst.elem<Floating>(I) = Floating(APFloat(0.0f));

  assert(S.getASTContext().FloatTy == RetVTy->getElementType() &&
         "cvtpd2ps requires float element type in return vector");

  // Convert double to float for enabled elements (only process source elements
  // that exist).
  for (unsigned I = 0; I != SrcElems; ++I) {
    if (IsMasked && !MaskVal[I])
      continue;

    APFloat SrcVal = Src.elem<Floating>(I).getAPFloat();

    Floating Conv = S.allocFloat(
        S.getASTContext().getFloatTypeSemantics(RetVTy->getElementType()));
    if (!convertDoubleToFloatStrict(SrcVal, Conv, S, Call))
      return false;
    Dst.elem<Floating>(I) = Conv;
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_shuffle_generic(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<std::pair<unsigned, int>(unsigned, const APInt &)>
        GetSourceIndex) {

  assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);

  APInt ShuffleMask;
  Pointer A, MaskVector, B;
  bool IsVectorMask = false;
  bool IsSingleOperand = (Call->getNumArgs() == 2);

  if (IsSingleOperand) {
    QualType MaskType = Call->getArg(1)->getType();
    if (MaskType->isVectorType()) {
      IsVectorMask = true;
      MaskVector = S.Stk.pop<Pointer>();
      A = S.Stk.pop<Pointer>();
      B = A;
    } else if (MaskType->isIntegerType()) {
      APSInt MaskVal;
      if (!popToAPSInt(S, Call->getArg(1), MaskVal))
        return false;
      ShuffleMask = MaskVal;
      A = S.Stk.pop<Pointer>();
      B = A;
    } else {
      return false;
    }
  } else {
    QualType Arg2Type = Call->getArg(2)->getType();
    if (Arg2Type->isVectorType()) {
      IsVectorMask = true;
      B = S.Stk.pop<Pointer>();
      MaskVector = S.Stk.pop<Pointer>();
      A = S.Stk.pop<Pointer>();
    } else if (Arg2Type->isIntegerType()) {
      APSInt MaskVal;
      if (!popToAPSInt(S, Call->getArg(2), MaskVal))
        return false;
      ShuffleMask = MaskVal;
      B = S.Stk.pop<Pointer>();
      A = S.Stk.pop<Pointer>();
    } else {
      return false;
    }
  }

  QualType Arg0Type = Call->getArg(0)->getType();
  const auto *VecT = Arg0Type->castAs<VectorType>();
  PrimType ElemT = *S.getContext().classify(VecT->getElementType());
  unsigned NumElems = VecT->getNumElements();

  const Pointer &Dst = S.Stk.peek<Pointer>();

  PrimType MaskElemT = PT_Uint32;
  if (IsVectorMask) {
    QualType Arg1Type = Call->getArg(1)->getType();
    const auto *MaskVecT = Arg1Type->castAs<VectorType>();
    QualType MaskElemType = MaskVecT->getElementType();
    MaskElemT = *S.getContext().classify(MaskElemType);
  }

  for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
    if (IsVectorMask) {
      INT_TYPE_SWITCH(MaskElemT,
                      { ShuffleMask = MaskVector.elem<T>(DstIdx).toAPSInt(); });
    }

    auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);

    if (SrcIdx < 0) {
      // Zero out this element
      if (ElemT == PT_Float) {
        Dst.elem<Floating>(DstIdx) = Floating(
            S.getASTContext().getFloatTypeSemantics(VecT->getElementType()));
      } else {
        INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
      }
    } else {
      const Pointer &Src = (SrcVecIdx == 0) ? A : B;
      TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
    }
  }
  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_ia32_shuffle_generic(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
        GetSourceIndex) {
  return interp__builtin_ia32_shuffle_generic(
      S, OpPC, Call,
      [&GetSourceIndex](unsigned DstIdx,
                        const APInt &Mask) -> std::pair<unsigned, int> {
        return GetSourceIndex(DstIdx, Mask.getZExtValue());
      });
}

static bool interp__builtin_ia32_shift_with_count(
    InterpState &S, CodePtr OpPC, const CallExpr *Call,
    llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
    llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {

  assert(Call->getNumArgs() == 2);

  const Pointer &Count = S.Stk.pop<Pointer>();
  const Pointer &Source = S.Stk.pop<Pointer>();

  QualType SourceType = Call->getArg(0)->getType();
  QualType CountType = Call->getArg(1)->getType();
  assert(SourceType->isVectorType() && CountType->isVectorType());

  const auto *SourceVecT = SourceType->castAs<VectorType>();
  const auto *CountVecT = CountType->castAs<VectorType>();
  PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
  PrimType CountElemT = *S.getContext().classify(CountVecT->getElementType());

  const Pointer &Dst = S.Stk.peek<Pointer>();

  unsigned DestEltWidth =
      S.getASTContext().getTypeSize(SourceVecT->getElementType());
  bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
  unsigned DestLen = SourceVecT->getNumElements();
  unsigned CountEltWidth =
      S.getASTContext().getTypeSize(CountVecT->getElementType());
  unsigned NumBitsInQWord = 64;
  unsigned NumCountElts = NumBitsInQWord / CountEltWidth;

  uint64_t CountLQWord = 0;
  for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
    uint64_t Elt = 0;
    INT_TYPE_SWITCH(CountElemT,
                    { Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
    CountLQWord |= (Elt << (EltIdx * CountEltWidth));
  }

  for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
    APSInt Elt;
    INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });

    APInt Result;
    if (CountLQWord < DestEltWidth) {
      Result = ShiftOp(Elt, CountLQWord);
    } else {
      Result = OverflowOp(Elt, DestEltWidth);
    }
    if (IsDestUnsigned) {
      INT_TYPE_SWITCH(SourceElemT, {
        Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
      });
    } else {
      INT_TYPE_SWITCH(SourceElemT, {
        Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
      });
    }
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
                                                 const CallExpr *Call) {

  assert(Call->getNumArgs() == 3);

  QualType SourceType = Call->getArg(0)->getType();
  QualType ShuffleMaskType = Call->getArg(1)->getType();
  QualType ZeroMaskType = Call->getArg(2)->getType();
  if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
      !ZeroMaskType->isIntegerType()) {
    return false;
  }

  Pointer Source, ShuffleMask;
  APSInt ZeroMask;
  if (!popToAPSInt(S, Call->getArg(2), ZeroMask))
    return false;
  ShuffleMask = S.Stk.pop<Pointer>();
  Source = S.Stk.pop<Pointer>();

  const auto *SourceVecT = SourceType->castAs<VectorType>();
  const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
  assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
  assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());

  PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
  PrimType ShuffleMaskElemT =
      *S.getContext().classify(ShuffleMaskVecT->getElementType());

  unsigned NumBytesInQWord = 8;
  unsigned NumBitsInByte = 8;
  unsigned NumBytes = SourceVecT->getNumElements();
  unsigned NumQWords = NumBytes / NumBytesInQWord;
  unsigned RetWidth = ZeroMask.getBitWidth();
  APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);

  for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
    APInt SourceQWord(64, 0);
    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      uint64_t Byte = 0;
      INT_TYPE_SWITCH(SourceElemT, {
        Byte = static_cast<uint64_t>(
            Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
      });
      SourceQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
    }

    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
      unsigned M = 0;
      INT_TYPE_SWITCH(ShuffleMaskElemT, {
        M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
      });

      if (ZeroMask[SelIdx]) {
        RetMask.setBitVal(SelIdx, SourceQWord[M]);
      }
    }
  }

  pushInteger(S, RetMask, Call->getType());
  return true;
}

static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
                                           const CallExpr *Call) {
  // Arguments are: vector of floats, rounding immediate
  assert(Call->getNumArgs() == 2);

  APSInt Imm;
  if (!popToAPSInt(S, Call->getArg(1), Imm))
    return false;
  const Pointer &Src = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  assert(Src.getFieldDesc()->isPrimitiveArray());
  assert(Dst.getFieldDesc()->isPrimitiveArray());

  const auto *SrcVTy = Call->getArg(0)->getType()->castAs<VectorType>();
  unsigned SrcNumElems = SrcVTy->getNumElements();
  const auto *DstVTy = Call->getType()->castAs<VectorType>();
  unsigned DstNumElems = DstVTy->getNumElements();

  const llvm::fltSemantics &HalfSem =
      S.getASTContext().getFloatTypeSemantics(S.getASTContext().HalfTy);

  // imm[2] == 1 means use MXCSR rounding mode.
  // In that case, we can only evaluate if the conversion is exact.
  int ImmVal = Imm.getZExtValue();
  bool UseMXCSR = (ImmVal & 4) != 0;
  bool IsFPConstrained =
      Call->getFPFeaturesInEffect(S.getASTContext().getLangOpts())
          .isFPConstrained();

  llvm::RoundingMode RM;
  if (!UseMXCSR) {
    switch (ImmVal & 3) {
    case 0:
      RM = llvm::RoundingMode::NearestTiesToEven;
      break;
    case 1:
      RM = llvm::RoundingMode::TowardNegative;
      break;
    case 2:
      RM = llvm::RoundingMode::TowardPositive;
      break;
    case 3:
      RM = llvm::RoundingMode::TowardZero;
      break;
    default:
      llvm_unreachable("Invalid immediate rounding mode");
    }
  } else {
    // For MXCSR, we must check for exactness. We can use any rounding mode
    // for the trial conversion since the result is the same if it's exact.
    RM = llvm::RoundingMode::NearestTiesToEven;
  }

  QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
  PrimType DstElemT = *S.getContext().classify(DstElemQT);

  for (unsigned I = 0; I != SrcNumElems; ++I) {
    Floating SrcVal = Src.elem<Floating>(I);
    APFloat DstVal = SrcVal.getAPFloat();

    bool LostInfo;
    APFloat::opStatus St = DstVal.convert(HalfSem, RM, &LostInfo);

    if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
      S.FFDiag(S.Current->getSource(OpPC),
               diag::note_constexpr_dynamic_rounding);
      return false;
    }

    INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
      // Convert the destination value's bit pattern to an unsigned integer,
      // then reconstruct the element using the target type's 'from' method.
      uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
      Dst.elem<T>(I) = T::from(RawBits);
    });
  }

  // Zero out remaining elements if the destination has more elements
  // (e.g., vcvtps2ph converting 4 floats to 8 shorts).
  if (DstNumElems > SrcNumElems) {
    for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
      INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
    }
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
                                              const CallExpr *Call) {
  assert(Call->getNumArgs() == 2);

  QualType ATy = Call->getArg(0)->getType();
  QualType BTy = Call->getArg(1)->getType();
  if (!ATy->isVectorType() || !BTy->isVectorType()) {
    return false;
  }

  const Pointer &BPtr = S.Stk.pop<Pointer>();
  const Pointer &APtr = S.Stk.pop<Pointer>();
  const auto *AVecT = ATy->castAs<VectorType>();
  assert(AVecT->getNumElements() ==
         BTy->castAs<VectorType>()->getNumElements());

  PrimType ElemT = *S.getContext().classify(AVecT->getElementType());

  unsigned NumBytesInQWord = 8;
  unsigned NumBitsInByte = 8;
  unsigned NumBytes = AVecT->getNumElements();
  unsigned NumQWords = NumBytes / NumBytesInQWord;
  const Pointer &Dst = S.Stk.peek<Pointer>();

  for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
    APInt BQWord(64, 0);
    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
      INT_TYPE_SWITCH(ElemT, {
        uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
        BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
      });
    }

    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
      uint64_t Ctrl = 0;
      INT_TYPE_SWITCH(
          ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });

      APInt Byte(8, 0);
      for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
        Byte.setBitVal(BitIdx, BQWord[(Ctrl + BitIdx) & 0x3F]);
      }
      INT_TYPE_SWITCH(ElemT,
                      { Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
    }
  }

  Dst.initializeAllElements();

  return true;
}

static bool interp__builtin_ia32_gfni_affine(InterpState &S, CodePtr OpPC,
                                             const CallExpr *Call,
                                             bool Inverse) {
  assert(Call->getNumArgs() == 3);
  QualType XType = Call->getArg(0)->getType();
  QualType AType = Call->getArg(1)->getType();
  QualType ImmType = Call->getArg(2)->getType();
  if (!XType->isVectorType() || !AType->isVectorType() ||
      !ImmType->isIntegerType()) {
    return false;
  }

  Pointer X, A;
  APSInt Imm;
  if (!popToAPSInt(S, Call->getArg(2), Imm))
    return false;
  A = S.Stk.pop<Pointer>();
  X = S.Stk.pop<Pointer>();

  const Pointer &Dst = S.Stk.peek<Pointer>();
  const auto *AVecT = AType->castAs<VectorType>();
  assert(XType->castAs<VectorType>()->getNumElements() ==
         AVecT->getNumElements());
  unsigned NumBytesInQWord = 8;
  unsigned NumBytes = AVecT->getNumElements();
  unsigned NumBitsInQWord = 64;
  unsigned NumQWords = NumBytes / NumBytesInQWord;
  unsigned NumBitsInByte = 8;
  PrimType AElemT = *S.getContext().classify(AVecT->getElementType());

  // computing A*X + Imm
  for (unsigned QWordIdx = 0; QWordIdx != NumQWords; ++QWordIdx) {
    // Extract the QWords from X, A
    APInt XQWord(NumBitsInQWord, 0);
    APInt AQWord(NumBitsInQWord, 0);
    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
      uint8_t XByte;
      uint8_t AByte;
      INT_TYPE_SWITCH(AElemT, {
        XByte = static_cast<uint8_t>(X.elem<T>(Idx));
        AByte = static_cast<uint8_t>(A.elem<T>(Idx));
      });

      XQWord.insertBits(APInt(NumBitsInByte, XByte), ByteIdx * NumBitsInByte);
      AQWord.insertBits(APInt(NumBitsInByte, AByte), ByteIdx * NumBitsInByte);
    }

    for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
      unsigned Idx = QWordIdx * NumBytesInQWord + ByteIdx;
      uint8_t XByte =
          XQWord.lshr(ByteIdx * NumBitsInByte).getLoBits(8).getZExtValue();
      INT_TYPE_SWITCH(AElemT, {
        Dst.elem<T>(Idx) = T::from(GFNIAffine(XByte, AQWord, Imm, Inverse));
      });
    }
  }
  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_gfni_mul(InterpState &S, CodePtr OpPC,
                                          const CallExpr *Call) {
  assert(Call->getNumArgs() == 2);

  QualType AType = Call->getArg(0)->getType();
  QualType BType = Call->getArg(1)->getType();
  if (!AType->isVectorType() || !BType->isVectorType()) {
    return false;
  }

  Pointer A, B;
  B = S.Stk.pop<Pointer>();
  A = S.Stk.pop<Pointer>();

  const Pointer &Dst = S.Stk.peek<Pointer>();
  const auto *AVecT = AType->castAs<VectorType>();
  assert(AVecT->getNumElements() ==
         BType->castAs<VectorType>()->getNumElements());

  PrimType AElemT = *S.getContext().classify(AVecT->getElementType());
  unsigned NumBytes = A.getNumElems();

  for (unsigned ByteIdx = 0; ByteIdx != NumBytes; ++ByteIdx) {
    uint8_t AByte, BByte;
    INT_TYPE_SWITCH(AElemT, {
      AByte = static_cast<uint8_t>(A.elem<T>(ByteIdx));
      BByte = static_cast<uint8_t>(B.elem<T>(ByteIdx));
      Dst.elem<T>(ByteIdx) = T::from(GFNIMul(AByte, BByte));
    });
  }

  Dst.initializeAllElements();
  return true;
}

static bool interp__builtin_ia32_vpdp(InterpState &S, CodePtr OpPC,
                                      const CallExpr *Call, bool IsSaturating) {
  assert(Call->getNumArgs() == 3);

  QualType SrcT = Call->getArg(0)->getType();
  QualType OpAT = Call->getArg(1)->getType();
  QualType OpBT = Call->getArg(2)->getType();
  QualType DstT = Call->getType();
  if (!SrcT->isVectorType() || !OpAT->isVectorType() || !OpBT->isVectorType() ||
      !DstT->isVectorType())
    return false;

  const auto *SrcVecT = SrcT->castAs<VectorType>();
  const auto *OpAVecT = OpAT->castAs<VectorType>();
  const auto *OpBVecT = OpBT->castAs<VectorType>();
  const auto *DstVecT = DstT->castAs<VectorType>();

  assert(OpAVecT->getNumElements() == OpBVecT->getNumElements());

  unsigned NumSrcElems = SrcVecT->getNumElements();
  unsigned NumOperandElems = OpAVecT->getNumElements();
  unsigned ElemsPerLane = NumOperandElems / NumSrcElems;

  PrimType SrcElemT = *S.getContext().classify(SrcVecT->getElementType());
  PrimType OpAElemT = *S.getContext().classify(OpAVecT->getElementType());
  PrimType OpBElemT = *S.getContext().classify(OpBVecT->getElementType());
  PrimType DstElemT = *S.getContext().classify(DstVecT->getElementType());

  assert(SrcElemT == DstElemT);

  const Pointer &OpBPtr = S.Stk.pop<Pointer>();
  const Pointer &OpAPtr = S.Stk.pop<Pointer>();
  const Pointer &SrcPtr = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  for (unsigned I = 0; I != NumSrcElems; ++I) {
    APSInt Acc;
    INT_TYPE_SWITCH_NO_BOOL(SrcElemT, { Acc = SrcPtr.elem<T>(I).toAPSInt(); });
    Acc = Acc.sext(64);
    for (unsigned J = 0; J != ElemsPerLane; ++J) {
      APSInt OpA, OpB;
      INT_TYPE_SWITCH_NO_BOOL(
          OpAElemT, { OpA = OpAPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
      INT_TYPE_SWITCH_NO_BOOL(
          OpBElemT, { OpB = OpBPtr.elem<T>(ElemsPerLane * I + J).toAPSInt(); });
      OpA = APSInt(OpA.extend(64), false);
      OpB = APSInt(OpB.extend(64), false);
      Acc += OpA * OpB;
    }
    if (IsSaturating)
      Acc = APSInt(Acc.truncSSat(32), false);
    else
      Acc = APSInt(Acc.trunc(32), false);
    INT_TYPE_SWITCH_NO_BOOL(DstElemT,
                            { Dst.elem<T>(I) = static_cast<T>(Acc); });
  }
  Dst.initializeAllElements();
  return true;
}

// Bit Matrix Multiply and Accumulate (AVX512BMM). Each 256-bit lane holds a
// 16x16 bit matrix as 16 x i16 elements; element i is row i and bit j of that
// element is entry [i][j]. The accumulator (third argument, src1 in the AMD
// ISA) provides the initial value of each result bit, into which the bit-matrix
// product of the first two arguments (src2 * src3) is reduced with OR (vbmacor)
// or XOR (vbmacxor):
//   for i in 0..15, j in 0..15:
//     bit = C[16*i+j]
//     for k in 0..15: bit OP= A[16*i+k] & B[16*k+j]
//     dest[16*i+j] = bit
static bool interp__builtin_ia32_bmac(InterpState &S, CodePtr OpPC,
                                      const CallExpr *Call, bool IsXor) {
  assert(Call->getNumArgs() == 3);

  // AST-based type checks before popping the stack.
  QualType AType = Call->getArg(0)->getType();
  QualType BType = Call->getArg(1)->getType();
  QualType CType = Call->getArg(2)->getType();
  if (!AType->isVectorType() || !BType->isVectorType() ||
      !CType->isVectorType())
    return false;

  const Pointer &C = S.Stk.pop<Pointer>();
  const Pointer &B = S.Stk.pop<Pointer>();
  const Pointer &A = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  // check if all three primitive arrays are with 16-bit elements.
  auto isValid16BitArray = [](const Pointer &P) {
    const Descriptor *D = P.getFieldDesc();
    if (!D->isPrimitiveArray())
      return false;
    PrimType PT = D->getPrimType();
    return ((PT == PT_Sint16) || (PT == PT_Uint16));
  };

  if (!isValid16BitArray(A) || !isValid16BitArray(B) || !isValid16BitArray(C))
    return false;

  PrimType ElemT = A.getFieldDesc()->getPrimType();
  unsigned NumElems = A.getNumElems();
  assert(NumElems % 16 == 0 && "BMM operates on 256-bit lanes of 16 x i16");
  bool DstUnsigned = ElemT == PT_Uint16;

  // Lanes are always 16-bit; gather them so the reduction below is untyped.
  SmallVector<uint16_t> AVals(NumElems), BVals(NumElems), Acc(NumElems);
  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
    for (unsigned I = 0; I != NumElems; ++I) {
      AVals[I] = (uint16_t)A.elem<T>(I).toAPSInt().getZExtValue();
      BVals[I] = (uint16_t)B.elem<T>(I).toAPSInt().getZExtValue();
      Acc[I] = (uint16_t)C.elem<T>(I).toAPSInt().getZExtValue();
    }
  });

  for (unsigned Lane = 0; Lane != NumElems; Lane += 16) {
    for (unsigned I = 0; I != 16; ++I) {
      uint16_t AVal = AVals[Lane + I], DVal = Acc[Lane + I];
      for (unsigned J = 0; J != 16; ++J) {
        // Seed the reduction with the accumulator bit, then fold in each
        // product term with the same operator (OR for vbmacor, XOR for
        // vbmacxor).
        unsigned Bit = (DVal >> J) & 1u;
        for (unsigned K = 0; K != 16; ++K) {
          unsigned Product = ((AVal >> K) & 1u) & ((BVals[Lane + K] >> J) & 1u);
          Bit = IsXor ? (Bit ^ Product) : (Bit | Product);
        }
        DVal = (DVal & ~(uint16_t(1) << J)) | (uint16_t(Bit) << J);
      }
      Acc[Lane + I] = DVal;
    }
  }

  INT_TYPE_SWITCH_NO_BOOL(ElemT, {
    for (unsigned I = 0; I != NumElems; ++I)
      Dst.elem<T>(I) = static_cast<T>(APSInt(APInt(16, Acc[I]), DstUnsigned));
  });
  Dst.initializeAllElements();
  return true;
}

static bool interp_builtin_ia32_cvt_scalar_to_int(InterpState &S, CodePtr OpPC,
                                                  const CallExpr *E) {
  Pointer SrcVecPtr = S.Stk.pop<Pointer>();
  const Floating &FloatElem = SrcVecPtr.elem<Floating>(0);

  unsigned BitWidth = S.getASTContext().getIntWidth(E->getType());
  bool IsUnsigned = E->getType()->isUnsignedIntegerType();

  llvm::APSInt IntResult(BitWidth, IsUnsigned);
  bool IsExact = false;
  // We only allow exact conversions so rounding mode does not matter for cvt*
  // and cvtt* builtins
  FloatElem.getAPFloat().convertToInteger(
      IntResult, llvm::APFloat::rmTowardZero, &IsExact);
  if (!IsExact)
    return false;

  pushInteger(S, IntResult, E->getType());
  return true;
}

static bool interp_builtin_ia32_cvt_vector_to_int(InterpState &S, CodePtr OpPC,
                                                  const CallExpr *E) {
  Pointer SrcVecPtr = S.Stk.pop<Pointer>();
  const Pointer &Dst = S.Stk.peek<Pointer>();

  unsigned NumSrcElems = SrcVecPtr.getNumElems();
  unsigned NumDstElems = Dst.getNumElems();

  if (NumSrcElems > NumDstElems)
    return false;

  QualType ElemType = Dst.getFieldDesc()->getElemQualType();
  unsigned BitWidth = S.getASTContext().getIntWidth(ElemType);
  bool IsUnsigned = ElemType->isUnsignedIntegerType();

  PrimType ElemT = *S.getContext().classify(ElemType);
  for (unsigned I = 0; I != NumSrcElems; ++I) {
    const Floating &FloatElem = SrcVecPtr.elem<Floating>(I);
    llvm::APSInt IntResult(BitWidth, IsUnsigned);

    bool IsExact = false;
    // We only allow exact conversions so rounding mode does not matter for
    // cvt* and cvtt* builtins
    FloatElem.getAPFloat().convertToInteger(
        IntResult, llvm::APFloat::rmTowardZero, &IsExact);
    if (!IsExact)
      return false;
    INT_TYPE_SWITCH_NO_BOOL(
        ElemT, { Dst.elem<T>(I) = T::from(IntResult.getZExtValue()); });
  }

  // Zero out remaining elements if the destination has more elements
  // (e.g., cvtpd2dq converting 2 doubles(_m128d) to 2 ints stored in _m128i).
  for (unsigned I = NumSrcElems; I != NumDstElems; ++I)
    INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(I) = T::from(0); });

  Dst.initializeAllElements();
  return true;
}

bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
                      uint32_t BuiltinID) {
  const ASTContext &ASTCtx = S.getASTContext();

  // BuiltinID is the raw ID baked into the bytecode. The "is constant
  // evaluated" gate needs the raw ID so that auxiliary-target IDs resolve into
  // the correct (aux-target) builtin records.
  if (!ASTCtx.BuiltinInfo.isConstantEvaluated(BuiltinID))
    return Invalid(S, OpPC);

  // Convert an auxiliary x86 target builtin ID to its canonical X86::BI* value
  // so the target-specific cases below (and the handlers they call) match. This
  // is a cheap integer operation (a single comparison for the common,
  // target-independent case); we deliberately avoid re-deriving the ID from the
  // call expression, which is comparatively slow.
  BuiltinID = ConvertBuiltinIDToX86BuiltinID(ASTCtx, BuiltinID);

  const InterpFrame *Frame = S.Current;
  switch (BuiltinID) {
  case Builtin::BI__builtin_is_constant_evaluated:
    return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_assume:
  case Builtin::BI__assume:
    return interp__builtin_assume(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_strcmp:
  case Builtin::BIstrcmp:
  case Builtin::BI__builtin_strncmp:
  case Builtin::BIstrncmp:
  case Builtin::BI__builtin_wcsncmp:
  case Builtin::BIwcsncmp:
  case Builtin::BI__builtin_wcscmp:
  case Builtin::BIwcscmp:
    return interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_strlen:
  case Builtin::BIstrlen:
  case Builtin::BI__builtin_wcslen:
  case Builtin::BIwcslen:
    return interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_nan:
  case Builtin::BI__builtin_nanf:
  case Builtin::BI__builtin_nanl:
  case Builtin::BI__builtin_nanf16:
  case Builtin::BI__builtin_nanf128:
    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);

  case Builtin::BI__builtin_nans:
  case Builtin::BI__builtin_nansf:
  case Builtin::BI__builtin_nansl:
  case Builtin::BI__builtin_nansf16:
  case Builtin::BI__builtin_nansf128:
    return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);

  case Builtin::BI__builtin_huge_val:
  case Builtin::BI__builtin_huge_valf:
  case Builtin::BI__builtin_huge_vall:
  case Builtin::BI__builtin_huge_valf16:
  case Builtin::BI__builtin_huge_valf128:
  case Builtin::BI__builtin_inf:
  case Builtin::BI__builtin_inff:
  case Builtin::BI__builtin_infl:
  case Builtin::BI__builtin_inff16:
  case Builtin::BI__builtin_inff128:
    return interp__builtin_inf(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_copysign:
  case Builtin::BI__builtin_copysignf:
  case Builtin::BI__builtin_copysignl:
  case Builtin::BI__builtin_copysignf128:
    return interp__builtin_copysign(S, OpPC, Frame);

  case Builtin::BI__builtin_fmin:
  case Builtin::BI__builtin_fminf:
  case Builtin::BI__builtin_fminl:
  case Builtin::BI__builtin_fminf16:
  case Builtin::BI__builtin_fminf128:
    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);

  case Builtin::BI__builtin_fminimum_num:
  case Builtin::BI__builtin_fminimum_numf:
  case Builtin::BI__builtin_fminimum_numl:
  case Builtin::BI__builtin_fminimum_numf16:
  case Builtin::BI__builtin_fminimum_numf128:
    return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);

  case Builtin::BI__builtin_fmax:
  case Builtin::BI__builtin_fmaxf:
  case Builtin::BI__builtin_fmaxl:
  case Builtin::BI__builtin_fmaxf16:
  case Builtin::BI__builtin_fmaxf128:
    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);

  case Builtin::BI__builtin_fmaximum_num:
  case Builtin::BI__builtin_fmaximum_numf:
  case Builtin::BI__builtin_fmaximum_numl:
  case Builtin::BI__builtin_fmaximum_numf16:
  case Builtin::BI__builtin_fmaximum_numf128:
    return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);

  case Builtin::BI__builtin_isnan:
    return interp__builtin_isnan(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_issignaling:
    return interp__builtin_issignaling(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_isinf:
    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call);

  case Builtin::BI__builtin_isinf_sign:
    return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call);

  case Builtin::BI__builtin_isfinite:
    return interp__builtin_isfinite(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_isnormal:
    return interp__builtin_isnormal(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_issubnormal:
    return interp__builtin_issubnormal(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_iszero:
    return interp__builtin_iszero(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_signbit:
  case Builtin::BI__builtin_signbitf:
  case Builtin::BI__builtin_signbitl:
    return interp__builtin_signbit(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_isgreater:
  case Builtin::BI__builtin_isgreaterequal:
  case Builtin::BI__builtin_isless:
  case Builtin::BI__builtin_islessequal:
  case Builtin::BI__builtin_islessgreater:
  case Builtin::BI__builtin_isunordered:
    return interp_floating_comparison(S, OpPC, Call, BuiltinID);

  case Builtin::BI__builtin_isfpclass:
    return interp__builtin_isfpclass(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_fpclassify:
    return interp__builtin_fpclassify(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_fabs:
  case Builtin::BI__builtin_fabsf:
  case Builtin::BI__builtin_fabsl:
  case Builtin::BI__builtin_fabsf128:
    return interp__builtin_fabs(S, OpPC, Frame);

  case Builtin::BI__builtin_abs:
  case Builtin::BI__builtin_labs:
  case Builtin::BI__builtin_llabs:
    return interp__builtin_abs(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_popcount:
  case Builtin::BI__builtin_popcountl:
  case Builtin::BI__builtin_popcountll:
  case Builtin::BI__builtin_popcountg:
  case Builtin::BI__popcnt16: // Microsoft variants of popcount
  case Builtin::BI__popcnt:
  case Builtin::BI__popcnt64:
    return interp__builtin_popcount(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_parity:
  case Builtin::BI__builtin_parityl:
  case Builtin::BI__builtin_parityll:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) {
          return APInt(Val.getBitWidth(), Val.popcount() % 2);
        });
  case Builtin::BI__builtin_clrsb:
  case Builtin::BI__builtin_clrsbl:
  case Builtin::BI__builtin_clrsbll:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) {
          return APInt(Val.getBitWidth(),
                       Val.getBitWidth() - Val.getSignificantBits());
        });
  case Builtin::BI__builtin_bitreverseg:
  case Builtin::BI__builtin_bitreverse8:
  case Builtin::BI__builtin_bitreverse16:
  case Builtin::BI__builtin_bitreverse32:
  case Builtin::BI__builtin_bitreverse64:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) { return Val.reverseBits(); });

  case Builtin::BI__builtin_classify_type:
    return interp__builtin_classify_type(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_expect:
  case Builtin::BI__builtin_expect_with_probability:
    return interp__builtin_expect(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_rotateleft8:
  case Builtin::BI__builtin_rotateleft16:
  case Builtin::BI__builtin_rotateleft32:
  case Builtin::BI__builtin_rotateleft64:
  case Builtin::BI__builtin_stdc_rotate_left:
  case Builtin::BIstdc_rotate_left_uc:
  case Builtin::BIstdc_rotate_left_us:
  case Builtin::BIstdc_rotate_left_ui:
  case Builtin::BIstdc_rotate_left_ul:
  case Builtin::BIstdc_rotate_left_ull:
  case Builtin::BI_rotl8: // Microsoft variants of rotate left
  case Builtin::BI_rotl16:
  case Builtin::BI_rotl:
  case Builtin::BI_lrotl:
  case Builtin::BI_rotl64:
  case Builtin::BI__builtin_rotateright8:
  case Builtin::BI__builtin_rotateright16:
  case Builtin::BI__builtin_rotateright32:
  case Builtin::BI__builtin_rotateright64:
  case Builtin::BI__builtin_stdc_rotate_right:
  case Builtin::BIstdc_rotate_right_uc:
  case Builtin::BIstdc_rotate_right_us:
  case Builtin::BIstdc_rotate_right_ui:
  case Builtin::BIstdc_rotate_right_ul:
  case Builtin::BIstdc_rotate_right_ull:
  case Builtin::BI_rotr8: // Microsoft variants of rotate right
  case Builtin::BI_rotr16:
  case Builtin::BI_rotr:
  case Builtin::BI_lrotr:
  case Builtin::BI_rotr64: {
    // Determine if this is a rotate right operation
    bool IsRotateRight;
    switch (BuiltinID) {
    case Builtin::BI__builtin_rotateright8:
    case Builtin::BI__builtin_rotateright16:
    case Builtin::BI__builtin_rotateright32:
    case Builtin::BI__builtin_rotateright64:
    case Builtin::BI__builtin_stdc_rotate_right:
    case Builtin::BIstdc_rotate_right_uc:
    case Builtin::BIstdc_rotate_right_us:
    case Builtin::BIstdc_rotate_right_ui:
    case Builtin::BIstdc_rotate_right_ul:
    case Builtin::BIstdc_rotate_right_ull:
    case Builtin::BI_rotr8:
    case Builtin::BI_rotr16:
    case Builtin::BI_rotr:
    case Builtin::BI_lrotr:
    case Builtin::BI_rotr64:
      IsRotateRight = true;
      break;
    default:
      IsRotateRight = false;
      break;
    }

    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [IsRotateRight](const APSInt &Value, APSInt Amount) {
          Amount = NormalizeRotateAmount(Value, Amount);
          return IsRotateRight ? Value.rotr(Amount.getZExtValue())
                               : Value.rotl(Amount.getZExtValue());
        });
  }

  case Builtin::BIstdc_leading_zeros_uc:
  case Builtin::BIstdc_leading_zeros_us:
  case Builtin::BIstdc_leading_zeros_ui:
  case Builtin::BIstdc_leading_zeros_ul:
  case Builtin::BIstdc_leading_zeros_ull:
  case Builtin::BI__builtin_stdc_leading_zeros: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.countl_zero());
        });
  }

  case Builtin::BIstdc_leading_ones_uc:
  case Builtin::BIstdc_leading_ones_us:
  case Builtin::BIstdc_leading_ones_ui:
  case Builtin::BIstdc_leading_ones_ul:
  case Builtin::BIstdc_leading_ones_ull:
  case Builtin::BI__builtin_stdc_leading_ones: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.countl_one());
        });
  }

  case Builtin::BIstdc_trailing_zeros_uc:
  case Builtin::BIstdc_trailing_zeros_us:
  case Builtin::BIstdc_trailing_zeros_ui:
  case Builtin::BIstdc_trailing_zeros_ul:
  case Builtin::BIstdc_trailing_zeros_ull:
  case Builtin::BI__builtin_stdc_trailing_zeros: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.countr_zero());
        });
  }

  case Builtin::BIstdc_trailing_ones_uc:
  case Builtin::BIstdc_trailing_ones_us:
  case Builtin::BIstdc_trailing_ones_ui:
  case Builtin::BIstdc_trailing_ones_ul:
  case Builtin::BIstdc_trailing_ones_ull:
  case Builtin::BI__builtin_stdc_trailing_ones: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.countr_one());
        });
  }

  case Builtin::BIstdc_first_leading_zero_uc:
  case Builtin::BIstdc_first_leading_zero_us:
  case Builtin::BIstdc_first_leading_zero_ui:
  case Builtin::BIstdc_first_leading_zero_ul:
  case Builtin::BIstdc_first_leading_zero_ull:
  case Builtin::BI__builtin_stdc_first_leading_zero: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countl_one() + 1);
        });
  }

  case Builtin::BIstdc_first_leading_one_uc:
  case Builtin::BIstdc_first_leading_one_us:
  case Builtin::BIstdc_first_leading_one_ui:
  case Builtin::BIstdc_first_leading_one_ul:
  case Builtin::BIstdc_first_leading_one_ull:
  case Builtin::BI__builtin_stdc_first_leading_one: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.isZero() ? 0 : Val.countl_zero() + 1);
        });
  }

  case Builtin::BIstdc_first_trailing_zero_uc:
  case Builtin::BIstdc_first_trailing_zero_us:
  case Builtin::BIstdc_first_trailing_zero_ui:
  case Builtin::BIstdc_first_trailing_zero_ul:
  case Builtin::BIstdc_first_trailing_zero_ull:
  case Builtin::BI__builtin_stdc_first_trailing_zero: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.isAllOnes() ? 0 : Val.countr_one() + 1);
        });
  }

  case Builtin::BIstdc_first_trailing_one_uc:
  case Builtin::BIstdc_first_trailing_one_us:
  case Builtin::BIstdc_first_trailing_one_ui:
  case Builtin::BIstdc_first_trailing_one_ul:
  case Builtin::BIstdc_first_trailing_one_ull:
  case Builtin::BI__builtin_stdc_first_trailing_one: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.isZero() ? 0 : Val.countr_zero() + 1);
        });
  }

  case Builtin::BIstdc_count_zeros_uc:
  case Builtin::BIstdc_count_zeros_us:
  case Builtin::BIstdc_count_zeros_ui:
  case Builtin::BIstdc_count_zeros_ul:
  case Builtin::BIstdc_count_zeros_ull:
  case Builtin::BI__builtin_stdc_count_zeros: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          unsigned BitWidth = Val.getBitWidth();
          return APInt(ResWidth, BitWidth - Val.popcount());
        });
  }

  case Builtin::BIstdc_count_ones_uc:
  case Builtin::BIstdc_count_ones_us:
  case Builtin::BIstdc_count_ones_ui:
  case Builtin::BIstdc_count_ones_ul:
  case Builtin::BIstdc_count_ones_ull:
  case Builtin::BI__builtin_stdc_count_ones: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.popcount());
        });
  }

  case Builtin::BIstdc_has_single_bit_uc:
  case Builtin::BIstdc_has_single_bit_us:
  case Builtin::BIstdc_has_single_bit_ui:
  case Builtin::BIstdc_has_single_bit_ul:
  case Builtin::BIstdc_has_single_bit_ull:
  case Builtin::BI__builtin_stdc_has_single_bit: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          return APInt(ResWidth, Val.popcount() == 1 ? 1 : 0);
        });
  }

  case Builtin::BIstdc_bit_width_uc:
  case Builtin::BIstdc_bit_width_us:
  case Builtin::BIstdc_bit_width_ui:
  case Builtin::BIstdc_bit_width_ul:
  case Builtin::BIstdc_bit_width_ull:
  case Builtin::BI__builtin_stdc_bit_width: {
    unsigned ResWidth = S.getASTContext().getIntWidth(Call->getType());
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [ResWidth](const APSInt &Val) {
          unsigned BitWidth = Val.getBitWidth();
          return APInt(ResWidth, BitWidth - Val.countl_zero());
        });
  }

  case Builtin::BIstdc_bit_floor_uc:
  case Builtin::BIstdc_bit_floor_us:
  case Builtin::BIstdc_bit_floor_ui:
  case Builtin::BIstdc_bit_floor_ul:
  case Builtin::BIstdc_bit_floor_ull:
  case Builtin::BI__builtin_stdc_bit_floor:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) {
          unsigned BitWidth = Val.getBitWidth();
          if (Val.isZero())
            return APInt::getZero(BitWidth);
          return APInt::getOneBitSet(BitWidth,
                                     BitWidth - Val.countl_zero() - 1);
        });

  case Builtin::BIstdc_bit_ceil_uc:
  case Builtin::BIstdc_bit_ceil_us:
  case Builtin::BIstdc_bit_ceil_ui:
  case Builtin::BIstdc_bit_ceil_ul:
  case Builtin::BIstdc_bit_ceil_ull:
  case Builtin::BI__builtin_stdc_bit_ceil:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) {
          unsigned BitWidth = Val.getBitWidth();
          if (Val.ule(1))
            return APInt(BitWidth, 1);
          APInt V = Val;
          APInt ValMinusOne = V - 1;
          unsigned LeadingZeros = ValMinusOne.countl_zero();
          if (LeadingZeros == 0)
            return APInt(BitWidth, 0); // overflows; wrap to 0
          return APInt::getOneBitSet(BitWidth, BitWidth - LeadingZeros);
        });

  case Builtin::BI__builtin_ffs:
  case Builtin::BI__builtin_ffsl:
  case Builtin::BI__builtin_ffsll:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Val) {
          return APInt(Val.getBitWidth(),
                       Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
        });

  case Builtin::BIaddressof:
  case Builtin::BI__addressof:
  case Builtin::BI__builtin_addressof:
    assert(isNoopBuiltin(BuiltinID));
    return interp__builtin_addressof(S, OpPC, Frame, Call);

  case Builtin::BIas_const:
  case Builtin::BIforward:
  case Builtin::BIforward_like:
  case Builtin::BImove:
  case Builtin::BImove_if_noexcept:
    assert(isNoopBuiltin(BuiltinID));
    return interp__builtin_move(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_eh_return_data_regno:
    return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_launder:
    assert(isNoopBuiltin(BuiltinID));
    return true;

  case Builtin::BI__builtin_add_overflow:
  case Builtin::BI__builtin_sub_overflow:
  case Builtin::BI__builtin_mul_overflow:
  case Builtin::BI__builtin_sadd_overflow:
  case Builtin::BI__builtin_uadd_overflow:
  case Builtin::BI__builtin_uaddl_overflow:
  case Builtin::BI__builtin_uaddll_overflow:
  case Builtin::BI__builtin_usub_overflow:
  case Builtin::BI__builtin_usubl_overflow:
  case Builtin::BI__builtin_usubll_overflow:
  case Builtin::BI__builtin_umul_overflow:
  case Builtin::BI__builtin_umull_overflow:
  case Builtin::BI__builtin_umulll_overflow:
  case Builtin::BI__builtin_saddl_overflow:
  case Builtin::BI__builtin_saddll_overflow:
  case Builtin::BI__builtin_ssub_overflow:
  case Builtin::BI__builtin_ssubl_overflow:
  case Builtin::BI__builtin_ssubll_overflow:
  case Builtin::BI__builtin_smul_overflow:
  case Builtin::BI__builtin_smull_overflow:
  case Builtin::BI__builtin_smulll_overflow:
    return interp__builtin_overflowop(S, OpPC, Call, BuiltinID);

  case Builtin::BI__builtin_addcb:
  case Builtin::BI__builtin_addcs:
  case Builtin::BI__builtin_addc:
  case Builtin::BI__builtin_addcl:
  case Builtin::BI__builtin_addcll:
  case Builtin::BI__builtin_subcb:
  case Builtin::BI__builtin_subcs:
  case Builtin::BI__builtin_subc:
  case Builtin::BI__builtin_subcl:
  case Builtin::BI__builtin_subcll:
    return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_clz:
  case Builtin::BI__builtin_clzl:
  case Builtin::BI__builtin_clzll:
  case Builtin::BI__builtin_clzs:
  case Builtin::BI__builtin_clzg:
  case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
  case Builtin::BI__lzcnt:
  case Builtin::BI__lzcnt64:
    return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_ctz:
  case Builtin::BI__builtin_ctzl:
  case Builtin::BI__builtin_ctzll:
  case Builtin::BI__builtin_ctzs:
  case Builtin::BI__builtin_ctzg:
    return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_elementwise_clzg:
  case Builtin::BI__builtin_elementwise_ctzg:
    return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
                                                   BuiltinID);
  case Builtin::BI__builtin_bswapg:
  case Builtin::BI__builtin_bswap16:
  case Builtin::BI__builtin_bswap32:
  case Builtin::BI__builtin_bswap64:
  case Builtin::BIstdc_memreverse8u8:
  case Builtin::BIstdc_memreverse8u16:
  case Builtin::BIstdc_memreverse8u32:
  case Builtin::BIstdc_memreverse8u64:
    return interp__builtin_bswap(S, OpPC, Frame, Call);

  case Builtin::BI__atomic_always_lock_free:
  case Builtin::BI__atomic_is_lock_free:
    return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__c11_atomic_is_lock_free:
    return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_complex:
    return interp__builtin_complex(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_is_aligned:
  case Builtin::BI__builtin_align_up:
  case Builtin::BI__builtin_align_down:
    return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_assume_aligned:
    return interp__builtin_assume_aligned(S, OpPC, Frame, Call);

  case clang::X86::BI__builtin_ia32_crc32qi:
    return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 1);
  case clang::X86::BI__builtin_ia32_crc32hi:
    return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 2);
  case clang::X86::BI__builtin_ia32_crc32si:
    return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 4);
  case clang::X86::BI__builtin_ia32_crc32di:
    return interp__builtin_ia32_crc32(S, OpPC, Frame, Call, 8);

  case clang::X86::BI__builtin_ia32_bextr_u32:
  case clang::X86::BI__builtin_ia32_bextr_u64:
  case clang::X86::BI__builtin_ia32_bextri_u32:
  case clang::X86::BI__builtin_ia32_bextri_u64:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
          unsigned BitWidth = Val.getBitWidth();
          uint64_t Shift = Idx.extractBitsAsZExtValue(8, 0);
          uint64_t Length = Idx.extractBitsAsZExtValue(8, 8);
          if (Length > BitWidth) {
            Length = BitWidth;
          }

          // Handle out of bounds cases.
          if (Length == 0 || Shift >= BitWidth)
            return APInt(BitWidth, 0);

          uint64_t Result = Val.getZExtValue() >> Shift;
          Result &= llvm::maskTrailingOnes<uint64_t>(Length);
          return APInt(BitWidth, Result);
        });

  case clang::X86::BI__builtin_ia32_bzhi_si:
  case clang::X86::BI__builtin_ia32_bzhi_di:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
          unsigned BitWidth = Val.getBitWidth();
          uint64_t Index = Idx.extractBitsAsZExtValue(8, 0);
          APSInt Result = Val;

          if (Index < BitWidth)
            Result.clearHighBits(BitWidth - Index);

          return Result;
        });

  case clang::X86::BI__builtin_ia32_ktestcqi:
  case clang::X86::BI__builtin_ia32_ktestchi:
  case clang::X86::BI__builtin_ia32_ktestcsi:
  case clang::X86::BI__builtin_ia32_ktestcdi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
          return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
        });

  case clang::X86::BI__builtin_ia32_ktestzqi:
  case clang::X86::BI__builtin_ia32_ktestzhi:
  case clang::X86::BI__builtin_ia32_ktestzsi:
  case clang::X86::BI__builtin_ia32_ktestzdi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
          return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
        });

  case clang::X86::BI__builtin_ia32_kortestcqi:
  case clang::X86::BI__builtin_ia32_kortestchi:
  case clang::X86::BI__builtin_ia32_kortestcsi:
  case clang::X86::BI__builtin_ia32_kortestcdi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
          return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
        });

  case clang::X86::BI__builtin_ia32_kortestzqi:
  case clang::X86::BI__builtin_ia32_kortestzhi:
  case clang::X86::BI__builtin_ia32_kortestzsi:
  case clang::X86::BI__builtin_ia32_kortestzdi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
          return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
        });

  case clang::X86::BI__builtin_ia32_kshiftliqi:
  case clang::X86::BI__builtin_ia32_kshiftlihi:
  case clang::X86::BI__builtin_ia32_kshiftlisi:
  case clang::X86::BI__builtin_ia32_kshiftlidi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          unsigned Amt = RHS.getZExtValue() & 0xFF;
          if (Amt >= LHS.getBitWidth())
            return APInt::getZero(LHS.getBitWidth());
          return LHS.shl(Amt);
        });

  case clang::X86::BI__builtin_ia32_kshiftriqi:
  case clang::X86::BI__builtin_ia32_kshiftrihi:
  case clang::X86::BI__builtin_ia32_kshiftrisi:
  case clang::X86::BI__builtin_ia32_kshiftridi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          unsigned Amt = RHS.getZExtValue() & 0xFF;
          if (Amt >= LHS.getBitWidth())
            return APInt::getZero(LHS.getBitWidth());
          return LHS.lshr(Amt);
        });

  case clang::X86::BI__builtin_ia32_lzcnt_u16:
  case clang::X86::BI__builtin_ia32_lzcnt_u32:
  case clang::X86::BI__builtin_ia32_lzcnt_u64:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) {
          return APInt(Src.getBitWidth(), Src.countLeadingZeros());
        });

  case clang::X86::BI__builtin_ia32_tzcnt_u16:
  case clang::X86::BI__builtin_ia32_tzcnt_u32:
  case clang::X86::BI__builtin_ia32_tzcnt_u64:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) {
          return APInt(Src.getBitWidth(), Src.countTrailingZeros());
        });

  case clang::X86::BI__builtin_ia32_addcarryx_u32:
  case clang::X86::BI__builtin_ia32_addcarryx_u64:
    return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
                                                   /*IsAdd=*/true);

  case clang::X86::BI__builtin_ia32_subborrow_u32:
  case clang::X86::BI__builtin_ia32_subborrow_u64:
    return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
                                                   /*IsAdd=*/false);

  case Builtin::BI__builtin_os_log_format_buffer_size:
    return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_ptrauth_string_discriminator:
    return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_infer_alloc_token:
    return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);

  case Builtin::BI__noop:
    pushInteger(S, 0, Call->getType());
    return true;

  case Builtin::BI__builtin_operator_new:
    return interp__builtin_operator_new(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_operator_delete:
    return interp__builtin_operator_delete(S, OpPC, Frame, Call);

  case Builtin::BI__arithmetic_fence:
    return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);

  case Builtin::BI__builtin_reduce_add:
  case Builtin::BI__builtin_reduce_mul:
  case Builtin::BI__builtin_reduce_and:
  case Builtin::BI__builtin_reduce_or:
  case Builtin::BI__builtin_reduce_xor:
  case Builtin::BI__builtin_reduce_min:
  case Builtin::BI__builtin_reduce_max:
    return interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID);

  case Builtin::BI__builtin_elementwise_popcount:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) {
          return APInt(Src.getBitWidth(), Src.popcount());
        });
  case Builtin::BI__builtin_elementwise_bitreverse:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) { return Src.reverseBits(); });

  case Builtin::BI__builtin_elementwise_abs:
    return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_memcpy:
  case Builtin::BImemcpy:
  case Builtin::BI__builtin_wmemcpy:
  case Builtin::BIwmemcpy:
  case Builtin::BI__builtin_memmove:
  case Builtin::BImemmove:
  case Builtin::BI__builtin_wmemmove:
  case Builtin::BIwmemmove:
    return interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BI__builtin_memcmp:
  case Builtin::BImemcmp:
  case Builtin::BI__builtin_bcmp:
  case Builtin::BIbcmp:
  case Builtin::BI__builtin_wmemcmp:
  case Builtin::BIwmemcmp:
    return interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID);

  case Builtin::BImemchr:
  case Builtin::BI__builtin_memchr:
  case Builtin::BIstrchr:
  case Builtin::BI__builtin_strchr:
  case Builtin::BIwmemchr:
  case Builtin::BI__builtin_wmemchr:
  case Builtin::BIwcschr:
  case Builtin::BI__builtin_wcschr:
  case Builtin::BI__builtin_char_memchr:
    return interp__builtin_memchr(S, OpPC, Call, BuiltinID);

  case Builtin::BI__builtin_object_size:
    return interp__builtin_object_size(S, OpPC, Frame, Call,
                                       /*IsDynamic=*/false);
  case Builtin::BI__builtin_dynamic_object_size:
    return interp__builtin_object_size(S, OpPC, Frame, Call,
                                       /*IsDynamic=*/true);

  case Builtin::BI__builtin_is_within_lifetime:
    return interp__builtin_is_within_lifetime(S, OpPC, Call);

  case Builtin::BI__builtin_elementwise_add_sat:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
        });

  case Builtin::BI__builtin_elementwise_sub_sat:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
        });

  case Builtin::BI__builtin_elementwise_pdep:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::pdep);

  case Builtin::BI__builtin_elementwise_pext:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::pext);

  case X86::BI__builtin_ia32_extract128i256:
  case X86::BI__builtin_ia32_vextractf128_pd256:
  case X86::BI__builtin_ia32_vextractf128_ps256:
  case X86::BI__builtin_ia32_vextractf128_si256:
    return interp__builtin_ia32_extract_vector(S, OpPC, Call, BuiltinID);

  case X86::BI__builtin_ia32_extractf32x4_256_mask:
  case X86::BI__builtin_ia32_extractf32x4_mask:
  case X86::BI__builtin_ia32_extractf32x8_mask:
  case X86::BI__builtin_ia32_extractf64x2_256_mask:
  case X86::BI__builtin_ia32_extractf64x2_512_mask:
  case X86::BI__builtin_ia32_extractf64x4_mask:
  case X86::BI__builtin_ia32_extracti32x4_256_mask:
  case X86::BI__builtin_ia32_extracti32x4_mask:
  case X86::BI__builtin_ia32_extracti32x8_mask:
  case X86::BI__builtin_ia32_extracti64x2_256_mask:
  case X86::BI__builtin_ia32_extracti64x2_512_mask:
  case X86::BI__builtin_ia32_extracti64x4_mask:
    return interp__builtin_ia32_extract_vector_masked(S, OpPC, Call, BuiltinID);

  case clang::X86::BI__builtin_ia32_pmulhrsw128:
  case clang::X86::BI__builtin_ia32_pmulhrsw256:
  case clang::X86::BI__builtin_ia32_pmulhrsw512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          return (llvm::APIntOps::mulsExtended(LHS, RHS).ashr(14) + 1)
              .extractBits(16, 1);
        });

  case clang::X86::BI__builtin_ia32_movmskps:
  case clang::X86::BI__builtin_ia32_movmskpd:
  case clang::X86::BI__builtin_ia32_pmovmskb128:
  case clang::X86::BI__builtin_ia32_pmovmskb256:
  case clang::X86::BI__builtin_ia32_movmskps256:
  case clang::X86::BI__builtin_ia32_movmskpd256: {
    return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
  }

  case X86::BI__builtin_ia32_psignb128:
  case X86::BI__builtin_ia32_psignb256:
  case X86::BI__builtin_ia32_psignw128:
  case X86::BI__builtin_ia32_psignw256:
  case X86::BI__builtin_ia32_psignd128:
  case X86::BI__builtin_ia32_psignd256:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APInt &AElem, const APInt &BElem) {
          if (BElem.isZero())
            return APInt::getZero(AElem.getBitWidth());
          if (BElem.isNegative())
            return -AElem;
          return AElem;
        });

  case clang::X86::BI__builtin_ia32_pavgb128:
  case clang::X86::BI__builtin_ia32_pavgw128:
  case clang::X86::BI__builtin_ia32_pavgb256:
  case clang::X86::BI__builtin_ia32_pavgw256:
  case clang::X86::BI__builtin_ia32_pavgb512:
  case clang::X86::BI__builtin_ia32_pavgw512:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::avgCeilU);

  case clang::X86::BI__builtin_ia32_pmaddubsw128:
  case clang::X86::BI__builtin_ia32_pmaddubsw256:
  case clang::X86::BI__builtin_ia32_pmaddubsw512:
    return interp__builtin_ia32_pmul(
        S, OpPC, Call,
        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
           const APSInt &HiRHS) {
          unsigned BitWidth = 2 * LoLHS.getBitWidth();
          return (LoLHS.zext(BitWidth) * LoRHS.sext(BitWidth))
              .sadd_sat((HiLHS.zext(BitWidth) * HiRHS.sext(BitWidth)));
        });

  case clang::X86::BI__builtin_ia32_pmaddwd128:
  case clang::X86::BI__builtin_ia32_pmaddwd256:
  case clang::X86::BI__builtin_ia32_pmaddwd512:
    return interp__builtin_ia32_pmul(
        S, OpPC, Call,
        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
           const APSInt &HiRHS) {
          unsigned BitWidth = 2 * LoLHS.getBitWidth();
          return (LoLHS.sext(BitWidth) * LoRHS.sext(BitWidth)) +
                 (HiLHS.sext(BitWidth) * HiRHS.sext(BitWidth));
        });

  case clang::X86::BI__builtin_ia32_psadbw128:
  case clang::X86::BI__builtin_ia32_psadbw256:
  case clang::X86::BI__builtin_ia32_psadbw512:
    return interp__builtin_ia32_psadbw(S, OpPC, Call);

  case clang::X86::BI__builtin_ia32_dbpsadbw128:
  case clang::X86::BI__builtin_ia32_dbpsadbw256:
  case clang::X86::BI__builtin_ia32_dbpsadbw512:
    return interp__builtin_ia32_dbpsadbw(S, OpPC, Call);

  case clang::X86::BI__builtin_ia32_mpsadbw128:
  case clang::X86::BI__builtin_ia32_mpsadbw256:
    return interp__builtin_ia32_mpsadbw(S, OpPC, Call);

  case clang::X86::BI__builtin_ia32_pmulhuw128:
  case clang::X86::BI__builtin_ia32_pmulhuw256:
  case clang::X86::BI__builtin_ia32_pmulhuw512:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::mulhu);

  case clang::X86::BI__builtin_ia32_pmulhw128:
  case clang::X86::BI__builtin_ia32_pmulhw256:
  case clang::X86::BI__builtin_ia32_pmulhw512:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::mulhs);

  case clang::X86::BI__builtin_ia32_psllv2di:
  case clang::X86::BI__builtin_ia32_psllv4di:
  case clang::X86::BI__builtin_ia32_psllv4si:
  case clang::X86::BI__builtin_ia32_psllv8di:
  case clang::X86::BI__builtin_ia32_psllv8hi:
  case clang::X86::BI__builtin_ia32_psllv8si:
  case clang::X86::BI__builtin_ia32_psllv16hi:
  case clang::X86::BI__builtin_ia32_psllv16si:
  case clang::X86::BI__builtin_ia32_psllv32hi:
  case clang::X86::BI__builtin_ia32_psllwi128:
  case clang::X86::BI__builtin_ia32_psllwi256:
  case clang::X86::BI__builtin_ia32_psllwi512:
  case clang::X86::BI__builtin_ia32_pslldi128:
  case clang::X86::BI__builtin_ia32_pslldi256:
  case clang::X86::BI__builtin_ia32_pslldi512:
  case clang::X86::BI__builtin_ia32_psllqi128:
  case clang::X86::BI__builtin_ia32_psllqi256:
  case clang::X86::BI__builtin_ia32_psllqi512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          if (RHS.uge(LHS.getBitWidth())) {
            return APInt::getZero(LHS.getBitWidth());
          }
          return LHS.shl(RHS.getZExtValue());
        });

  case clang::X86::BI__builtin_ia32_psrav4si:
  case clang::X86::BI__builtin_ia32_psrav8di:
  case clang::X86::BI__builtin_ia32_psrav8hi:
  case clang::X86::BI__builtin_ia32_psrav8si:
  case clang::X86::BI__builtin_ia32_psrav16hi:
  case clang::X86::BI__builtin_ia32_psrav16si:
  case clang::X86::BI__builtin_ia32_psrav32hi:
  case clang::X86::BI__builtin_ia32_psravq128:
  case clang::X86::BI__builtin_ia32_psravq256:
  case clang::X86::BI__builtin_ia32_psrawi128:
  case clang::X86::BI__builtin_ia32_psrawi256:
  case clang::X86::BI__builtin_ia32_psrawi512:
  case clang::X86::BI__builtin_ia32_psradi128:
  case clang::X86::BI__builtin_ia32_psradi256:
  case clang::X86::BI__builtin_ia32_psradi512:
  case clang::X86::BI__builtin_ia32_psraqi128:
  case clang::X86::BI__builtin_ia32_psraqi256:
  case clang::X86::BI__builtin_ia32_psraqi512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          if (RHS.uge(LHS.getBitWidth())) {
            return LHS.ashr(LHS.getBitWidth() - 1);
          }
          return LHS.ashr(RHS.getZExtValue());
        });

  case clang::X86::BI__builtin_ia32_psrlv2di:
  case clang::X86::BI__builtin_ia32_psrlv4di:
  case clang::X86::BI__builtin_ia32_psrlv4si:
  case clang::X86::BI__builtin_ia32_psrlv8di:
  case clang::X86::BI__builtin_ia32_psrlv8hi:
  case clang::X86::BI__builtin_ia32_psrlv8si:
  case clang::X86::BI__builtin_ia32_psrlv16hi:
  case clang::X86::BI__builtin_ia32_psrlv16si:
  case clang::X86::BI__builtin_ia32_psrlv32hi:
  case clang::X86::BI__builtin_ia32_psrlwi128:
  case clang::X86::BI__builtin_ia32_psrlwi256:
  case clang::X86::BI__builtin_ia32_psrlwi512:
  case clang::X86::BI__builtin_ia32_psrldi128:
  case clang::X86::BI__builtin_ia32_psrldi256:
  case clang::X86::BI__builtin_ia32_psrldi512:
  case clang::X86::BI__builtin_ia32_psrlqi128:
  case clang::X86::BI__builtin_ia32_psrlqi256:
  case clang::X86::BI__builtin_ia32_psrlqi512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
          if (RHS.uge(LHS.getBitWidth())) {
            return APInt::getZero(LHS.getBitWidth());
          }
          return LHS.lshr(RHS.getZExtValue());
        });
  case clang::X86::BI__builtin_ia32_packsswb128:
  case clang::X86::BI__builtin_ia32_packsswb256:
  case clang::X86::BI__builtin_ia32_packsswb512:
  case clang::X86::BI__builtin_ia32_packssdw128:
  case clang::X86::BI__builtin_ia32_packssdw256:
  case clang::X86::BI__builtin_ia32_packssdw512:
    return interp__builtin_ia32_pack(S, OpPC, Call, [](const APSInt &Src) {
      return APInt(Src).truncSSat(Src.getBitWidth() / 2);
    });
  case clang::X86::BI__builtin_ia32_packusdw128:
  case clang::X86::BI__builtin_ia32_packusdw256:
  case clang::X86::BI__builtin_ia32_packusdw512:
  case clang::X86::BI__builtin_ia32_packuswb128:
  case clang::X86::BI__builtin_ia32_packuswb256:
  case clang::X86::BI__builtin_ia32_packuswb512:
    return interp__builtin_ia32_pack(S, OpPC, Call, [](const APSInt &Src) {
      return APInt(Src).truncSSatU(Src.getBitWidth() / 2);
    });

  case clang::X86::BI__builtin_ia32_selectss_128:
  case clang::X86::BI__builtin_ia32_selectsd_128:
  case clang::X86::BI__builtin_ia32_selectsh_128:
  case clang::X86::BI__builtin_ia32_selectsbf_128:
    return interp__builtin_ia32_select_scalar(S, Call);
  case clang::X86::BI__builtin_ia32_vprotbi:
  case clang::X86::BI__builtin_ia32_vprotdi:
  case clang::X86::BI__builtin_ia32_vprotqi:
  case clang::X86::BI__builtin_ia32_vprotwi:
  case clang::X86::BI__builtin_ia32_prold128:
  case clang::X86::BI__builtin_ia32_prold256:
  case clang::X86::BI__builtin_ia32_prold512:
  case clang::X86::BI__builtin_ia32_prolq128:
  case clang::X86::BI__builtin_ia32_prolq256:
  case clang::X86::BI__builtin_ia32_prolq512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(RHS); });

  case clang::X86::BI__builtin_ia32_prord128:
  case clang::X86::BI__builtin_ia32_prord256:
  case clang::X86::BI__builtin_ia32_prord512:
  case clang::X86::BI__builtin_ia32_prorq128:
  case clang::X86::BI__builtin_ia32_prorq256:
  case clang::X86::BI__builtin_ia32_prorq512:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(RHS); });

  case Builtin::BI__builtin_elementwise_max:
  case Builtin::BI__builtin_elementwise_min:
    return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);

  case clang::X86::BI__builtin_ia32_phaddw128:
  case clang::X86::BI__builtin_ia32_phaddw256:
  case clang::X86::BI__builtin_ia32_phaddd128:
  case clang::X86::BI__builtin_ia32_phaddd256:
    return interp_builtin_horizontal_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
  case clang::X86::BI__builtin_ia32_phaddsw128:
  case clang::X86::BI__builtin_ia32_phaddsw256:
    return interp_builtin_horizontal_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
  case clang::X86::BI__builtin_ia32_phsubw128:
  case clang::X86::BI__builtin_ia32_phsubw256:
  case clang::X86::BI__builtin_ia32_phsubd128:
  case clang::X86::BI__builtin_ia32_phsubd256:
    return interp_builtin_horizontal_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
  case clang::X86::BI__builtin_ia32_phsubsw128:
  case clang::X86::BI__builtin_ia32_phsubsw256:
    return interp_builtin_horizontal_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
  case clang::X86::BI__builtin_ia32_haddpd:
  case clang::X86::BI__builtin_ia32_haddps:
  case clang::X86::BI__builtin_ia32_haddpd256:
  case clang::X86::BI__builtin_ia32_haddps256:
    return interp_builtin_horizontal_fp_binop(
        S, OpPC, Call,
        [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
          APFloat F = LHS;
          F.add(RHS, RM);
          return F;
        });
  case clang::X86::BI__builtin_ia32_hsubpd:
  case clang::X86::BI__builtin_ia32_hsubps:
  case clang::X86::BI__builtin_ia32_hsubpd256:
  case clang::X86::BI__builtin_ia32_hsubps256:
    return interp_builtin_horizontal_fp_binop(
        S, OpPC, Call,
        [](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
          APFloat F = LHS;
          F.subtract(RHS, RM);
          return F;
        });
  case clang::X86::BI__builtin_ia32_addsubpd:
  case clang::X86::BI__builtin_ia32_addsubps:
  case clang::X86::BI__builtin_ia32_addsubpd256:
  case clang::X86::BI__builtin_ia32_addsubps256:
    return interp__builtin_ia32_addsub(S, OpPC, Call);

  case clang::X86::BI__builtin_ia32_pmuldq128:
  case clang::X86::BI__builtin_ia32_pmuldq256:
  case clang::X86::BI__builtin_ia32_pmuldq512:
    return interp__builtin_ia32_pmul(
        S, OpPC, Call,
        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
           const APSInt &HiRHS) {
          return llvm::APIntOps::mulsExtended(LoLHS, LoRHS);
        });

  case clang::X86::BI__builtin_ia32_pmuludq128:
  case clang::X86::BI__builtin_ia32_pmuludq256:
  case clang::X86::BI__builtin_ia32_pmuludq512:
    return interp__builtin_ia32_pmul(
        S, OpPC, Call,
        [](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
           const APSInt &HiRHS) {
          return llvm::APIntOps::muluExtended(LoLHS, LoRHS);
        });

  case clang::X86::BI__builtin_ia32_pclmulqdq128:
  case clang::X86::BI__builtin_ia32_pclmulqdq256:
  case clang::X86::BI__builtin_ia32_pclmulqdq512:
    return interp__builtin_ia32_pclmulqdq(S, OpPC, Call);
  case Builtin::BI__builtin_elementwise_clmul:
    return interp__builtin_elementwise_int_binop(S, OpPC, Call,
                                                 llvm::APIntOps::clmul);

  case Builtin::BI__builtin_elementwise_fma:
    return interp__builtin_elementwise_triop_fp(
        S, OpPC, Call,
        [](const APFloat &X, const APFloat &Y, const APFloat &Z,
           llvm::RoundingMode RM) {
          APFloat F = X;
          F.fusedMultiplyAdd(Y, Z, RM);
          return F;
        });

  case X86::BI__builtin_ia32_vpmadd52luq128:
  case X86::BI__builtin_ia32_vpmadd52luq256:
  case X86::BI__builtin_ia32_vpmadd52luq512:
    return interp__builtin_elementwise_triop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
          return A + (B.trunc(52) * C.trunc(52)).zext(64);
        });
  case X86::BI__builtin_ia32_vpmadd52huq128:
  case X86::BI__builtin_ia32_vpmadd52huq256:
  case X86::BI__builtin_ia32_vpmadd52huq512:
    return interp__builtin_elementwise_triop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
          return A + llvm::APIntOps::mulhu(B.trunc(52), C.trunc(52)).zext(64);
        });

  case X86::BI__builtin_ia32_vpshldd128:
  case X86::BI__builtin_ia32_vpshldd256:
  case X86::BI__builtin_ia32_vpshldd512:
  case X86::BI__builtin_ia32_vpshldq128:
  case X86::BI__builtin_ia32_vpshldq256:
  case X86::BI__builtin_ia32_vpshldq512:
  case X86::BI__builtin_ia32_vpshldw128:
  case X86::BI__builtin_ia32_vpshldw256:
  case X86::BI__builtin_ia32_vpshldw512:
    return interp__builtin_elementwise_triop(
        S, OpPC, Call,
        [](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
          return llvm::APIntOps::fshl(Hi, Lo, Amt);
        });

  case X86::BI__builtin_ia32_vpshrdd128:
  case X86::BI__builtin_ia32_vpshrdd256:
  case X86::BI__builtin_ia32_vpshrdd512:
  case X86::BI__builtin_ia32_vpshrdq128:
  case X86::BI__builtin_ia32_vpshrdq256:
  case X86::BI__builtin_ia32_vpshrdq512:
  case X86::BI__builtin_ia32_vpshrdw128:
  case X86::BI__builtin_ia32_vpshrdw256:
  case X86::BI__builtin_ia32_vpshrdw512:
    // NOTE: Reversed Hi/Lo operands.
    return interp__builtin_elementwise_triop(
        S, OpPC, Call,
        [](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
          return llvm::APIntOps::fshr(Hi, Lo, Amt);
        });
  case X86::BI__builtin_ia32_vpconflictsi_128:
  case X86::BI__builtin_ia32_vpconflictsi_256:
  case X86::BI__builtin_ia32_vpconflictsi_512:
  case X86::BI__builtin_ia32_vpconflictdi_128:
  case X86::BI__builtin_ia32_vpconflictdi_256:
  case X86::BI__builtin_ia32_vpconflictdi_512:
    return interp__builtin_ia32_vpconflict(S, OpPC, Call);
  case X86::BI__builtin_ia32_compressdf128_mask:
  case X86::BI__builtin_ia32_compressdf256_mask:
  case X86::BI__builtin_ia32_compressdf512_mask:
  case X86::BI__builtin_ia32_compressdi128_mask:
  case X86::BI__builtin_ia32_compressdi256_mask:
  case X86::BI__builtin_ia32_compressdi512_mask:
  case X86::BI__builtin_ia32_compresshi128_mask:
  case X86::BI__builtin_ia32_compresshi256_mask:
  case X86::BI__builtin_ia32_compresshi512_mask:
  case X86::BI__builtin_ia32_compressqi128_mask:
  case X86::BI__builtin_ia32_compressqi256_mask:
  case X86::BI__builtin_ia32_compressqi512_mask:
  case X86::BI__builtin_ia32_compresssf128_mask:
  case X86::BI__builtin_ia32_compresssf256_mask:
  case X86::BI__builtin_ia32_compresssf512_mask:
  case X86::BI__builtin_ia32_compresssi128_mask:
  case X86::BI__builtin_ia32_compresssi256_mask:
  case X86::BI__builtin_ia32_compresssi512_mask: {
    unsigned NumElems =
        Call->getArg(0)->getType()->castAs<VectorType>()->getNumElements();
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [NumElems](unsigned DstIdx, const APInt &ShuffleMask) {
          APInt CompressMask = ShuffleMask.trunc(NumElems);
          if (DstIdx < CompressMask.popcount()) {
            while (DstIdx != 0) {
              CompressMask = CompressMask & (CompressMask - 1);
              DstIdx--;
            }
            return std::pair<unsigned, int>{
                0, static_cast<int>(CompressMask.countr_zero())};
          }
          return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
        });
  }
  case X86::BI__builtin_ia32_expanddf128_mask:
  case X86::BI__builtin_ia32_expanddf256_mask:
  case X86::BI__builtin_ia32_expanddf512_mask:
  case X86::BI__builtin_ia32_expanddi128_mask:
  case X86::BI__builtin_ia32_expanddi256_mask:
  case X86::BI__builtin_ia32_expanddi512_mask:
  case X86::BI__builtin_ia32_expandhi128_mask:
  case X86::BI__builtin_ia32_expandhi256_mask:
  case X86::BI__builtin_ia32_expandhi512_mask:
  case X86::BI__builtin_ia32_expandqi128_mask:
  case X86::BI__builtin_ia32_expandqi256_mask:
  case X86::BI__builtin_ia32_expandqi512_mask:
  case X86::BI__builtin_ia32_expandsf128_mask:
  case X86::BI__builtin_ia32_expandsf256_mask:
  case X86::BI__builtin_ia32_expandsf512_mask:
  case X86::BI__builtin_ia32_expandsi128_mask:
  case X86::BI__builtin_ia32_expandsi256_mask:
  case X86::BI__builtin_ia32_expandsi512_mask: {
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, const APInt &ShuffleMask) {
          // Trunc to the sub-mask for the dst index and count the number of
          // src elements used prior to that.
          APInt ExpandMask = ShuffleMask.trunc(DstIdx + 1);
          if (ExpandMask[DstIdx]) {
            int SrcIdx = ExpandMask.popcount() - 1;
            return std::pair<unsigned, int>{0, SrcIdx};
          }
          return std::pair<unsigned, int>{1, static_cast<int>(DstIdx)};
        });
  }
  case clang::X86::BI__builtin_ia32_blendpd:
  case clang::X86::BI__builtin_ia32_blendpd256:
  case clang::X86::BI__builtin_ia32_blendps:
  case clang::X86::BI__builtin_ia32_blendps256:
  case clang::X86::BI__builtin_ia32_pblendw128:
  case clang::X86::BI__builtin_ia32_pblendw256:
  case clang::X86::BI__builtin_ia32_pblendd128:
  case clang::X86::BI__builtin_ia32_pblendd256:
    return interp__builtin_ia32_shuffle_generic(
      S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
        // Bit index for mask.
        unsigned MaskBit = (ShuffleMask >> (DstIdx % 8)) & 0x1;
        unsigned SrcVecIdx = MaskBit ? 1 : 0;  // 1 = TrueVec, 0 = FalseVec
        return std::pair<unsigned, int>{SrcVecIdx, static_cast<int>(DstIdx)};
      });


  case clang::X86::BI__builtin_ia32_blendvpd:
  case clang::X86::BI__builtin_ia32_blendvpd256:
  case clang::X86::BI__builtin_ia32_blendvps:
  case clang::X86::BI__builtin_ia32_blendvps256:
    return interp__builtin_elementwise_triop_fp(
        S, OpPC, Call,
        [](const APFloat &F, const APFloat &T, const APFloat &C,
           llvm::RoundingMode) { return C.isNegative() ? T : F; });

  case clang::X86::BI__builtin_ia32_pblendvb128:
  case clang::X86::BI__builtin_ia32_pblendvb256:
    return interp__builtin_elementwise_triop(
        S, OpPC, Call, [](const APSInt &F, const APSInt &T, const APSInt &C) {
          return ((APInt)C).isNegative() ? T : F;
        });
  case X86::BI__builtin_ia32_ptestz128:
  case X86::BI__builtin_ia32_ptestz256:
  case X86::BI__builtin_ia32_vtestzps:
  case X86::BI__builtin_ia32_vtestzps256:
  case X86::BI__builtin_ia32_vtestzpd:
  case X86::BI__builtin_ia32_vtestzpd256:
    return interp__builtin_ia32_test_op(
        S, OpPC, Call,
        [](const APInt &A, const APInt &B) { return (A & B) == 0; });
  case X86::BI__builtin_ia32_ptestc128:
  case X86::BI__builtin_ia32_ptestc256:
  case X86::BI__builtin_ia32_vtestcps:
  case X86::BI__builtin_ia32_vtestcps256:
  case X86::BI__builtin_ia32_vtestcpd:
  case X86::BI__builtin_ia32_vtestcpd256:
    return interp__builtin_ia32_test_op(
        S, OpPC, Call,
        [](const APInt &A, const APInt &B) { return (~A & B) == 0; });
  case X86::BI__builtin_ia32_ptestnzc128:
  case X86::BI__builtin_ia32_ptestnzc256:
  case X86::BI__builtin_ia32_vtestnzcps:
  case X86::BI__builtin_ia32_vtestnzcps256:
  case X86::BI__builtin_ia32_vtestnzcpd:
  case X86::BI__builtin_ia32_vtestnzcpd256:
    return interp__builtin_ia32_test_op(
        S, OpPC, Call, [](const APInt &A, const APInt &B) {
          return ((A & B) != 0) && ((~A & B) != 0);
        });
  case X86::BI__builtin_ia32_selectb_128:
  case X86::BI__builtin_ia32_selectb_256:
  case X86::BI__builtin_ia32_selectb_512:
  case X86::BI__builtin_ia32_selectw_128:
  case X86::BI__builtin_ia32_selectw_256:
  case X86::BI__builtin_ia32_selectw_512:
  case X86::BI__builtin_ia32_selectd_128:
  case X86::BI__builtin_ia32_selectd_256:
  case X86::BI__builtin_ia32_selectd_512:
  case X86::BI__builtin_ia32_selectq_128:
  case X86::BI__builtin_ia32_selectq_256:
  case X86::BI__builtin_ia32_selectq_512:
  case X86::BI__builtin_ia32_selectph_128:
  case X86::BI__builtin_ia32_selectph_256:
  case X86::BI__builtin_ia32_selectph_512:
  case X86::BI__builtin_ia32_selectpbf_128:
  case X86::BI__builtin_ia32_selectpbf_256:
  case X86::BI__builtin_ia32_selectpbf_512:
  case X86::BI__builtin_ia32_selectps_128:
  case X86::BI__builtin_ia32_selectps_256:
  case X86::BI__builtin_ia32_selectps_512:
  case X86::BI__builtin_ia32_selectpd_128:
  case X86::BI__builtin_ia32_selectpd_256:
  case X86::BI__builtin_ia32_selectpd_512:
    return interp__builtin_ia32_select(S, OpPC, Call);

  case X86::BI__builtin_ia32_shufps:
  case X86::BI__builtin_ia32_shufps256:
  case X86::BI__builtin_ia32_shufps512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned NumElemPerLane = 4;
          unsigned NumSelectableElems = NumElemPerLane / 2;
          unsigned BitsPerElem = 2;
          unsigned IndexMask = 0x3;
          unsigned MaskBits = 8;
          unsigned Lane = DstIdx / NumElemPerLane;
          unsigned ElemInLane = DstIdx % NumElemPerLane;
          unsigned LaneOffset = Lane * NumElemPerLane;
          unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
          unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
          return std::pair<unsigned, int>{SrcIdx,
                                          static_cast<int>(LaneOffset + Index)};
        });
  case X86::BI__builtin_ia32_shufpd:
  case X86::BI__builtin_ia32_shufpd256:
  case X86::BI__builtin_ia32_shufpd512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned NumElemPerLane = 2;
          unsigned NumSelectableElems = NumElemPerLane / 2;
          unsigned BitsPerElem = 1;
          unsigned IndexMask = 0x1;
          unsigned MaskBits = 8;
          unsigned Lane = DstIdx / NumElemPerLane;
          unsigned ElemInLane = DstIdx % NumElemPerLane;
          unsigned LaneOffset = Lane * NumElemPerLane;
          unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
          unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
          return std::pair<unsigned, int>{SrcIdx,
                                          static_cast<int>(LaneOffset + Index)};
        });

  case X86::BI__builtin_ia32_vgf2p8affineinvqb_v16qi:
  case X86::BI__builtin_ia32_vgf2p8affineinvqb_v32qi:
  case X86::BI__builtin_ia32_vgf2p8affineinvqb_v64qi:
    return interp__builtin_ia32_gfni_affine(S, OpPC, Call, true);
  case X86::BI__builtin_ia32_vgf2p8affineqb_v16qi:
  case X86::BI__builtin_ia32_vgf2p8affineqb_v32qi:
  case X86::BI__builtin_ia32_vgf2p8affineqb_v64qi:
    return interp__builtin_ia32_gfni_affine(S, OpPC, Call, false);

  case X86::BI__builtin_ia32_vgf2p8mulb_v16qi:
  case X86::BI__builtin_ia32_vgf2p8mulb_v32qi:
  case X86::BI__builtin_ia32_vgf2p8mulb_v64qi:
    return interp__builtin_ia32_gfni_mul(S, OpPC, Call);

  case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
  case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
    return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/false);
  case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
  case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
    return interp__builtin_ia32_bmac(S, OpPC, Call, /*IsXor=*/true);

  case X86::BI__builtin_ia32_insertps128:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned Mask) {
          // Bits [3:0]: zero mask - if bit is set, zero this element
          if ((Mask & (1 << DstIdx)) != 0) {
            return std::pair<unsigned, int>{0, -1};
          }
          // Bits [7:6]: select element from source vector Y (0-3)
          // Bits [5:4]: select destination position (0-3)
          unsigned SrcElem = (Mask >> 6) & 0x3;
          unsigned DstElem = (Mask >> 4) & 0x3;
          if (DstIdx == DstElem) {
            // Insert element from source vector (B) at this position
            return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
          } else {
            // Copy from destination vector (A)
            return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
          }
        });
  case X86::BI__builtin_ia32_permvarsi256:
  case X86::BI__builtin_ia32_permvarsf256:
  case X86::BI__builtin_ia32_permvardf512:
  case X86::BI__builtin_ia32_permvardi512:
  case X86::BI__builtin_ia32_permvarhi128:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x7;
          return std::pair<unsigned, int>{0, Offset};
        });
  case X86::BI__builtin_ia32_permvarqi128:
  case X86::BI__builtin_ia32_permvarhi256:
  case X86::BI__builtin_ia32_permvarsi512:
  case X86::BI__builtin_ia32_permvarsf512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0xF;
          return std::pair<unsigned, int>{0, Offset};
        });
  case X86::BI__builtin_ia32_permvardi256:
  case X86::BI__builtin_ia32_permvardf256:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x3;
          return std::pair<unsigned, int>{0, Offset};
        });
  case X86::BI__builtin_ia32_permvarqi256:
  case X86::BI__builtin_ia32_permvarhi512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x1F;
          return std::pair<unsigned, int>{0, Offset};
        });
  case X86::BI__builtin_ia32_permvarqi512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x3F;
          return std::pair<unsigned, int>{0, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2varq128:
  case X86::BI__builtin_ia32_vpermi2varpd128:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x1;
          unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2vard128:
  case X86::BI__builtin_ia32_vpermi2varps128:
  case X86::BI__builtin_ia32_vpermi2varq256:
  case X86::BI__builtin_ia32_vpermi2varpd256:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x3;
          unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2varhi128:
  case X86::BI__builtin_ia32_vpermi2vard256:
  case X86::BI__builtin_ia32_vpermi2varps256:
  case X86::BI__builtin_ia32_vpermi2varq512:
  case X86::BI__builtin_ia32_vpermi2varpd512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x7;
          unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2varqi128:
  case X86::BI__builtin_ia32_vpermi2varhi256:
  case X86::BI__builtin_ia32_vpermi2vard512:
  case X86::BI__builtin_ia32_vpermi2varps512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0xF;
          unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2varqi256:
  case X86::BI__builtin_ia32_vpermi2varhi512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x1F;
          unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vpermi2varqi512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          int Offset = ShuffleMask & 0x3F;
          unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
          return std::pair<unsigned, int>{SrcIdx, Offset};
        });
  case X86::BI__builtin_ia32_vperm2f128_pd256:
  case X86::BI__builtin_ia32_vperm2f128_ps256:
  case X86::BI__builtin_ia32_vperm2f128_si256:
  case X86::BI__builtin_ia32_permti256: {
    unsigned NumElements =
        Call->getArg(0)->getType()->castAs<VectorType>()->getNumElements();
    unsigned PreservedBitsCnt = NumElements >> 2;
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call,
        [PreservedBitsCnt](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned ControlBitsCnt = DstIdx >> PreservedBitsCnt << 2;
          unsigned ControlBits = ShuffleMask >> ControlBitsCnt;

          if (ControlBits & 0b1000)
            return std::make_pair(0u, -1);

          unsigned SrcVecIdx = (ControlBits & 0b10) >> 1;
          unsigned PreservedBitsMask = (1 << PreservedBitsCnt) - 1;
          int SrcIdx = ((ControlBits & 0b1) << PreservedBitsCnt) |
                       (DstIdx & PreservedBitsMask);
          return std::make_pair(SrcVecIdx, SrcIdx);
        });
  }
  case X86::BI__builtin_ia32_pshufb128:
  case X86::BI__builtin_ia32_pshufb256:
  case X86::BI__builtin_ia32_pshufb512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
          if (Ctlb & 0x80)
            return std::make_pair(0, -1);

          unsigned LaneBase = (DstIdx / 16) * 16;
          unsigned SrcOffset = Ctlb & 0x0F;
          unsigned SrcIdx = LaneBase + SrcOffset;
          return std::make_pair(0, static_cast<int>(SrcIdx));
        });

  case X86::BI__builtin_ia32_pshuflw:
  case X86::BI__builtin_ia32_pshuflw256:
  case X86::BI__builtin_ia32_pshuflw512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned LaneBase = (DstIdx / 8) * 8;
          unsigned LaneIdx = DstIdx % 8;
          if (LaneIdx < 4) {
            unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
            return std::make_pair(0, static_cast<int>(LaneBase + Sel));
          }

          return std::make_pair(0, static_cast<int>(DstIdx));
        });

  case X86::BI__builtin_ia32_pshufhw:
  case X86::BI__builtin_ia32_pshufhw256:
  case X86::BI__builtin_ia32_pshufhw512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned LaneBase = (DstIdx / 8) * 8;
          unsigned LaneIdx = DstIdx % 8;
          if (LaneIdx >= 4) {
            unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
            return std::make_pair(0, static_cast<int>(LaneBase + 4 + Sel));
          }

          return std::make_pair(0, static_cast<int>(DstIdx));
        });

  case X86::BI__builtin_ia32_pshufd:
  case X86::BI__builtin_ia32_pshufd256:
  case X86::BI__builtin_ia32_pshufd512:
  case X86::BI__builtin_ia32_vpermilps:
  case X86::BI__builtin_ia32_vpermilps256:
  case X86::BI__builtin_ia32_vpermilps512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned LaneBase = (DstIdx / 4) * 4;
          unsigned LaneIdx = DstIdx % 4;
          unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
          return std::make_pair(0, static_cast<int>(LaneBase + Sel));
        });

  case X86::BI__builtin_ia32_vpermilvarpd:
  case X86::BI__builtin_ia32_vpermilvarpd256:
  case X86::BI__builtin_ia32_vpermilvarpd512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned NumElemPerLane = 2;
          unsigned Lane = DstIdx / NumElemPerLane;
          unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
          return std::make_pair(
              0, static_cast<int>(Lane * NumElemPerLane + Offset));
        });

  case X86::BI__builtin_ia32_vpermilvarps:
  case X86::BI__builtin_ia32_vpermilvarps256:
  case X86::BI__builtin_ia32_vpermilvarps512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
          unsigned NumElemPerLane = 4;
          unsigned Lane = DstIdx / NumElemPerLane;
          unsigned Offset = ShuffleMask & 0b11;
          return std::make_pair(
              0, static_cast<int>(Lane * NumElemPerLane + Offset));
        });

  case X86::BI__builtin_ia32_vpermilpd:
  case X86::BI__builtin_ia32_vpermilpd256:
  case X86::BI__builtin_ia32_vpermilpd512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {
          unsigned NumElemPerLane = 2;
          unsigned BitsPerElem = 1;
          unsigned MaskBits = 8;
          unsigned IndexMask = 0x1;
          unsigned Lane = DstIdx / NumElemPerLane;
          unsigned LaneOffset = Lane * NumElemPerLane;
          unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
          unsigned Index = (Control >> BitIndex) & IndexMask;
          return std::make_pair(0, static_cast<int>(LaneOffset + Index));
        });

  case X86::BI__builtin_ia32_permdf256:
  case X86::BI__builtin_ia32_permdi256:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {
          // permute4x64 operates on 4 64-bit elements
          // For element i (0-3), extract bits [2*i+1:2*i] from Control
          unsigned Index = (Control >> (2 * DstIdx)) & 0x3;
          return std::make_pair(0, static_cast<int>(Index));
        });

  case X86::BI__builtin_ia32_vpmultishiftqb128:
  case X86::BI__builtin_ia32_vpmultishiftqb256:
  case X86::BI__builtin_ia32_vpmultishiftqb512:
    return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
  case X86::BI__builtin_ia32_kandqi:
  case X86::BI__builtin_ia32_kandhi:
  case X86::BI__builtin_ia32_kandsi:
  case X86::BI__builtin_ia32_kanddi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });

  case X86::BI__builtin_ia32_kandnqi:
  case X86::BI__builtin_ia32_kandnhi:
  case X86::BI__builtin_ia32_kandnsi:
  case X86::BI__builtin_ia32_kandndi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });

  case X86::BI__builtin_ia32_korqi:
  case X86::BI__builtin_ia32_korhi:
  case X86::BI__builtin_ia32_korsi:
  case X86::BI__builtin_ia32_kordi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });

  case X86::BI__builtin_ia32_kxnorqi:
  case X86::BI__builtin_ia32_kxnorhi:
  case X86::BI__builtin_ia32_kxnorsi:
  case X86::BI__builtin_ia32_kxnordi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });

  case X86::BI__builtin_ia32_kxorqi:
  case X86::BI__builtin_ia32_kxorhi:
  case X86::BI__builtin_ia32_kxorsi:
  case X86::BI__builtin_ia32_kxordi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });

  case X86::BI__builtin_ia32_knotqi:
  case X86::BI__builtin_ia32_knothi:
  case X86::BI__builtin_ia32_knotsi:
  case X86::BI__builtin_ia32_knotdi:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) { return ~Src; });

  case X86::BI__builtin_ia32_kaddqi:
  case X86::BI__builtin_ia32_kaddhi:
  case X86::BI__builtin_ia32_kaddsi:
  case X86::BI__builtin_ia32_kadddi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call,
        [](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });

  case X86::BI__builtin_ia32_kmovb:
  case X86::BI__builtin_ia32_kmovw:
  case X86::BI__builtin_ia32_kmovd:
  case X86::BI__builtin_ia32_kmovq:
    return interp__builtin_elementwise_int_unaryop(
        S, OpPC, Call, [](const APSInt &Src) { return Src; });

  case X86::BI__builtin_ia32_kunpckhi:
  case X86::BI__builtin_ia32_kunpckdi:
  case X86::BI__builtin_ia32_kunpcksi:
    return interp__builtin_elementwise_int_binop(
        S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
          // Generic kunpack: extract lower half of each operand and concatenate
          // Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
          unsigned BW = A.getBitWidth();
          return APSInt(A.trunc(BW / 2).concat(B.trunc(BW / 2)),
                        A.isUnsigned());
        });

  case X86::BI__builtin_ia32_phminposuw128:
    return interp__builtin_ia32_phminposuw(S, OpPC, Call);

  case X86::BI__builtin_ia32_psraq128:
  case X86::BI__builtin_ia32_psraq256:
  case X86::BI__builtin_ia32_psraq512:
  case X86::BI__builtin_ia32_psrad128:
  case X86::BI__builtin_ia32_psrad256:
  case X86::BI__builtin_ia32_psrad512:
  case X86::BI__builtin_ia32_psraw128:
  case X86::BI__builtin_ia32_psraw256:
  case X86::BI__builtin_ia32_psraw512:
    return interp__builtin_ia32_shift_with_count(
        S, OpPC, Call,
        [](const APInt &Elt, uint64_t Count) { return Elt.ashr(Count); },
        [](const APInt &Elt, unsigned Width) { return Elt.ashr(Width - 1); });

  case X86::BI__builtin_ia32_psllq128:
  case X86::BI__builtin_ia32_psllq256:
  case X86::BI__builtin_ia32_psllq512:
  case X86::BI__builtin_ia32_pslld128:
  case X86::BI__builtin_ia32_pslld256:
  case X86::BI__builtin_ia32_pslld512:
  case X86::BI__builtin_ia32_psllw128:
  case X86::BI__builtin_ia32_psllw256:
  case X86::BI__builtin_ia32_psllw512:
    return interp__builtin_ia32_shift_with_count(
        S, OpPC, Call,
        [](const APInt &Elt, uint64_t Count) { return Elt.shl(Count); },
        [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });

  case X86::BI__builtin_ia32_psrlq128:
  case X86::BI__builtin_ia32_psrlq256:
  case X86::BI__builtin_ia32_psrlq512:
  case X86::BI__builtin_ia32_psrld128:
  case X86::BI__builtin_ia32_psrld256:
  case X86::BI__builtin_ia32_psrld512:
  case X86::BI__builtin_ia32_psrlw128:
  case X86::BI__builtin_ia32_psrlw256:
  case X86::BI__builtin_ia32_psrlw512:
    return interp__builtin_ia32_shift_with_count(
        S, OpPC, Call,
        [](const APInt &Elt, uint64_t Count) { return Elt.lshr(Count); },
        [](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });

  case X86::BI__builtin_ia32_pternlogd128_mask:
  case X86::BI__builtin_ia32_pternlogd256_mask:
  case X86::BI__builtin_ia32_pternlogd512_mask:
  case X86::BI__builtin_ia32_pternlogq128_mask:
  case X86::BI__builtin_ia32_pternlogq256_mask:
  case X86::BI__builtin_ia32_pternlogq512_mask:
    return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
  case X86::BI__builtin_ia32_pternlogd128_maskz:
  case X86::BI__builtin_ia32_pternlogd256_maskz:
  case X86::BI__builtin_ia32_pternlogd512_maskz:
  case X86::BI__builtin_ia32_pternlogq128_maskz:
  case X86::BI__builtin_ia32_pternlogq256_maskz:
  case X86::BI__builtin_ia32_pternlogq512_maskz:
    return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
  case Builtin::BI__builtin_elementwise_fshl:
    return interp__builtin_elementwise_triop(S, OpPC, Call,
                                             llvm::APIntOps::fshl);
  case Builtin::BI__builtin_elementwise_fshr:
    return interp__builtin_elementwise_triop(S, OpPC, Call,
                                             llvm::APIntOps::fshr);

  case X86::BI__builtin_ia32_shuf_f32x4_256:
  case X86::BI__builtin_ia32_shuf_i32x4_256:
  case X86::BI__builtin_ia32_shuf_f64x2_256:
  case X86::BI__builtin_ia32_shuf_i64x2_256:
  case X86::BI__builtin_ia32_shuf_f32x4:
  case X86::BI__builtin_ia32_shuf_i32x4:
  case X86::BI__builtin_ia32_shuf_f64x2:
  case X86::BI__builtin_ia32_shuf_i64x2: {
    // Destination and sources A, B all have the same type.
    QualType VecQT = Call->getArg(0)->getType();
    const auto *VecT = VecQT->castAs<VectorType>();
    unsigned NumElems = VecT->getNumElements();
    unsigned ElemBits = S.getASTContext().getTypeSize(VecT->getElementType());
    unsigned LaneBits = 128u;
    unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
    unsigned NumElemsPerLane = LaneBits / ElemBits;

    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call,
        [NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
          // DstIdx determines source. ShuffleMask selects lane in source.
          unsigned BitsPerElem = NumLanes / 2;
          unsigned IndexMask = (1u << BitsPerElem) - 1;
          unsigned Lane = DstIdx / NumElemsPerLane;
          unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
          unsigned BitIdx = BitsPerElem * Lane;
          unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
          unsigned ElemInLane = DstIdx % NumElemsPerLane;
          unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
          return std::pair<unsigned, int>{SrcIdx, IdxToPick};
        });
  }

  case X86::BI__builtin_ia32_insertf32x4_256:
  case X86::BI__builtin_ia32_inserti32x4_256:
  case X86::BI__builtin_ia32_insertf64x2_256:
  case X86::BI__builtin_ia32_inserti64x2_256:
  case X86::BI__builtin_ia32_insertf32x4:
  case X86::BI__builtin_ia32_inserti32x4:
  case X86::BI__builtin_ia32_insertf64x2_512:
  case X86::BI__builtin_ia32_inserti64x2_512:
  case X86::BI__builtin_ia32_insertf32x8:
  case X86::BI__builtin_ia32_inserti32x8:
  case X86::BI__builtin_ia32_insertf64x4:
  case X86::BI__builtin_ia32_inserti64x4:
  case X86::BI__builtin_ia32_vinsertf128_ps256:
  case X86::BI__builtin_ia32_vinsertf128_pd256:
  case X86::BI__builtin_ia32_vinsertf128_si256:
  case X86::BI__builtin_ia32_insert128i256:
    return interp__builtin_ia32_insert_subvector(S, OpPC, Call, BuiltinID);

  case clang::X86::BI__builtin_ia32_vcvtps2ph:
  case clang::X86::BI__builtin_ia32_vcvtps2ph256:
    return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);

  case X86::BI__builtin_ia32_vec_ext_v4hi:
  case X86::BI__builtin_ia32_vec_ext_v16qi:
  case X86::BI__builtin_ia32_vec_ext_v8hi:
  case X86::BI__builtin_ia32_vec_ext_v4si:
  case X86::BI__builtin_ia32_vec_ext_v2di:
  case X86::BI__builtin_ia32_vec_ext_v32qi:
  case X86::BI__builtin_ia32_vec_ext_v16hi:
  case X86::BI__builtin_ia32_vec_ext_v8si:
  case X86::BI__builtin_ia32_vec_ext_v4di:
  case X86::BI__builtin_ia32_vec_ext_v4sf:
    return interp__builtin_ia32_vec_ext(S, OpPC, Call, BuiltinID);

  case X86::BI__builtin_ia32_vec_set_v4hi:
  case X86::BI__builtin_ia32_vec_set_v16qi:
  case X86::BI__builtin_ia32_vec_set_v8hi:
  case X86::BI__builtin_ia32_vec_set_v4si:
  case X86::BI__builtin_ia32_vec_set_v2di:
  case X86::BI__builtin_ia32_vec_set_v32qi:
  case X86::BI__builtin_ia32_vec_set_v16hi:
  case X86::BI__builtin_ia32_vec_set_v8si:
  case X86::BI__builtin_ia32_vec_set_v4di:
    return interp__builtin_ia32_vec_set(S, OpPC, Call, BuiltinID);

  case X86::BI__builtin_ia32_cvtb2mask128:
  case X86::BI__builtin_ia32_cvtb2mask256:
  case X86::BI__builtin_ia32_cvtb2mask512:
  case X86::BI__builtin_ia32_cvtw2mask128:
  case X86::BI__builtin_ia32_cvtw2mask256:
  case X86::BI__builtin_ia32_cvtw2mask512:
  case X86::BI__builtin_ia32_cvtd2mask128:
  case X86::BI__builtin_ia32_cvtd2mask256:
  case X86::BI__builtin_ia32_cvtd2mask512:
  case X86::BI__builtin_ia32_cvtq2mask128:
  case X86::BI__builtin_ia32_cvtq2mask256:
  case X86::BI__builtin_ia32_cvtq2mask512:
    return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, BuiltinID);

  case X86::BI__builtin_ia32_cvtmask2b128:
  case X86::BI__builtin_ia32_cvtmask2b256:
  case X86::BI__builtin_ia32_cvtmask2b512:
  case X86::BI__builtin_ia32_cvtmask2w128:
  case X86::BI__builtin_ia32_cvtmask2w256:
  case X86::BI__builtin_ia32_cvtmask2w512:
  case X86::BI__builtin_ia32_cvtmask2d128:
  case X86::BI__builtin_ia32_cvtmask2d256:
  case X86::BI__builtin_ia32_cvtmask2d512:
  case X86::BI__builtin_ia32_cvtmask2q128:
  case X86::BI__builtin_ia32_cvtmask2q256:
  case X86::BI__builtin_ia32_cvtmask2q512:
    return interp__builtin_ia32_cvt_mask2vec(S, OpPC, Call, BuiltinID);

  case X86::BI__builtin_ia32_cvtsd2ss:
    return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, false);

  case X86::BI__builtin_ia32_cvtsd2ss_round_mask:
    return interp__builtin_ia32_cvtsd2ss(S, OpPC, Call, true);

  case X86::BI__builtin_ia32_cvtpd2ps:
  case X86::BI__builtin_ia32_cvtpd2ps256:
    return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, false, false);
  case X86::BI__builtin_ia32_cvtpd2ps_mask:
    return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, true, false);
  case X86::BI__builtin_ia32_cvtpd2ps512_mask:
    return interp__builtin_ia32_cvtpd2ps(S, OpPC, Call, true, true);

  case X86::BI__builtin_ia32_cmpb128_mask:
  case X86::BI__builtin_ia32_cmpw128_mask:
  case X86::BI__builtin_ia32_cmpd128_mask:
  case X86::BI__builtin_ia32_cmpq128_mask:
  case X86::BI__builtin_ia32_cmpb256_mask:
  case X86::BI__builtin_ia32_cmpw256_mask:
  case X86::BI__builtin_ia32_cmpd256_mask:
  case X86::BI__builtin_ia32_cmpq256_mask:
  case X86::BI__builtin_ia32_cmpb512_mask:
  case X86::BI__builtin_ia32_cmpw512_mask:
  case X86::BI__builtin_ia32_cmpd512_mask:
  case X86::BI__builtin_ia32_cmpq512_mask:
    return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
                                         /*IsUnsigned=*/false);

  case X86::BI__builtin_ia32_ucmpb128_mask:
  case X86::BI__builtin_ia32_ucmpw128_mask:
  case X86::BI__builtin_ia32_ucmpd128_mask:
  case X86::BI__builtin_ia32_ucmpq128_mask:
  case X86::BI__builtin_ia32_ucmpb256_mask:
  case X86::BI__builtin_ia32_ucmpw256_mask:
  case X86::BI__builtin_ia32_ucmpd256_mask:
  case X86::BI__builtin_ia32_ucmpq256_mask:
  case X86::BI__builtin_ia32_ucmpb512_mask:
  case X86::BI__builtin_ia32_ucmpw512_mask:
  case X86::BI__builtin_ia32_ucmpd512_mask:
  case X86::BI__builtin_ia32_ucmpq512_mask:
    return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
                                         /*IsUnsigned=*/true);

  case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
  case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
  case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
    return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);

  case X86::BI__builtin_ia32_pslldqi128_byteshift:
  case X86::BI__builtin_ia32_pslldqi256_byteshift:
  case X86::BI__builtin_ia32_pslldqi512_byteshift:
    // These SLLDQ intrinsics always operate on byte elements (8 bits).
    // The lane width is hardcoded to 16 to match the SIMD register size,
    // but the algorithm processes one byte per iteration,
    // so APInt(8, ...) is correct and intentional.
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call,
        [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
          unsigned LaneBase = (DstIdx / 16) * 16;
          unsigned LaneIdx = DstIdx % 16;
          if (LaneIdx < Shift)
            return std::make_pair(0, -1);

          return std::make_pair(0,
                                static_cast<int>(LaneBase + LaneIdx - Shift));
        });

  case X86::BI__builtin_ia32_psrldqi128_byteshift:
  case X86::BI__builtin_ia32_psrldqi256_byteshift:
  case X86::BI__builtin_ia32_psrldqi512_byteshift:
    // These SRLDQ intrinsics always operate on byte elements (8 bits).
    // The lane width is hardcoded to 16 to match the SIMD register size,
    // but the algorithm processes one byte per iteration,
    // so APInt(8, ...) is correct and intentional.
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call,
        [](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
          unsigned LaneBase = (DstIdx / 16) * 16;
          unsigned LaneIdx = DstIdx % 16;
          if (LaneIdx + Shift < 16)
            return std::make_pair(0,
                                  static_cast<int>(LaneBase + LaneIdx + Shift));

          return std::make_pair(0, -1);
        });

  case X86::BI__builtin_ia32_palignr128:
  case X86::BI__builtin_ia32_palignr256:
  case X86::BI__builtin_ia32_palignr512:
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [](unsigned DstIdx, unsigned Shift) {
          // Default to -1 → zero-fill this destination element
          unsigned VecIdx = 1;
          int ElemIdx = -1;

          int Lane = DstIdx / 16;
          int Offset = DstIdx % 16;

          // Elements come from VecB first, then VecA after the shift boundary
          unsigned ShiftedIdx = Offset + (Shift & 0xFF);
          if (ShiftedIdx < 16) { // from VecB
            ElemIdx = ShiftedIdx + (Lane * 16);
          } else if (ShiftedIdx < 32) { // from VecA
            VecIdx = 0;
            ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
          }

          return std::pair<unsigned, int>{VecIdx, ElemIdx};
        });

  case X86::BI__builtin_ia32_alignd128:
  case X86::BI__builtin_ia32_alignd256:
  case X86::BI__builtin_ia32_alignd512:
  case X86::BI__builtin_ia32_alignq128:
  case X86::BI__builtin_ia32_alignq256:
  case X86::BI__builtin_ia32_alignq512: {
    unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
    return interp__builtin_ia32_shuffle_generic(
        S, OpPC, Call, [NumElems](unsigned DstIdx, unsigned Shift) {
          unsigned Imm = Shift & 0xFF;
          unsigned EffectiveShift = Imm & (NumElems - 1);
          unsigned SourcePos = DstIdx + EffectiveShift;
          unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
          unsigned ElemIdx = SourcePos & (NumElems - 1);
          return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
        });
  }

  case clang::X86::BI__builtin_ia32_minps:
  case clang::X86::BI__builtin_ia32_minpd:
  case clang::X86::BI__builtin_ia32_minph128:
  case clang::X86::BI__builtin_ia32_minph256:
  case clang::X86::BI__builtin_ia32_minps256:
  case clang::X86::BI__builtin_ia32_minpd256:
  case clang::X86::BI__builtin_ia32_minps512:
  case clang::X86::BI__builtin_ia32_minpd512:
  case clang::X86::BI__builtin_ia32_minph512:
    return interp__builtin_elementwise_fp_binop(
        S, OpPC, Call,
        [](const APFloat &A, const APFloat &B,
           std::optional<APSInt>) -> std::optional<APFloat> {
          if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
              B.isInfinity() || B.isDenormal())
            return std::nullopt;
          if (A.isZero() && B.isZero())
            return B;
          return llvm::minimum(A, B);
        });

  case clang::X86::BI__builtin_ia32_minss:
  case clang::X86::BI__builtin_ia32_minsd:
    return interp__builtin_elementwise_fp_binop(
        S, OpPC, Call,
        [](const APFloat &A, const APFloat &B,
           std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
          return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/true);
        },
        /*IsScalar=*/true);

  case clang::X86::BI__builtin_ia32_minsd_round_mask:
  case clang::X86::BI__builtin_ia32_minss_round_mask:
  case clang::X86::BI__builtin_ia32_minsh_round_mask:
  case clang::X86::BI__builtin_ia32_maxsd_round_mask:
  case clang::X86::BI__builtin_ia32_maxss_round_mask:
  case clang::X86::BI__builtin_ia32_maxsh_round_mask: {
    bool IsMin = BuiltinID == clang::X86::BI__builtin_ia32_minsd_round_mask ||
                 BuiltinID == clang::X86::BI__builtin_ia32_minss_round_mask ||
                 BuiltinID == clang::X86::BI__builtin_ia32_minsh_round_mask;
    return interp__builtin_scalar_fp_round_mask_binop(
        S, OpPC, Call,
        [IsMin](const APFloat &A, const APFloat &B,
                std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
          return EvalScalarMinMaxFp(A, B, RoundingMode, IsMin);
        });
  }

  case clang::X86::BI__builtin_ia32_maxps:
  case clang::X86::BI__builtin_ia32_maxpd:
  case clang::X86::BI__builtin_ia32_maxph128:
  case clang::X86::BI__builtin_ia32_maxph256:
  case clang::X86::BI__builtin_ia32_maxps256:
  case clang::X86::BI__builtin_ia32_maxpd256:
  case clang::X86::BI__builtin_ia32_maxps512:
  case clang::X86::BI__builtin_ia32_maxpd512:
  case clang::X86::BI__builtin_ia32_maxph512:
    return interp__builtin_elementwise_fp_binop(
        S, OpPC, Call,
        [](const APFloat &A, const APFloat &B,
           std::optional<APSInt>) -> std::optional<APFloat> {
          if (A.isNaN() || A.isInfinity() || A.isDenormal() || B.isNaN() ||
              B.isInfinity() || B.isDenormal())
            return std::nullopt;
          if (A.isZero() && B.isZero())
            return B;
          return llvm::maximum(A, B);
        });

  case clang::X86::BI__builtin_ia32_maxss:
  case clang::X86::BI__builtin_ia32_maxsd:
    return interp__builtin_elementwise_fp_binop(
        S, OpPC, Call,
        [](const APFloat &A, const APFloat &B,
           std::optional<APSInt> RoundingMode) -> std::optional<APFloat> {
          return EvalScalarMinMaxFp(A, B, RoundingMode, /*IsMin=*/false);
        },
        /*IsScalar=*/true);
  case X86::BI__builtin_ia32_vpdpwssd128:
  case X86::BI__builtin_ia32_vpdpwssd256:
  case X86::BI__builtin_ia32_vpdpwssd512:
  case X86::BI__builtin_ia32_vpdpbusd128:
  case X86::BI__builtin_ia32_vpdpbusd256:
  case X86::BI__builtin_ia32_vpdpbusd512:
    return interp__builtin_ia32_vpdp(S, OpPC, Call, false);
  case X86::BI__builtin_ia32_vpdpwssds128:
  case X86::BI__builtin_ia32_vpdpwssds256:
  case X86::BI__builtin_ia32_vpdpwssds512:
  case X86::BI__builtin_ia32_vpdpbusds128:
  case X86::BI__builtin_ia32_vpdpbusds256:
  case X86::BI__builtin_ia32_vpdpbusds512:
    return interp__builtin_ia32_vpdp(S, OpPC, Call, true);
  case X86::BI__builtin_ia32_cvtss2si:
  case X86::BI__builtin_ia32_cvtsd2si:
  case X86::BI__builtin_ia32_cvttss2si:
  case X86::BI__builtin_ia32_cvttsd2si:
  case X86::BI__builtin_ia32_cvtss2si64:
  case X86::BI__builtin_ia32_cvtsd2si64:
  case X86::BI__builtin_ia32_cvttss2si64:
  case X86::BI__builtin_ia32_cvttsd2si64:
    return interp_builtin_ia32_cvt_scalar_to_int(S, OpPC, Call);
  case X86::BI__builtin_ia32_cvtpd2dq:
  case X86::BI__builtin_ia32_cvttpd2dq:
  case X86::BI__builtin_ia32_cvtps2dq:
  case X86::BI__builtin_ia32_cvtpd2dq256:
  case X86::BI__builtin_ia32_cvtps2dq256:
  case X86::BI__builtin_ia32_cvttps2dq:
  case X86::BI__builtin_ia32_cvttpd2dq256:
  case X86::BI__builtin_ia32_cvttps2dq256:
    return interp_builtin_ia32_cvt_vector_to_int(S, OpPC, Call);
  default:
    S.FFDiag(S.Current->getLocation(OpPC),
             diag::note_invalid_subexpr_in_const_expr)
        << S.Current->getRange(OpPC);

    return false;
  }

  llvm_unreachable("Unhandled builtin ID");
}

bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
                       ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
  S.getASTContext().recordOffsetOfEvaluation(E);
  CharUnits Result;
  unsigned N = E->getNumComponents();
  assert(N > 0);

  unsigned ArrayIndex = 0;
  QualType CurrentType = E->getTypeSourceInfo()->getType();
  for (unsigned I = 0; I != N; ++I) {
    const OffsetOfNode &Node = E->getComponent(I);
    switch (Node.getKind()) {
    case OffsetOfNode::Field: {
      const FieldDecl *MemberDecl = Node.getField();
      const auto *RD = CurrentType->getAsRecordDecl();
      if (!RD || RD->isInvalidDecl())
        return false;
      const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);
      unsigned FieldIndex = MemberDecl->getFieldIndex();
      assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
      Result +=
          S.getASTContext().toCharUnitsFromBits(RL.getFieldOffset(FieldIndex));
      CurrentType = MemberDecl->getType().getNonReferenceType();
      break;
    }
    case OffsetOfNode::Array: {
      // When generating bytecode, we put all the index expressions as Sint64 on
      // the stack.
      int64_t Index = ArrayIndices[ArrayIndex];
      if (Index < 0)
        return Invalid(S, OpPC);
      const ArrayType *AT = S.getASTContext().getAsArrayType(CurrentType);
      if (!AT)
        return false;
      CurrentType = AT->getElementType();
      CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(CurrentType);
      int64_t ElemSize = ElementSize.getQuantity();
      if (Index != 0 && ElemSize > (llvm::maxIntN(64) / Index)) {
        S.FFDiag(S.Current->getLocation(OpPC),
                 diag::note_constexpr_offsetof_overflow)
            << S.Current->getRange(OpPC);
        return false;
      }
      int64_t Offset = Index * ElemSize;
      if (Result.getQuantity() > llvm::maxIntN(64) - Offset) {
        S.FFDiag(S.Current->getLocation(OpPC),
                 diag::note_constexpr_offsetof_overflow)
            << S.Current->getRange(OpPC);
        return false;
      }
      Result += CharUnits::fromQuantity(Offset);
      ++ArrayIndex;
      break;
    }
    case OffsetOfNode::Base: {
      const CXXBaseSpecifier *BaseSpec = Node.getBase();
      if (BaseSpec->isVirtual())
        return false;

      // Find the layout of the class whose base we are looking into.
      const auto *RD = CurrentType->getAsCXXRecordDecl();
      if (!RD || RD->isInvalidDecl())
        return false;
      const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);

      // Find the base class itself.
      CurrentType = BaseSpec->getType();
      const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
      if (!BaseRD)
        return false;

      // Add the offset to the base.
      Result += RL.getBaseClassOffset(BaseRD);
      break;
    }
    case OffsetOfNode::Identifier:
      llvm_unreachable("Dependent OffsetOfExpr?");
    }
  }

  IntResult = Result.getQuantity();

  return true;
}

bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
                                const Pointer &Ptr, const APSInt &IntValue) {

  const Record *R = Ptr.getRecord();
  assert(R);
  assert(R->getNumFields() == 1);

  unsigned FieldOffset = R->getField(0u)->Offset;
  PtrView FieldPtr = Ptr.view().atField(FieldOffset);
  PrimType FieldT = FieldPtr.getFieldDesc()->getPrimType();

  INT_TYPE_SWITCH(FieldT,
                  FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
  FieldPtr.initialize();
  return true;
}

static void zeroAll(PtrView Dest) {
  const Descriptor *Desc = Dest.getFieldDesc();

  if (Desc->isPrimitive()) {
    TYPE_SWITCH(Desc->getPrimType(), {
      Dest.deref<T>().~T();
      new (&Dest.deref<T>()) T();
    });
    return;
  }

  if (Desc->isRecord()) {
    const Record *R = Desc->ElemRecord;
    for (const Record::Field &F : R->fields()) {
      PtrView FieldPtr = Dest.atField(F.Offset);
      zeroAll(FieldPtr);
    }
    return;
  }

  if (Desc->isPrimitiveArray()) {
    for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
      TYPE_SWITCH(Desc->getPrimType(), {
        Dest.deref<T>().~T();
        new (&Dest.deref<T>()) T();
      });
    }
    return;
  }

  if (Desc->isCompositeArray()) {
    for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
      PtrView ElemPtr = Dest.atIndex(I).narrow();
      zeroAll(ElemPtr);
    }
    return;
  }
}

static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
                          PtrView Dest, bool Activate, bool Diagnose);
static bool copyRecord(InterpState &S, CodePtr OpPC, PtrView Src, PtrView Dest,
                       bool Activate = false, bool Diagnose = true) {
  [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
  const Descriptor *DestDesc = Dest.getFieldDesc();

  auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
    PtrView DestField = Dest.atField(F.Offset);
    PtrView SrcField = Src.atField(F.Offset);

    if (OptPrimType FT = F.T) {
      if (!SrcField.isInitialized()) {
        if (Diagnose)
          return diagnoseUninitialized(S, OpPC, false, SrcField.block(),
                                       SrcField.getLifetime(), AK_Read);
        // Just skip.
        return true;
      }

      TYPE_SWITCH(*FT, DestField.deref<T>() = SrcField.deref<T>(););
      if (DestField.canBeInitialized())
        DestField.initialize();
      if (Activate)
        DestField.activate();
      return true;
    }

    return copyComposite(S, OpPC, SrcField, DestField, Activate, Diagnose);
  };

  assert(SrcDesc->isRecord());
  assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
  const Record *R = DestDesc->ElemRecord;
  for (const Record::Field &F : R->fields()) {
    PtrView FP = Src.atField(F.Offset);

    if (!CheckMutable(S, OpPC, FP))
      return false;

    if (R->isUnion()) {
      // For unions, only copy the active field. Zero all others.
      if (FP.isActive()) {
        if (!copyField(F, /*Activate=*/true))
          return false;
      } else {
        PtrView DestField = Dest.atField(F.Offset);
        zeroAll(DestField);
      }
    } else {
      if (!copyField(F, Activate))
        return false;
    }
  }

  for (const Record::Base &B : R->bases()) {
    PtrView DestBase = Dest.atField(B.Offset);
    if (!copyRecord(S, OpPC, Src.atField(B.Offset), DestBase, Activate,
                    Diagnose))
      return false;
  }

  Dest.initialize();
  if (Activate)
    Dest.activate();
  return true;
}

static bool copyComposite(InterpState &S, CodePtr OpPC, PtrView Src,
                          PtrView Dest, bool Activate = false,
                          bool Diagnose = false) {
  assert(Src.isLive() && Dest.isLive());

  [[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
  const Descriptor *DestDesc = Dest.getFieldDesc();

  assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());

  if (DestDesc->isPrimitiveArray()) {
    if (!SrcDesc->isPrimitiveArray())
      return false;
    // For floating types, check the actual QualType so we don't accidentally
    // mix up semantics.
    if (SrcDesc->getPrimType() == PT_Float) {
      if (!S.getASTContext().hasSimilarType(SrcDesc->getElemQualType(),
                                            DestDesc->getElemQualType()))
        return false;
    }

    assert(SrcDesc->isPrimitiveArray());
    assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
    assert(SrcDesc->getPrimType() == DestDesc->getPrimType());
    PrimType ET = DestDesc->getPrimType();
    for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
      PtrView DestElem = Dest.atIndex(I);
      TYPE_SWITCH(ET, { DestElem.deref<T>() = Src.elem<T>(I); });
      DestElem.initializeElement(I);
    }
    return true;
  }

  if (DestDesc->isCompositeArray()) {
    if (!SrcDesc->isCompositeArray())
      return false;
    assert(SrcDesc->isCompositeArray());
    assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
    for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
      PtrView SrcElem = Src.atIndex(I).narrow();
      PtrView DestElem = Dest.atIndex(I).narrow();
      if (!copyComposite(S, OpPC, SrcElem, DestElem, Activate))
        return false;
    }
    return true;
  }

  if (DestDesc->isRecord()) {
    if (!SrcDesc->isRecord())
      return false;
    return copyRecord(S, OpPC, Src, Dest, Activate, Diagnose);
  }
  return Invalid(S, OpPC);
}

bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest,
              bool Activate, bool Diagnose) {
  if (!Src.isBlockPointer() || Src.getFieldDesc()->isPrimitive())
    return false;
  if (!Dest.isBlockPointer() || Dest.getFieldDesc()->isPrimitive())
    return false;

  return copyComposite(S, OpPC, Src.view(), Dest.view(), Activate, Diagnose);
}

} // namespace interp
} // namespace clang