//===--- Compiler.cpp - Code generator for expressions ---*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "Compiler.h"
#include "../ExprConstShared.h"
#include "ByteCodeEmitter.h"
#include "Context.h"
#include "FixedPoint.h"
#include "Floating.h"
#include "Function.h"
#include "InterpShared.h"
#include "PrimType.h"
#include "Program.h"
#include "clang/AST/Attr.h"
#include "clang/AST/DynamicRecursiveASTVisitor.h"
#include "llvm/Support/SaveAndRestore.h"

using namespace clang;
using namespace clang::interp;

using APSInt = llvm::APSInt;

namespace clang {
namespace interp {

static std::optional<bool> getBoolValue(const Expr *E) {
  if (const auto *CE = dyn_cast_if_present<ConstantExpr>(E);
      CE && CE->hasAPValueResult() &&
      CE->getResultAPValueKind() == APValue::ValueKind::Int) {
    return CE->getResultAsAPSInt().getBoolValue();
  }

  return std::nullopt;
}

/// Check if \c E has side-effects. This is used to avoid some temporary
/// variables and is supposed to be a quick check, not exhaustive. That's why
/// we're not using Expr::HasSideEffects().
static bool isSideEffectFree(const Expr *E) {
  if (isa<IntegerLiteral, FloatingLiteral, CharacterLiteral,
          CXXBoolLiteralExpr>(E))
    return true;
  if (isa<DeclRefExpr>(E))
    return true;

  return false;
}

/// Scope chain managing the variable lifetimes.
template <class Emitter> class VariableScope {
public:
  VariableScope(Compiler<Emitter> *Ctx, ScopeKind Kind = ScopeKind::Block)
      : Ctx(Ctx), Parent(Ctx->VarScope), Kind(Kind) {
    if (Parent)
      this->LocalsAlwaysEnabled = Parent->LocalsAlwaysEnabled;
    Ctx->VarScope = this;
  }

  virtual ~VariableScope() { Ctx->VarScope = this->Parent; }

  virtual void addLocal(Scope::Local Local) {
    llvm_unreachable("Shouldn't be called");
  }
  /// Like addExtended, but adds to the nearest scope of the given kind.
  void addForScopeKind(const Scope::Local &Local, ScopeKind Kind) {
    VariableScope *P = this;
    while (P) {
      // We found the right scope kind.
      if (P->Kind == Kind) {
        P->addLocal(Local);
        return;
      }
      // If we reached the root scope and we're looking for a Block scope,
      // attach it to the root instead of the current scope.
      if (!P->Parent && Kind == ScopeKind::Block) {
        P->addLocal(Local);
        return;
      }
      P = P->Parent;
      if (!P)
        break;
    }

    // Add to this scope.
    this->addLocal(Local);
  }

  virtual bool emitDestructors(const Expr *E = nullptr) { return true; }
  virtual bool destroyLocals(const Expr *E = nullptr) { return true; }
  virtual void forceInit() {}
  VariableScope *getParent() const { return Parent; }
  ScopeKind getKind() const { return Kind; }

  /// Whether locals added to this scope are enabled by default.
  /// This is almost always true, except for the two branches
  /// of a conditional operator.
  bool LocalsAlwaysEnabled = true;

protected:
  /// Compiler instance.
  Compiler<Emitter> *Ctx;
  /// Link to the parent scope.
  VariableScope *Parent;
  ScopeKind Kind;
};

/// Generic scope for local variables.
template <class Emitter> class LocalScope : public VariableScope<Emitter> {
public:
  LocalScope(Compiler<Emitter> *Ctx, ScopeKind Kind = ScopeKind::Block)
      : VariableScope<Emitter>(Ctx, Kind) {}

  /// Emit a Destroy op for this scope.
  ~LocalScope() override {
    if (!Idx || ExplicitlyDestroyed)
      return;
    this->Ctx->emitDestroy(*Idx, SourceInfo{});
    removeStoredOpaqueValues();
  }
  /// Explicit destruction of local variables.
  bool destroyLocals(const Expr *E = nullptr) override {
    if (!Idx)
      return true;

    // NB: We are *not* resetting Idx here as to allow multiple
    // calls to destroyLocals().
    bool Success = this->emitDestructors(E);
    this->Ctx->emitDestroy(*Idx, E);
    ExplicitlyDestroyed = true;
    return Success;
  }

  void addLocal(Scope::Local Local) override {
    if (!Idx) {
      Idx = static_cast<unsigned>(this->Ctx->Descriptors.size());
      this->Ctx->Descriptors.emplace_back();
      this->Ctx->emitInitScope(*Idx, {});
    }

    Local.EnabledByDefault = this->LocalsAlwaysEnabled;
    this->Ctx->Descriptors[*Idx].emplace_back(Local);
  }

  /// Force-initialize this scope. Usually, scopes are lazily initialized when
  /// the first local variable is created, but in scenarios with conditonal
  /// operators, we need to ensure scope is initialized just in case one of the
  /// arms will create a local and the other won't. In such a case, the
  /// InitScope() op would be part of the arm that created the local.
  void forceInit() override {
    if (!Idx) {
      Idx = static_cast<unsigned>(this->Ctx->Descriptors.size());
      this->Ctx->Descriptors.emplace_back();
      this->Ctx->emitInitScope(*Idx, {});
    }
  }

  bool emitDestructors(const Expr *E = nullptr) override {
    if (!Idx)
      return true;

    // Emit destructor calls for local variables of record
    // type with a destructor.
    for (Scope::Local &Local : llvm::reverse(this->Ctx->Descriptors[*Idx])) {
      if (Local.Desc->hasTrivialDtor())
        continue;

      if (!Local.EnabledByDefault) {
        typename Emitter::LabelTy EndLabel = this->Ctx->getLabel();
        if (!this->Ctx->emitGetLocalEnabled(Local.Offset, E))
          return false;
        if (!this->Ctx->jumpFalse(EndLabel, E))
          return false;

        if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
          return false;

        if (!this->Ctx->emitDestructionPop(Local.Desc, Local.Desc->getLoc()))
          return false;

        this->Ctx->fallthrough(EndLabel);
        this->Ctx->emitLabel(EndLabel);
      } else {
        if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
          return false;
        if (!this->Ctx->emitDestructionPop(Local.Desc, Local.Desc->getLoc()))
          return false;
      }

      removeIfStoredOpaqueValue(Local);
    }
    return true;
  }

  void removeStoredOpaqueValues() {
    if (!Idx)
      return;

    for (const Scope::Local &Local : this->Ctx->Descriptors[*Idx]) {
      removeIfStoredOpaqueValue(Local);
    }
  }

  void removeIfStoredOpaqueValue(const Scope::Local &Local) {
    if (const auto *OVE =
            llvm::dyn_cast_if_present<OpaqueValueExpr>(Local.Desc->asExpr())) {
      this->Ctx->OpaqueExprs.erase(OVE);
    };
  }

  /// Index of the scope in the chain.
  UnsignedOrNone Idx = std::nullopt;
  bool ExplicitlyDestroyed = false;
};

template <class Emitter> class ArrayIndexScope final {
public:
  ArrayIndexScope(Compiler<Emitter> *Ctx, uint64_t Index) : Ctx(Ctx) {
    OldArrayIndex = Ctx->ArrayIndex;
    Ctx->ArrayIndex = Index;
  }

  ~ArrayIndexScope() { Ctx->ArrayIndex = OldArrayIndex; }

private:
  Compiler<Emitter> *Ctx;
  std::optional<uint64_t> OldArrayIndex;
};

template <class Emitter> class SourceLocScope final {
public:
  SourceLocScope(Compiler<Emitter> *Ctx, const Expr *DefaultExpr) : Ctx(Ctx) {
    assert(DefaultExpr);
    // We only switch if the current SourceLocDefaultExpr is null.
    if (!Ctx->SourceLocDefaultExpr) {
      Enabled = true;
      Ctx->SourceLocDefaultExpr = DefaultExpr;
    }
  }

  ~SourceLocScope() {
    if (Enabled)
      Ctx->SourceLocDefaultExpr = nullptr;
  }

private:
  Compiler<Emitter> *Ctx;
  bool Enabled = false;
};

template <class Emitter> class InitLinkScope final {
public:
  InitLinkScope(Compiler<Emitter> *Ctx, InitLink &&Link) : Ctx(Ctx) {
    Ctx->InitStack.push_back(std::move(Link));
  }

  ~InitLinkScope() { this->Ctx->InitStack.pop_back(); }

public:
  Compiler<Emitter> *Ctx;
};

template <class Emitter> class InitStackScope final {
public:
  InitStackScope(Compiler<Emitter> *Ctx, bool Active)
      : Ctx(Ctx), OldValue(Ctx->InitStackActive), Active(Active) {
    // An explicit initializer nested in a default member initializer still
    // needs the surrounding default initializer's `this` reconstruction.
    Ctx->InitStackActive = OldValue || Active;
    if (Active)
      Ctx->InitStack.push_back(InitLink::DIE());
  }

  ~InitStackScope() {
    this->Ctx->InitStackActive = OldValue;
    if (Active)
      Ctx->InitStack.pop_back();
  }

private:
  Compiler<Emitter> *Ctx;
  bool OldValue;
  bool Active;
};

/// Scope used to handle temporaries in toplevel variable declarations.
template <class Emitter> class DeclScope final : public LocalScope<Emitter> {
public:
  DeclScope(Compiler<Emitter> *Ctx, const VarDecl *VD)
      : LocalScope<Emitter>(Ctx), Scope(Ctx->P),
        OldInitializingDecl(Ctx->InitializingDecl) {
    Ctx->InitializingDecl = VD;
    Ctx->InitStack.push_back(InitLink::Decl(VD));
  }

  ~DeclScope() {
    this->Ctx->InitializingDecl = OldInitializingDecl;
    this->Ctx->InitStack.pop_back();
  }

private:
  Program::DeclScope Scope;
  const VarDecl *OldInitializingDecl;
};

/// Scope used to handle initialization methods.
template <class Emitter> class OptionScope final {
public:
  /// Root constructor, compiling or discarding primitives.
  OptionScope(Compiler<Emitter> *Ctx, bool NewDiscardResult,
              bool NewInitializing, bool NewToLValue)
      : Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
        OldInitializing(Ctx->Initializing), OldToLValue(Ctx->ToLValue) {
    Ctx->DiscardResult = NewDiscardResult;
    Ctx->Initializing = NewInitializing;
    Ctx->ToLValue = NewToLValue;
  }

  ~OptionScope() {
    Ctx->DiscardResult = OldDiscardResult;
    Ctx->Initializing = OldInitializing;
    Ctx->ToLValue = OldToLValue;
  }

private:
  /// Parent context.
  Compiler<Emitter> *Ctx;
  /// Old discard flag to restore.
  bool OldDiscardResult;
  bool OldInitializing;
  bool OldToLValue;
};

template <class Emitter>
bool InitLink::emit(Compiler<Emitter> *Ctx, const Expr *E) const {
  switch (Kind) {
  case K_This:
    return Ctx->emitThis(E);
  case K_Field:
    // We're assuming there's a base pointer on the stack already.
    return Ctx->emitGetPtrFieldPop(Offset, E);
  case K_Base:
    return Ctx->emitGetPtrBasePop(Offset, false, E);
  case K_Temp:
    return Ctx->emitGetPtrLocal(Offset, E);
  case K_Decl:
    return Ctx->visitDeclRef(D, E);
  case K_Elem:
    if (!Ctx->emitConstUint32(Offset, E))
      return false;
    return Ctx->emitArrayElemPtrPopUint32(E);
  case K_RVO:
    return Ctx->emitRVOPtr(E);
  case K_InitList:
    return true;
  default:
    llvm_unreachable("Unhandled InitLink kind");
  }
  return true;
}

/// Sets the context for break/continue statements.
template <class Emitter> class LoopScope final {
public:
  using LabelTy = typename Compiler<Emitter>::LabelTy;
  using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
  using LabelInfo = typename Compiler<Emitter>::LabelInfo;

  LoopScope(Compiler<Emitter> *Ctx, const Stmt *Name, LabelTy BreakLabel,
            LabelTy ContinueLabel)
      : Ctx(Ctx) {
#ifndef NDEBUG
    for (const LabelInfo &LI : Ctx->LabelInfoStack)
      assert(LI.Name != Name);
#endif

    this->Ctx->LabelInfoStack.emplace_back(Name, BreakLabel, ContinueLabel,
                                           /*DefaultLabel=*/std::nullopt,
                                           Ctx->VarScope);
  }

  ~LoopScope() { this->Ctx->LabelInfoStack.pop_back(); }

private:
  Compiler<Emitter> *Ctx;
};

// Sets the context for a switch scope, mapping labels.
template <class Emitter> class SwitchScope final {
public:
  using LabelTy = typename Compiler<Emitter>::LabelTy;
  using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
  using CaseMap = typename Compiler<Emitter>::CaseMap;
  using LabelInfo = typename Compiler<Emitter>::LabelInfo;

  SwitchScope(Compiler<Emitter> *Ctx, const Stmt *Name, CaseMap &&CaseLabels,
              LabelTy BreakLabel, OptLabelTy DefaultLabel)
      : Ctx(Ctx), OldCaseLabels(std::move(this->Ctx->CaseLabels)) {
#ifndef NDEBUG
    for (const LabelInfo &LI : Ctx->LabelInfoStack)
      assert(LI.Name != Name);
#endif

    this->Ctx->CaseLabels = std::move(CaseLabels);
    this->Ctx->LabelInfoStack.emplace_back(Name, BreakLabel,
                                           /*ContinueLabel=*/std::nullopt,
                                           DefaultLabel, Ctx->VarScope);
  }

  ~SwitchScope() {
    this->Ctx->CaseLabels = std::move(OldCaseLabels);
    this->Ctx->LabelInfoStack.pop_back();
  }

private:
  Compiler<Emitter> *Ctx;
  CaseMap OldCaseLabels;
};

/// When generating code for e.g. implicit field initializers in constructors,
/// we don't have anything to point to in case the initializer causes an error.
/// In that case, we need to disable location tracking for the initializer so
/// we later point to the call range instead.
template <class Emitter> class LocOverrideScope final {
public:
  LocOverrideScope(Compiler<Emitter> *Ctx, SourceInfo NewValue,
                   bool Enabled = true)
      : Ctx(Ctx), OldFlag(Ctx->LocOverride), Enabled(Enabled) {

    if (Enabled)
      Ctx->LocOverride = NewValue;
  }

  ~LocOverrideScope() {
    if (Enabled)
      Ctx->LocOverride = OldFlag;
  }

private:
  Compiler<Emitter> *Ctx;
  std::optional<SourceInfo> OldFlag;
  bool Enabled;
};

} // namespace interp
} // namespace clang

template <class Emitter>
bool Compiler<Emitter>::VisitCastExpr(const CastExpr *E) {
  const Expr *SubExpr = E->getSubExpr();

  if (DiscardResult)
    return this->delegate(SubExpr);

  switch (E->getCastKind()) {
  case CK_LValueToRValue: {
    if (ToLValue && E->getType()->isPointerType()) {
      assert(!DiscardResult);
      if (!this->visit(SubExpr))
        return false;
      return this->emitLoadPopL(E);
    }

    if (SubExpr->getType().isVolatileQualified())
      return this->emitInvalidCast(CastKind::Volatile, /*Fatal=*/true, E);

    OptPrimType SubExprT = classify(SubExpr->getType());
    // Try to load the value directly. This is purely a performance
    // optimization.
    if (SubExprT) {
      if (const auto *DRE = dyn_cast<DeclRefExpr>(SubExpr)) {
        const ValueDecl *D = DRE->getDecl();
        bool IsReference = D->getType()->isReferenceType();

        if (!IsReference) {
          if (Context::shouldBeGloballyIndexed(D)) {
            if (auto GlobalIndex = P.getGlobal(D))
              return this->emitGetGlobal(*SubExprT, *GlobalIndex, E);
          } else if (auto It = Locals.find(D); It != Locals.end()) {
            return this->emitGetLocal(*SubExprT, It->second.Offset, E);
          } else if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
            if (auto It = this->Params.find(PVD); It != this->Params.end()) {
              return this->emitGetParam(*SubExprT, It->second.Index, E);
            }
          }
        }
      }
    }

    // Prepare storage for the result.
    if (!Initializing && !SubExprT) {
      UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    if (!this->visit(SubExpr))
      return false;

    if (SubExprT)
      return this->emitLoadPop(*SubExprT, E);

    // If the subexpr type is not primitive, we need to perform a copy here.
    // This happens for example in C when dereferencing a pointer of struct
    // type.
    return this->emitMemcpy(E);
  }

  case CK_DerivedToBaseMemberPointer: {
    if (E->containsErrors())
      return false;
    assert(classifyPrim(E) == PT_MemberPtr);
    assert(classifyPrim(SubExpr) == PT_MemberPtr);

    if (!this->delegate(SubExpr))
      return false;

    const CXXRecordDecl *CurDecl = SubExpr->getType()
                                       ->castAs<MemberPointerType>()
                                       ->getMostRecentCXXRecordDecl();
    for (const CXXBaseSpecifier *B : E->path()) {
      const CXXRecordDecl *ToDecl = B->getType()->getAsCXXRecordDecl();
      unsigned DerivedOffset = Ctx.collectBaseOffset(ToDecl, CurDecl);

      if (!this->emitCastMemberPtrBasePop(DerivedOffset, ToDecl, E))
        return false;
      CurDecl = ToDecl;
    }

    return true;
  }

  case CK_BaseToDerivedMemberPointer: {
    if (E->containsErrors())
      return false;
    assert(classifyPrim(E) == PT_MemberPtr);
    assert(classifyPrim(SubExpr) == PT_MemberPtr);

    if (!this->delegate(SubExpr))
      return false;

    const CXXRecordDecl *CurDecl = SubExpr->getType()
                                       ->castAs<MemberPointerType>()
                                       ->getMostRecentCXXRecordDecl();
    // Base-to-derived member pointer casts store the path in derived-to-base
    // order, so iterate backwards. The CXXBaseSpecifier also provides us with
    // the wrong end of the derived->base arc, so stagger the path by one class.
    typedef std::reverse_iterator<CastExpr::path_const_iterator> ReverseIter;
    for (ReverseIter PathI(E->path_end() - 1), PathE(E->path_begin());
         PathI != PathE; ++PathI) {
      const CXXRecordDecl *ToDecl = (*PathI)->getType()->getAsCXXRecordDecl();
      unsigned DerivedOffset = Ctx.collectBaseOffset(CurDecl, ToDecl);

      if (!this->emitCastMemberPtrDerivedPop(-DerivedOffset, ToDecl, E))
        return false;
      CurDecl = ToDecl;
    }

    const CXXRecordDecl *ToDecl =
        E->getType()->castAs<MemberPointerType>()->getMostRecentCXXRecordDecl();
    assert(ToDecl != CurDecl);
    unsigned DerivedOffset = Ctx.collectBaseOffset(CurDecl, ToDecl);

    if (!this->emitCastMemberPtrDerivedPop(-DerivedOffset, ToDecl, E))
      return false;

    return true;
  }

  case CK_UncheckedDerivedToBase:
  case CK_DerivedToBase: {
    if (!this->delegate(SubExpr))
      return false;

    const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
      if (const auto *PT = dyn_cast<PointerType>(Ty))
        return PT->getPointeeType()->getAsCXXRecordDecl();
      return Ty->getAsCXXRecordDecl();
    };

    // FIXME: We can express a series of non-virtual casts as a single
    // GetPtrBasePop op.
    QualType CurType = SubExpr->getType();
    for (const CXXBaseSpecifier *B : E->path()) {
      if (B->isVirtual()) {
        if (!this->emitGetPtrVirtBasePop(extractRecordDecl(B->getType()), E))
          return false;
        CurType = B->getType();
      } else {
        unsigned DerivedOffset = collectBaseOffset(B->getType(), CurType);
        if (!this->emitGetPtrBasePop(
                DerivedOffset, /*NullOK=*/E->getType()->isPointerType(), E))
          return false;
        CurType = B->getType();
      }
    }

    return true;
  }

  case CK_BaseToDerived: {
    if (!this->delegate(SubExpr))
      return false;
    unsigned DerivedOffset =
        collectBaseOffset(SubExpr->getType(), E->getType());

    const Type *TargetType = E->getType().getTypePtr();
    if (TargetType->isPointerOrReferenceType())
      TargetType = TargetType->getPointeeType().getTypePtr();
    return this->emitGetPtrDerivedPop(DerivedOffset,
                                      /*NullOK=*/E->getType()->isPointerType(),
                                      TargetType, E);
  }

  case CK_FloatingCast: {
    // HLSL uses CK_FloatingCast to cast between vectors.
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    if (!SubExpr->getType()->isFloatingType() ||
        !E->getType()->isFloatingType())
      return false;
    if (!this->visit(SubExpr))
      return false;
    const auto *TargetSemantics = &Ctx.getFloatSemantics(E->getType());
    return this->emitCastFP(TargetSemantics, getRoundingMode(E), E);
  }

  case CK_IntegralToFloating: {
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    if (!E->getType()->isRealFloatingType())
      return false;
    if (!this->visit(SubExpr))
      return false;
    const auto *TargetSemantics = &Ctx.getFloatSemantics(E->getType());
    return this->emitCastIntegralFloating(classifyPrim(SubExpr),
                                          TargetSemantics, getFPOptions(E), E);
  }

  case CK_FloatingToBoolean: {
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    if (!SubExpr->getType()->isRealFloatingType() ||
        !E->getType()->hasBooleanRepresentation())
      return false;
    if (const auto *FL = dyn_cast<FloatingLiteral>(SubExpr))
      return this->emitConstBool(FL->getValue().isNonZero(), E);
    if (!this->visit(SubExpr))
      return false;
    return this->emitCastFloatingIntegralBool(getFPOptions(E), E);
  }

  case CK_FloatingToIntegral: {
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    if (!E->getType()->isIntegralOrEnumerationType())
      return false;
    if (!this->visit(SubExpr))
      return false;
    PrimType ToT = classifyPrim(E);
    if (ToT == PT_IntAP)
      return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(E->getType()),
                                              getFPOptions(E), E);
    if (ToT == PT_IntAPS)
      return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(E->getType()),
                                               getFPOptions(E), E);

    return this->emitCastFloatingIntegral(ToT, getFPOptions(E), E);
  }

  case CK_NullToPointer:
  case CK_NullToMemberPointer: {
    if (!this->discard(SubExpr))
      return false;
    uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(E->getType());
    return this->emitNull(classifyPrim(E->getType()), Val,
                          E->getType().getTypePtr(), E);
  }

  case CK_PointerToIntegral: {
    if (!this->visit(SubExpr))
      return false;

    // If SubExpr doesn't result in a pointer, make it one.
    if (PrimType FromT = classifyPrim(SubExpr->getType()); FromT != PT_Ptr) {
      assert(isPtrType(FromT));
      if (!this->emitDecayPtr(FromT, PT_Ptr, E))
        return false;
    }

    PrimType T = classifyPrim(E->getType());
    if (T == PT_IntAP)
      return this->emitCastPointerIntegralAP(Ctx.getBitWidth(E->getType()), E);
    if (T == PT_IntAPS)
      return this->emitCastPointerIntegralAPS(Ctx.getBitWidth(E->getType()), E);
    return this->emitCastPointerIntegral(T, E);
  }

  case CK_ArrayToPointerDecay: {
    if (!this->visit(SubExpr))
      return false;
    return this->emitArrayDecay(E);
  }

  case CK_IntegralToPointer: {
    QualType IntType = SubExpr->getType();
    assert(IntType->isIntegralOrEnumerationType());
    if (!this->visit(SubExpr))
      return false;
    // FIXME: I think the discard is wrong since the int->ptr cast might cause a
    // diagnostic.
    PrimType T = classifyPrim(IntType);
    if (!this->emitGetIntPtr(T, E->getType().getTypePtr(), E))
      return false;

    QualType PtrType = E->getType();
    PrimType DestPtrT = classifyPrim(PtrType);
    if (DestPtrT == PT_Ptr)
      return true;

    // In case we're converting the integer to a non-Pointer.
    return this->emitDecayPtr(PT_Ptr, DestPtrT, E);
  }

  case CK_AtomicToNonAtomic:
  case CK_ConstructorConversion:
  case CK_FunctionToPointerDecay:
  case CK_NonAtomicToAtomic:
  case CK_NoOp:
  case CK_UserDefinedConversion:
  case CK_CPointerToObjCPointerCast:
    return this->delegate(SubExpr);

  case CK_AddressSpaceConversion: {
    if (E->containsErrors())
      return false;

    if (!this->visit(SubExpr))
      return false;

    uint64_t Val;
    if (E->getType()->isPointerType())
      Val = Ctx.getASTContext().getTargetNullPointerValue(E->getType());
    else
      Val = 0;

    if (!this->emitCastAddressSpace(Val, E->getType().getTypePtr(), E))
      return false;
    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  }

  case CK_BitCast: {
    if (E->containsErrors())
      return false;
    QualType ETy = E->getType();
    // Reject bitcasts to atomic types.
    if (ETy->isAtomicType()) {
      if (!this->discard(SubExpr))
        return false;
      return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, E);
    }
    QualType SubExprTy = SubExpr->getType();
    OptPrimType FromT = classify(SubExprTy);
    // Casts from integer/vector to vector.
    if (E->getType()->isVectorType())
      return this->emitBuiltinBitCast(E);

    OptPrimType ToT = classify(E->getType());
    if (!FromT || !ToT)
      return false;

    assert(isPtrType(*FromT));
    assert(isPtrType(*ToT));
    bool SrcIsVoidPtr = SubExprTy->isVoidPointerType();
    if (FromT == ToT) {
      if (E->getType()->isVoidPointerType() &&
          !SubExprTy->isFunctionPointerType()) {
        return this->delegate(SubExpr);
      }

      if (!this->visit(SubExpr))
        return false;
      if (!this->emitCheckBitCast(ETy->getPointeeType().getTypePtr(),
                                  SrcIsVoidPtr, E))
        return false;

      if (E->getType()->isFunctionPointerType() ||
          SubExprTy->isFunctionPointerType()) {
        return this->emitFnPtrCast(E);
      }
      if (FromT == PT_Ptr)
        return this->emitPtrPtrCast(SubExprTy->isVoidPointerType(),
                                    E->getType().getTypePtr(), E);
      return true;
    }

    if (!this->visit(SubExpr))
      return false;
    return this->emitDecayPtr(*FromT, *ToT, E);
  }
  case CK_IntegralToBoolean:
  case CK_FixedPointToBoolean: {
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    // HLSL uses this to cast to one-element vectors.
    OptPrimType FromT = classify(SubExpr->getType());
    if (!FromT)
      return false;

    if (const auto *IL = dyn_cast<IntegerLiteral>(SubExpr))
      return this->emitConst(IL->getValue(), E);
    if (!this->visit(SubExpr))
      return false;
    return this->emitCast(*FromT, classifyPrim(E), E);
  }

  case CK_IntegralCast:
    if (E->getType()->isVectorType())
      return this->emitVectorConversion(E->getSubExpr(), E);
    [[fallthrough]];
  case CK_BooleanToSignedIntegral: {
    OptPrimType FromT = classify(SubExpr->getType());
    OptPrimType ToT = classify(E->getType());
    if (!FromT || !ToT)
      return false;

    // Try to emit a casted known constant value directly.
    if (const auto *IL = dyn_cast<IntegerLiteral>(SubExpr)) {
      if (ToT != PT_IntAP && ToT != PT_IntAPS && FromT != PT_IntAP &&
          FromT != PT_IntAPS && !E->getType()->isEnumeralType())
        return this->emitConst(APSInt(IL->getValue(), !isSignedType(*FromT)),
                               E);
      if (!this->emitConst(IL->getValue(), SubExpr))
        return false;
    } else {
      if (!this->visit(SubExpr))
        return false;
    }

    // Possibly diagnose casts to enum types if the target type does not
    // have a fixed size.
    if (Ctx.getLangOpts().CPlusPlus && E->getType()->isEnumeralType()) {
      const auto *ED = E->getType()->castAsEnumDecl();
      if (!ED->isFixed()) {
        if (!this->emitCheckEnumValue(*FromT, ED, E))
          return false;
      }
    }

    if (ToT == PT_IntAP) {
      if (!this->emitCastAP(*FromT, Ctx.getBitWidth(E->getType()), E))
        return false;
    } else if (ToT == PT_IntAPS) {
      if (!this->emitCastAPS(*FromT, Ctx.getBitWidth(E->getType()), E))
        return false;
    } else {
      if (FromT == ToT)
        return true;
      if (!this->emitCast(*FromT, *ToT, E))
        return false;
    }
    if (E->getCastKind() == CK_BooleanToSignedIntegral)
      return this->emitNeg(*ToT, E);
    return true;
  }

  case CK_PointerToBoolean:
    if (!this->visit(SubExpr))
      return false;
    return this->emitIsNonNullPtr(E);

  case CK_MemberPointerToBoolean:
    if (!this->visit(SubExpr))
      return false;
    return this->emitIsNonNullMemberPtr(E);

  case CK_IntegralComplexToBoolean:
  case CK_FloatingComplexToBoolean: {
    if (!this->visit(SubExpr))
      return false;
    return this->emitComplexBoolCast(SubExpr);
  }

  case CK_IntegralComplexToReal:
  case CK_FloatingComplexToReal:
    return this->emitComplexReal(SubExpr);

  case CK_IntegralRealToComplex:
  case CK_FloatingRealToComplex: {
    // We're creating a complex value here, so we need to
    // allocate storage for it.
    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateTemporary(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    PrimType T = classifyPrim(SubExpr->getType());
    // Init the complex value to {SubExpr, 0}.
    if (!this->visitArrayElemInit(0, SubExpr, T))
      return false;
    // Zero-init the second element.
    if (!this->visitZeroInitializer(T, SubExpr->getType(), SubExpr))
      return false;
    return this->emitInitElem(T, 1, SubExpr);
  }

  case CK_IntegralComplexCast:
  case CK_FloatingComplexCast:
  case CK_IntegralComplexToFloatingComplex:
  case CK_FloatingComplexToIntegralComplex: {
    assert(E->getType()->isAnyComplexType());
    assert(SubExpr->getType()->isAnyComplexType());
    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    // Location for the SubExpr.
    // Since SubExpr is of complex type, visiting it results in a pointer
    // anyway, so we just create a temporary pointer variable.
    unsigned SubExprOffset =
        allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(SubExpr))
      return false;
    if (!this->emitSetLocal(PT_Ptr, SubExprOffset, E))
      return false;

    PrimType SourceElemT = classifyComplexElementType(SubExpr->getType());
    QualType DestElemType =
        E->getType()->getAs<ComplexType>()->getElementType();
    PrimType DestElemT = classifyPrim(DestElemType);
    // Cast both elements individually.
    for (unsigned I = 0; I != 2; ++I) {
      if (!this->emitGetLocal(PT_Ptr, SubExprOffset, E))
        return false;
      if (!this->emitArrayElemPop(SourceElemT, I, E))
        return false;

      // Do the cast.
      if (!this->emitPrimCast(SourceElemT, DestElemT, DestElemType, E))
        return false;

      // Save the value.
      if (!this->emitInitElem(DestElemT, I, E))
        return false;
    }
    return true;
  }

  case CK_VectorSplat: {
    assert(!canClassify(E->getType()));
    assert(E->getType()->isVectorType());

    if (!canClassify(SubExpr->getType()))
      return false;

    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    const auto *VT = E->getType()->getAs<VectorType>();
    PrimType ElemT = classifyPrim(SubExpr->getType());
    unsigned ElemOffset =
        allocateLocalPrimitive(SubExpr, ElemT, /*IsConst=*/true);

    // Prepare a local variable for the scalar value.
    if (!this->visit(SubExpr))
      return false;
    if (classifyPrim(SubExpr) == PT_Ptr && !this->emitLoadPop(ElemT, E))
      return false;

    if (!this->emitSetLocal(ElemT, ElemOffset, E))
      return false;

    for (unsigned I = 0; I != VT->getNumElements(); ++I) {
      if (!this->emitGetLocal(ElemT, ElemOffset, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }

    return true;
  }

  case CK_HLSLVectorTruncation: {
    assert(SubExpr->getType()->isVectorType());
    if (OptPrimType ResultT = classify(E)) {
      assert(!DiscardResult);
      // Result must be either a float or integer. Take the first element.
      if (!this->visit(SubExpr))
        return false;
      return this->emitArrayElemPop(*ResultT, 0, E);
    }
    // Otherwise, this truncates from one vector type to another.
    assert(E->getType()->isVectorType());

    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateTemporary(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }
    unsigned ToSize = E->getType()->getAs<VectorType>()->getNumElements();
    assert(SubExpr->getType()->getAs<VectorType>()->getNumElements() > ToSize);
    if (!this->visit(SubExpr))
      return false;
    return this->emitCopyArray(classifyVectorElementType(E->getType()), 0, 0,
                               ToSize, E);
  };

  case CK_IntegralToFixedPoint: {
    if (!this->visit(SubExpr))
      return false;

    auto Sem =
        Ctx.getASTContext().getFixedPointSemantics(E->getType()).toOpaqueInt();
    if (!this->emitCastIntegralFixedPoint(classifyPrim(SubExpr->getType()), Sem,
                                          E))
      return false;
    if (DiscardResult)
      return this->emitPopFixedPoint(E);
    return true;
  }
  case CK_FloatingToFixedPoint: {
    if (!this->visit(SubExpr))
      return false;

    auto Sem =
        Ctx.getASTContext().getFixedPointSemantics(E->getType()).toOpaqueInt();
    if (!this->emitCastFloatingFixedPoint(Sem, E))
      return false;
    if (DiscardResult)
      return this->emitPopFixedPoint(E);
    return true;
  }
  case CK_FixedPointToFloating: {
    if (!this->visit(SubExpr))
      return false;
    const auto *TargetSemantics = &Ctx.getFloatSemantics(E->getType());
    if (!this->emitCastFixedPointFloating(TargetSemantics, E))
      return false;
    if (DiscardResult)
      return this->emitPopFloat(E);
    return true;
  }
  case CK_FixedPointToIntegral: {
    if (!this->visit(SubExpr))
      return false;
    PrimType IntegralT = classifyPrim(E->getType());
    if (!this->emitCastFixedPointIntegral(IntegralT, E))
      return false;
    if (DiscardResult)
      return this->emitPop(IntegralT, E);
    return true;
  }
  case CK_FixedPointCast: {
    if (!this->visit(SubExpr))
      return false;
    auto Sem =
        Ctx.getASTContext().getFixedPointSemantics(E->getType()).toOpaqueInt();
    if (!this->emitCastFixedPoint(Sem, E))
      return false;
    if (DiscardResult)
      return this->emitPopFixedPoint(E);
    return true;
  }

  case CK_ToVoid:
    return discard(SubExpr);

  case CK_Dynamic:
    llvm_unreachable("CXXDynamicCastExpr has its own function");

  case CK_LValueBitCast:
    if (!this->emitInvalidCast(CastKind::ReinterpretLike, /*Fatal=*/false, E))
      return false;
    return this->delegate(SubExpr);

  case CK_HLSLArrayRValue: {
    // Non-decaying array rvalue cast - creates an rvalue copy of an lvalue
    // array, similar to LValueToRValue for composite types.
    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }
    if (!this->visit(SubExpr))
      return false;
    return this->emitMemcpy(E);
  }

  case CK_HLSLMatrixTruncation: {
    assert(SubExpr->getType()->isConstantMatrixType());
    if (OptPrimType ResultT = classify(E)) {
      assert(!DiscardResult);
      // Result must be either a float or integer. Take the first element.
      if (!this->visit(SubExpr))
        return false;
      return this->emitArrayElemPop(*ResultT, 0, E);
    }
    // Otherwise, this truncates to a a constant matrix type.
    assert(E->getType()->isConstantMatrixType());

    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateTemporary(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }
    unsigned ToSize =
        E->getType()->getAs<ConstantMatrixType>()->getNumElementsFlattened();
    if (!this->visit(SubExpr))
      return false;
    return this->emitCopyArray(classifyMatrixElementType(SubExpr->getType()), 0,
                               0, ToSize, E);
  }

  case CK_HLSLAggregateSplatCast: {
    // Aggregate splat cast: convert a scalar value to one of an aggregate type
    // by replicating and casting the scalar to every element of the destination
    // aggregate (vector, matrix, array, or struct).
    assert(canClassify(SubExpr->getType()));

    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    // The scalar to be splatted is stored in a local to be repeatedly loaded
    // once for every scalar element of the destination.
    PrimType SrcElemT = classifyPrim(SubExpr->getType());
    unsigned SrcOffset =
        allocateLocalPrimitive(SubExpr, SrcElemT, /*IsConst=*/true);

    if (!this->visit(SubExpr))
      return false;
    if (!this->emitSetLocal(SrcElemT, SrcOffset, E))
      return false;

    // Recursively splat the scalar into every element of the destination.
    return emitHLSLAggregateSplat(SrcElemT, SrcOffset, E->getType(), E);
  }

  case CK_HLSLElementwiseCast: {
    // Elementwise cast: flatten the elements of one aggregate source type and
    // store to a destination scalar or aggregate type of the same or fewer
    // number of elements. Casts are inserted element-wise to convert each
    // source scalar element to its corresponding destination scalar element.
    QualType SrcType = SubExpr->getType();
    QualType DestType = E->getType();

    if (OptPrimType DestT = classify(DestType)) {
      // When the destination is a scalar, we only need the first scalar
      // element of the source.
      unsigned SrcPtrOffset =
          allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
      if (!this->visit(SubExpr))
        return false;
      if (!this->emitSetLocal(PT_Ptr, SrcPtrOffset, E))
        return false;

      SmallVector<HLSLFlatElement, 1> Elements;
      if (!emitHLSLFlattenAggregate(SrcType, SrcPtrOffset, Elements, 1, E))
        return false;
      if (Elements.empty())
        return false;

      const HLSLFlatElement &Src = Elements[0];
      if (!this->emitGetLocal(Src.Type, Src.LocalOffset, E))
        return false;
      return this->emitPrimCast(Src.Type, *DestT, DestType, E);
    }

    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    unsigned SrcOffset =
        allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(SubExpr))
      return false;
    if (!this->emitSetLocal(PT_Ptr, SrcOffset, E))
      return false;

    // Only flatten as many source elements as the destination requires.
    unsigned ElemCount = countHLSLFlatElements(DestType);

    SmallVector<HLSLFlatElement, 16> Elements;
    Elements.reserve(ElemCount);
    if (!emitHLSLFlattenAggregate(SrcType, SrcOffset, Elements, ElemCount, E))
      return false;

    // Sema is expected to reject an elementwise cast whose source has fewer
    // scalar elements than the destination.
    assert(Elements.size() == ElemCount &&
           "Source type has fewer scalar elements than the destination type");

    return emitHLSLConstructAggregate(DestType, Elements, E);
  }

  case CK_ToUnion: {
    const FieldDecl *UnionField = E->getTargetUnionField();
    const Record *R = this->getRecord(E->getType());
    assert(R);
    const Record::Field *RF = R->getField(UnionField);

    if (OptPrimType PT = RF->T) {
      if (!this->visit(SubExpr))
        return false;
      if (RF->isBitField())
        return this->emitInitBitFieldActivate(*PT, RF->Offset, RF->bitWidth(),
                                              E);
      return this->emitInitFieldActivate(*PT, RF->Offset, E);
    }

    if (!this->emitGetPtrField(RF->Offset, E))
      return false;
    if (!this->emitActivate(E))
      return false;
    return this->visitInitializerPop(SubExpr);
  }

  default:
    return this->emitInvalid(E);
  }
  llvm_unreachable("Unhandled clang::CastKind enum");
}

template <class Emitter>
bool Compiler<Emitter>::VisitBuiltinBitCastExpr(const BuiltinBitCastExpr *E) {
  return this->emitBuiltinBitCast(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitIntegerLiteral(const IntegerLiteral *LE) {
  if (DiscardResult)
    return true;

  return this->emitConst(LE->getValue(), LE);
}

template <class Emitter>
bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
  if (DiscardResult)
    return true;

  APFloat F = E->getValue();
  return this->emitFloat(F, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
  assert(E->getType()->isAnyComplexType());
  if (DiscardResult)
    return true;

  if (!Initializing) {
    UnsignedOrNone LocalIndex = allocateTemporary(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  const Expr *SubExpr = E->getSubExpr();
  PrimType SubExprT = classifyPrim(SubExpr->getType());

  if (!this->visitZeroInitializer(SubExprT, SubExpr->getType(), SubExpr))
    return false;
  if (!this->emitInitElem(SubExprT, 0, SubExpr))
    return false;
  return this->visitArrayElemInit(1, SubExpr, SubExprT);
}

template <class Emitter>
bool Compiler<Emitter>::VisitFixedPointLiteral(const FixedPointLiteral *E) {
  assert(E->getType()->isFixedPointType());
  assert(classifyPrim(E) == PT_FixedPoint);

  if (DiscardResult)
    return true;

  auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
  APInt Value = E->getValue();
  return this->emitConstFixedPoint(FixedPoint(Value, Sem), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitParenExpr(const ParenExpr *E) {
  return this->delegate(E->getSubExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitBinaryOperator(const BinaryOperator *E) {
  // Need short-circuiting for these.
  if (E->isLogicalOp() && !E->getType()->isVectorType())
    return this->VisitLogicalBinOp(E);

  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();

  // Handle comma operators. Just discard the LHS
  // and delegate to RHS.
  if (E->isCommaOp()) {
    if (!this->discard(LHS))
      return false;
    if (RHS->getType()->isVoidType())
      return this->discard(RHS);

    return this->delegate(RHS);
  }

  if (E->getType()->isAnyComplexType())
    return this->VisitComplexBinOp(E);
  if (E->getType()->isVectorType())
    return this->VisitVectorBinOp(E);
  if ((LHS->getType()->isAnyComplexType() ||
       RHS->getType()->isAnyComplexType()) &&
      E->isComparisonOp())
    return this->emitComplexComparison(LHS, RHS, E);
  if (LHS->getType()->isFixedPointType() || RHS->getType()->isFixedPointType())
    return this->VisitFixedPointBinOp(E);

  if (E->isPtrMemOp()) {
    if (E->containsErrors())
      return false;

    if (!this->visit(LHS))
      return false;

    if (!this->visit(RHS))
      return false;

    if (!this->emitToMemberPtr(E))
      return false;

    if (classifyPrim(E) == PT_MemberPtr)
      return true;

    if (!this->emitCastMemberPtrPtr(E))
      return false;
    return DiscardResult ? this->emitPopPtr(E) : true;
  }

  // Typecheck the args.
  OptPrimType LT = classify(LHS);
  OptPrimType RT = classify(RHS);
  OptPrimType T = classify(E->getType());

  // Special case for C++'s three-way/spaceship operator <=>, which
  // returns a std::{strong,weak,partial}_ordering (which is a class, so doesn't
  // have a PrimType).
  if (!T && E->getOpcode() == BO_Cmp) {
    if (DiscardResult)
      return true;
    const ComparisonCategoryInfo *CmpInfo =
        Ctx.getASTContext().CompCategories.lookupInfoForType(E->getType());
    assert(CmpInfo);

    // We need a temporary variable holding our return value.
    if (!Initializing) {
      UnsignedOrNone ResultIndex = this->allocateLocal(E);
      if (!this->emitGetPtrLocal(*ResultIndex, E))
        return false;
    }

    if (!visit(LHS) || !visit(RHS))
      return false;

    return this->emitCMP3(*LT, CmpInfo, E);
  }

  if (!LT || !RT || !T)
    return false;

  // Pointer arithmetic special case.
  if (E->getOpcode() == BO_Add || E->getOpcode() == BO_Sub) {
    if (isPtrType(*T) || (isPtrType(*LT) && isPtrType(*RT)))
      return this->VisitPointerArithBinOp(E);
  }

  if (E->getOpcode() == BO_Assign)
    return this->visitAssignment(LHS, RHS, E);

  if (!visit(LHS) || !visit(RHS))
    return false;

  // For languages such as C, cast the result of one
  // of our comparision opcodes to T (which is usually int).
  auto MaybeCastToBool = [this, T, E](bool Result) {
    if (!Result)
      return false;
    if (DiscardResult)
      return this->emitPopBool(E);
    if (T != PT_Bool)
      return this->emitCast(PT_Bool, *T, E);
    return true;
  };

  auto Discard = [this, T, E](bool Result) {
    if (!Result)
      return false;
    return DiscardResult ? this->emitPop(*T, E) : true;
  };

  switch (E->getOpcode()) {
  case BO_EQ:
    return MaybeCastToBool(this->emitEQ(*LT, E));
  case BO_NE:
    return MaybeCastToBool(this->emitNE(*LT, E));
  case BO_LT:
    return MaybeCastToBool(this->emitLT(*LT, E));
  case BO_LE:
    return MaybeCastToBool(this->emitLE(*LT, E));
  case BO_GT:
    return MaybeCastToBool(this->emitGT(*LT, E));
  case BO_GE:
    return MaybeCastToBool(this->emitGE(*LT, E));
  case BO_Sub:
    if (E->getType()->isFloatingType())
      return Discard(this->emitSubf(getFPOptions(E), E));
    return Discard(this->emitSub(*T, E));
  case BO_Add:
    if (E->getType()->isFloatingType())
      return Discard(this->emitAddf(getFPOptions(E), E));
    return Discard(this->emitAdd(*T, E));
  case BO_Mul:
    if (E->getType()->isFloatingType())
      return Discard(this->emitMulf(getFPOptions(E), E));
    return Discard(this->emitMul(*T, E));
  case BO_Rem:
    return Discard(this->emitRem(*T, E));
  case BO_Div:
    if (E->getType()->isFloatingType())
      return Discard(this->emitDivf(getFPOptions(E), E));
    return Discard(this->emitDiv(*T, E));
  case BO_And:
    return Discard(this->emitBitAnd(*T, E));
  case BO_Or:
    return Discard(this->emitBitOr(*T, E));
  case BO_Shl:
    return Discard(this->emitShl(*LT, *RT, E));
  case BO_Shr:
    return Discard(this->emitShr(*LT, *RT, E));
  case BO_Xor:
    return Discard(this->emitBitXor(*T, E));
  case BO_LOr:
  case BO_LAnd:
    llvm_unreachable("Already handled earlier");
  default:
    return false;
  }

  llvm_unreachable("Unhandled binary op");
}

/// Perform addition/subtraction of a pointer and an integer or
/// subtraction of two pointers.
template <class Emitter>
bool Compiler<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
  BinaryOperatorKind Op = E->getOpcode();
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();

  if ((Op != BO_Add && Op != BO_Sub) ||
      (!LHS->getType()->isPointerType() && !RHS->getType()->isPointerType()))
    return false;

  OptPrimType LT = classify(LHS);
  OptPrimType RT = classify(RHS);

  if (!LT || !RT)
    return false;

  // Visit the given pointer expression and optionally convert to a PT_Ptr.
  auto visitAsPointer = [&](const Expr *E, PrimType T) -> bool {
    if (!this->visit(E))
      return false;
    if (T != PT_Ptr)
      return this->emitDecayPtr(T, PT_Ptr, E);
    return true;
  };

  if (LHS->getType()->isPointerType() && RHS->getType()->isPointerType()) {
    if (Op != BO_Sub)
      return false;

    assert(E->getType()->isIntegerType());
    if (!visitAsPointer(RHS, *RT) || !visitAsPointer(LHS, *LT))
      return false;

    QualType ElemType = LHS->getType()->getPointeeType();
    CharUnits ElemTypeSize;
    if (ElemType->isVoidType() || ElemType->isFunctionType())
      ElemTypeSize = CharUnits::One();
    else
      ElemTypeSize = Ctx.getASTContext().getTypeSizeInChars(ElemType);

    PrimType IntT = classifyPrim(E->getType());
    if (!this->emitSubPtr(IntT, ElemTypeSize.getQuantity(), E))
      return false;
    return DiscardResult ? this->emitPop(IntT, E) : true;
  }

  PrimType OffsetType;
  if (LHS->getType()->isIntegerType()) {
    if (!visitAsPointer(RHS, *RT))
      return false;
    if (!this->visit(LHS))
      return false;
    OffsetType = *LT;
  } else if (RHS->getType()->isIntegerType()) {
    if (!visitAsPointer(LHS, *LT))
      return false;
    if (!this->visit(RHS))
      return false;
    OffsetType = *RT;
  } else {
    return false;
  }

  // Do the operation and optionally transform to
  // result pointer type.
  switch (Op) {
  case BO_Add:
    if (!this->emitAddOffset(OffsetType, E))
      return false;
    break;
  case BO_Sub:
    if (!this->emitSubOffset(OffsetType, E))
      return false;
    break;
  default:
    return false;
  }

  PrimType ExprT = classifyPrim(E);
  if (ExprT != PT_Ptr) {
    if (!this->emitDecayPtr(PT_Ptr, ExprT, E))
      return false;
  }

  if (DiscardResult)
    return this->emitPop(ExprT, E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
  assert(E->isLogicalOp());
  BinaryOperatorKind Op = E->getOpcode();
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();

  if (Op == BO_LOr) {
    // Logical OR. Visit LHS and only evaluate RHS if LHS was FALSE.
    LabelTy LabelTrue = this->getLabel();
    LabelTy LabelEnd = this->getLabel();

    if (!this->visitBool(LHS))
      return false;
    if (!this->jumpTrue(LabelTrue, E))
      return false;

    if (!this->visitBool(RHS))
      return false;
    if (!this->jump(LabelEnd, E))
      return false;

    this->emitLabel(LabelTrue);
    this->emitConstBool(true, E);
    this->fallthrough(LabelEnd);
    this->emitLabel(LabelEnd);

  } else {
    assert(Op == BO_LAnd);
    // Logical AND.
    // Visit LHS. Only visit RHS if LHS was TRUE.
    LabelTy LabelFalse = this->getLabel();
    LabelTy LabelEnd = this->getLabel();

    if (!this->visitBool(LHS))
      return false;
    if (!this->jumpFalse(LabelFalse, E))
      return false;

    if (!this->visitBool(RHS))
      return false;
    if (!this->jump(LabelEnd, E))
      return false;

    this->emitLabel(LabelFalse);
    this->emitConstBool(false, E);
    this->fallthrough(LabelEnd);
    this->emitLabel(LabelEnd);
  }

  if (DiscardResult)
    return this->emitPopBool(E);

  // For C, cast back to integer type.
  if (!E->getType()->isBooleanType()) {
    PrimType T = classifyPrim(E->getType());
    return this->emitCast(PT_Bool, T, E);
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
  // Prepare storage for result.
  if (!Initializing) {
    UnsignedOrNone LocalIndex = allocateTemporary(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  // Both LHS and RHS might _not_ be of complex type, but one of them
  // needs to be.
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();

  PrimType ResultElemT = this->classifyComplexElementType(E->getType());
  unsigned ResultOffset = ~0u;
  if (!DiscardResult)
    ResultOffset = this->allocateLocalPrimitive(E, PT_Ptr, /*IsConst=*/true);

  // Save result pointer in ResultOffset
  if (!this->DiscardResult) {
    if (!this->emitDupPtr(E))
      return false;
    if (!this->emitSetLocal(PT_Ptr, ResultOffset, E))
      return false;
  }
  QualType LHSType = LHS->getType();
  if (const auto *AT = LHSType->getAs<AtomicType>())
    LHSType = AT->getValueType();
  QualType RHSType = RHS->getType();
  if (const auto *AT = RHSType->getAs<AtomicType>())
    RHSType = AT->getValueType();

  bool LHSIsComplex = LHSType->isAnyComplexType();
  unsigned LHSOffset;
  bool RHSIsComplex = RHSType->isAnyComplexType();

  // For ComplexComplex Mul, we have special ops to make their implementation
  // easier.
  BinaryOperatorKind Op = E->getOpcode();
  if (Op == BO_Mul && LHSIsComplex && RHSIsComplex) {
    assert(classifyPrim(LHSType->getAs<ComplexType>()->getElementType()) ==
           classifyPrim(RHSType->getAs<ComplexType>()->getElementType()));
    PrimType ElemT =
        classifyPrim(LHSType->getAs<ComplexType>()->getElementType());
    if (!this->visit(LHS))
      return false;
    if (!this->visit(RHS))
      return false;
    if (!this->emitMulc(ElemT, E))
      return false;
    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  }

  if (Op == BO_Div && RHSIsComplex) {
    QualType ElemQT = RHSType->getAs<ComplexType>()->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);
    // If the LHS is not complex, we still need to do the full complex
    // division, so just stub create a complex value and stub it out with
    // the LHS and a zero.

    if (!LHSIsComplex) {
      // This is using the RHS type for the fake-complex LHS.
      UnsignedOrNone LocalIndex = allocateTemporary(RHS);
      if (!LocalIndex)
        return false;
      LHSOffset = *LocalIndex;

      if (!this->emitGetPtrLocal(LHSOffset, E))
        return false;

      if (!this->visit(LHS))
        return false;
      // real is LHS
      if (!this->emitInitElem(ElemT, 0, E))
        return false;
      // imag is zero
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, 1, E))
        return false;
    } else {
      if (!this->visit(LHS))
        return false;
    }

    if (!this->visit(RHS))
      return false;
    if (!this->emitDivc(ElemT, E))
      return false;
    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  }

  // Evaluate LHS and save value to LHSOffset.
  if (LHSType->isAnyComplexType()) {
    LHSOffset = this->allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(LHS))
      return false;
    if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
      return false;
  } else {
    PrimType LHST = classifyPrim(LHSType);
    LHSOffset = this->allocateLocalPrimitive(LHS, LHST, /*IsConst=*/true);
    if (!this->visit(LHS))
      return false;
    if (!this->emitSetLocal(LHST, LHSOffset, E))
      return false;
  }

  // Same with RHS.
  unsigned RHSOffset;
  if (RHSType->isAnyComplexType()) {
    RHSOffset = this->allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(RHS))
      return false;
    if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
      return false;
  } else {
    PrimType RHST = classifyPrim(RHSType);
    RHSOffset = this->allocateLocalPrimitive(RHS, RHST, /*IsConst=*/true);
    if (!this->visit(RHS))
      return false;
    if (!this->emitSetLocal(RHST, RHSOffset, E))
      return false;
  }

  // For both LHS and RHS, either load the value from the complex pointer, or
  // directly from the local variable. For index 1 (i.e. the imaginary part),
  // just load 0 and do the operation anyway.
  auto loadComplexValue = [this](bool IsComplex, bool LoadZero,
                                 unsigned ElemIndex, unsigned Offset,
                                 const Expr *E) -> bool {
    if (IsComplex) {
      if (!this->emitGetLocal(PT_Ptr, Offset, E))
        return false;
      return this->emitArrayElemPop(classifyComplexElementType(E->getType()),
                                    ElemIndex, E);
    }
    if (ElemIndex == 0 || !LoadZero)
      return this->emitGetLocal(classifyPrim(E->getType()), Offset, E);
    return this->visitZeroInitializer(classifyPrim(E->getType()), E->getType(),
                                      E);
  };

  // Now we can get pointers to the LHS and RHS from the offsets above.
  for (unsigned ElemIndex = 0; ElemIndex != 2; ++ElemIndex) {
    // Result pointer for the store later.
    if (!this->DiscardResult) {
      if (!this->emitGetLocal(PT_Ptr, ResultOffset, E))
        return false;
    }

    // The actual operation.
    switch (Op) {
    case BO_Add:
      if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
        return false;

      if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
        return false;
      if (ResultElemT == PT_Float) {
        if (!this->emitAddf(getFPOptions(E), E))
          return false;
      } else {
        if (!this->emitAdd(ResultElemT, E))
          return false;
      }
      break;
    case BO_Sub:
      if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
        return false;

      if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
        return false;
      if (ResultElemT == PT_Float) {
        if (!this->emitSubf(getFPOptions(E), E))
          return false;
      } else {
        if (!this->emitSub(ResultElemT, E))
          return false;
      }
      break;
    case BO_Mul:
      if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
        return false;

      if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
        return false;

      if (ResultElemT == PT_Float) {
        if (!this->emitMulf(getFPOptions(E), E))
          return false;
      } else {
        if (!this->emitMul(ResultElemT, E))
          return false;
      }
      break;
    case BO_Div:
      assert(!RHSIsComplex);
      if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
        return false;

      if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
        return false;

      if (ResultElemT == PT_Float) {
        if (!this->emitDivf(getFPOptions(E), E))
          return false;
      } else {
        if (!this->emitDiv(ResultElemT, E))
          return false;
      }
      break;

    default:
      return false;
    }

    if (!this->DiscardResult) {
      // Initialize array element with the value we just computed.
      if (!this->emitInitElemPop(ResultElemT, ElemIndex, E))
        return false;
    } else {
      if (!this->emitPop(ResultElemT, E))
        return false;
      // Remove the Complex temporary pointer we created ourselves at the
      // beginning of this function.
      if (!Initializing)
        return this->emitPopPtr(E);
    }
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  assert(!E->isCommaOp() &&
         "Comma op should be handled in VisitBinaryOperator");

  QualType LHSType = LHS->getType();
  if (const auto *AT = LHSType->getAs<AtomicType>())
    LHSType = AT->getValueType();
  QualType RHSType = RHS->getType();
  if (const auto *AT = RHSType->getAs<AtomicType>())
    RHSType = AT->getValueType();
  assert(E->getType()->isVectorType());
  assert(LHSType->isVectorType());
  assert(RHSType->isVectorType());

  // We can only handle vectors with primitive element types.
  if (!canClassify(LHSType->castAs<VectorType>()->getElementType()))
    return false;

  // Prepare storage for result.
  if (!Initializing && !E->isCompoundAssignmentOp() && !E->isAssignmentOp()) {
    UnsignedOrNone LocalIndex = allocateTemporary(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  const auto *VecTy = E->getType()->getAs<VectorType>();
  auto Op = E->isCompoundAssignmentOp()
                ? BinaryOperator::getOpForCompoundAssignment(E->getOpcode())
                : E->getOpcode();

  PrimType ElemT = this->classifyVectorElementType(LHSType);
  PrimType RHSElemT = this->classifyVectorElementType(RHSType);
  PrimType ResultElemT = this->classifyVectorElementType(E->getType());

  if (E->getOpcode() == BO_Assign) {
    assert(Ctx.getASTContext().hasSameUnqualifiedType(
        LHSType->castAs<VectorType>()->getElementType(),
        RHSType->castAs<VectorType>()->getElementType()));
    if (!this->visit(LHS))
      return false;
    if (!this->visit(RHS))
      return false;
    if (!this->emitCopyArray(ElemT, 0, 0, VecTy->getNumElements(), E))
      return false;
    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  }

  // Evaluate LHS and save value to LHSOffset.
  unsigned LHSOffset =
      this->allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true);
  if (!this->visit(LHS))
    return false;
  if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
    return false;

  // Evaluate RHS and save value to RHSOffset.
  unsigned RHSOffset =
      this->allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true);
  if (!this->visit(RHS))
    return false;
  if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
    return false;

  if (E->isCompoundAssignmentOp() && !this->emitGetLocal(PT_Ptr, LHSOffset, E))
    return false;

  // BitAdd/BitOr/BitXor/Shl/Shr doesn't support bool type, we need perform the
  // integer promotion.
  bool NeedIntPromot = ElemT == PT_Bool && (E->isBitwiseOp() || E->isShiftOp());
  QualType PromotTy;
  PrimType PromotT = PT_Bool;
  PrimType OpT = ElemT;
  if (NeedIntPromot) {
    PromotTy =
        Ctx.getASTContext().getPromotedIntegerType(Ctx.getASTContext().BoolTy);
    PromotT = classifyPrim(PromotTy);
    OpT = PromotT;
  }

  auto getElem = [=](unsigned Offset, PrimType ElemT, unsigned Index) {
    if (!this->emitGetLocal(PT_Ptr, Offset, E))
      return false;
    if (!this->emitArrayElemPop(ElemT, Index, E))
      return false;
    if (E->isLogicalOp()) {
      if (!this->emitPrimCast(ElemT, PT_Bool, Ctx.getASTContext().BoolTy, E))
        return false;
      if (!this->emitPrimCast(PT_Bool, ResultElemT, VecTy->getElementType(), E))
        return false;
    } else if (NeedIntPromot) {
      if (!this->emitPrimCast(ElemT, PromotT, PromotTy, E))
        return false;
    }
    return true;
  };

#define EMIT_ARITH_OP(OP)                                                      \
  {                                                                            \
    if (ElemT == PT_Float) {                                                   \
      if (!this->emit##OP##f(getFPOptions(E), E))                              \
        return false;                                                          \
    } else {                                                                   \
      if (!this->emit##OP(ElemT, E))                                           \
        return false;                                                          \
    }                                                                          \
    break;                                                                     \
  }

  for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
    if (!getElem(LHSOffset, ElemT, I))
      return false;
    if (!getElem(RHSOffset, RHSElemT, I))
      return false;
    switch (Op) {
    case BO_Add:
      EMIT_ARITH_OP(Add)
    case BO_Sub:
      EMIT_ARITH_OP(Sub)
    case BO_Mul:
      EMIT_ARITH_OP(Mul)
    case BO_Div:
      EMIT_ARITH_OP(Div)
    case BO_Rem:
      if (!this->emitRem(ElemT, E))
        return false;
      break;
    case BO_And:
      if (!this->emitBitAnd(OpT, E))
        return false;
      break;
    case BO_Or:
      if (!this->emitBitOr(OpT, E))
        return false;
      break;
    case BO_Xor:
      if (!this->emitBitXor(OpT, E))
        return false;
      break;
    case BO_Shl:
      if (!this->emitShl(OpT, RHSElemT, E))
        return false;
      break;
    case BO_Shr:
      if (!this->emitShr(OpT, RHSElemT, E))
        return false;
      break;
    case BO_EQ:
      if (!this->emitEQ(ElemT, E))
        return false;
      break;
    case BO_NE:
      if (!this->emitNE(ElemT, E))
        return false;
      break;
    case BO_LE:
      if (!this->emitLE(ElemT, E))
        return false;
      break;
    case BO_LT:
      if (!this->emitLT(ElemT, E))
        return false;
      break;
    case BO_GE:
      if (!this->emitGE(ElemT, E))
        return false;
      break;
    case BO_GT:
      if (!this->emitGT(ElemT, E))
        return false;
      break;
    case BO_LAnd:
      // a && b is equivalent to a!=0 & b!=0
      if (!this->emitBitAnd(ResultElemT, E))
        return false;
      break;
    case BO_LOr:
      // a || b is equivalent to a!=0 | b!=0
      if (!this->emitBitOr(ResultElemT, E))
        return false;
      break;
    default:
      return this->emitInvalid(E);
    }

    // The result of the comparison is a vector of the same width and number
    // of elements as the comparison operands with a signed integral element
    // type.
    //
    // https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
    if (E->isComparisonOp()) {
      if (!this->emitPrimCast(PT_Bool, ResultElemT, VecTy->getElementType(), E))
        return false;
      if (!this->emitNeg(ResultElemT, E))
        return false;
    }

    // If we performed an integer promotion, we need to cast the compute result
    // into result vector element type.
    if (NeedIntPromot &&
        !this->emitPrimCast(PromotT, ResultElemT, VecTy->getElementType(), E))
      return false;

    // Initialize array element with the value we just computed.
    if (!this->emitInitElem(ResultElemT, I, E))
      return false;
  }

  if (DiscardResult && E->isCompoundAssignmentOp() && !this->emitPopPtr(E))
    return false;
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitFixedPointBinOp(const BinaryOperator *E) {
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  const ASTContext &ASTCtx = Ctx.getASTContext();

  assert(LHS->getType()->isFixedPointType() ||
         RHS->getType()->isFixedPointType());

  auto LHSSema = ASTCtx.getFixedPointSemantics(LHS->getType());
  auto LHSSemaInt = LHSSema.toOpaqueInt();
  auto RHSSema = ASTCtx.getFixedPointSemantics(RHS->getType());
  auto RHSSemaInt = RHSSema.toOpaqueInt();

  if (!this->visit(LHS))
    return false;
  if (!LHS->getType()->isFixedPointType()) {
    if (!this->emitCastIntegralFixedPoint(classifyPrim(LHS->getType()),
                                          LHSSemaInt, E))
      return false;
  }

  if (!this->visit(RHS))
    return false;
  if (!RHS->getType()->isFixedPointType()) {
    if (!this->emitCastIntegralFixedPoint(classifyPrim(RHS->getType()),
                                          RHSSemaInt, E))
      return false;
  }

  // Convert the result to the target semantics.
  auto ConvertResult = [&](bool R) -> bool {
    if (!R)
      return false;
    auto ResultSema = ASTCtx.getFixedPointSemantics(E->getType()).toOpaqueInt();
    auto CommonSema = LHSSema.getCommonSemantics(RHSSema).toOpaqueInt();
    if (ResultSema != CommonSema)
      return this->emitCastFixedPoint(ResultSema, E);
    return true;
  };

  auto MaybeCastToBool = [&](bool Result) {
    if (!Result)
      return false;
    PrimType T = classifyPrim(E);
    if (DiscardResult)
      return this->emitPop(T, E);
    if (T != PT_Bool)
      return this->emitCast(PT_Bool, T, E);
    return true;
  };

  switch (E->getOpcode()) {
  case BO_EQ:
    return MaybeCastToBool(this->emitEQFixedPoint(E));
  case BO_NE:
    return MaybeCastToBool(this->emitNEFixedPoint(E));
  case BO_LT:
    return MaybeCastToBool(this->emitLTFixedPoint(E));
  case BO_LE:
    return MaybeCastToBool(this->emitLEFixedPoint(E));
  case BO_GT:
    return MaybeCastToBool(this->emitGTFixedPoint(E));
  case BO_GE:
    return MaybeCastToBool(this->emitGEFixedPoint(E));
  case BO_Add:
    return ConvertResult(this->emitAddFixedPoint(E));
  case BO_Sub:
    return ConvertResult(this->emitSubFixedPoint(E));
  case BO_Mul:
    return ConvertResult(this->emitMulFixedPoint(E));
  case BO_Div:
    return ConvertResult(this->emitDivFixedPoint(E));
  case BO_Shl:
    return ConvertResult(this->emitShiftFixedPoint(/*Left=*/true, E));
  case BO_Shr:
    return ConvertResult(this->emitShiftFixedPoint(/*Left=*/false, E));

  default:
    return this->emitInvalid(E);
  }

  llvm_unreachable("unhandled binop opcode");
}

template <class Emitter>
bool Compiler<Emitter>::VisitFixedPointUnaryOperator(const UnaryOperator *E) {
  const Expr *SubExpr = E->getSubExpr();
  assert(SubExpr->getType()->isFixedPointType());

  switch (E->getOpcode()) {
  case UO_Plus:
    return this->delegate(SubExpr);
  case UO_Minus:
    if (!this->visit(SubExpr))
      return false;
    if (!this->emitNegFixedPoint(E))
      return false;
    if (DiscardResult)
      return this->emitPopFixedPoint(E);
    return true;
  default:
    return false;
  }

  llvm_unreachable("Unhandled unary opcode");
}

template <class Emitter>
bool Compiler<Emitter>::VisitImplicitValueInitExpr(
    const ImplicitValueInitExpr *E) {
  if (DiscardResult)
    return true;

  QualType QT = E->getType();

  if (OptPrimType T = classify(QT))
    return this->visitZeroInitializer(*T, QT, E);

  if (QT->isRecordType()) {
    const RecordDecl *RD = QT->getAsRecordDecl();
    assert(RD);
    if (RD->isInvalidDecl())
      return false;

    const Record *R = getRecord(QT);
    if (!R)
      return false;

    assert(Initializing);
    return this->visitZeroRecordInitializer(R, E);
  }

  if (QT->isIncompleteArrayType())
    return true;

  if (QT->isArrayType())
    return this->visitZeroArrayInitializer(QT, E);

  if (const auto *ComplexTy = E->getType()->getAs<ComplexType>()) {
    assert(Initializing);
    QualType ElemQT = ComplexTy->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);
    for (unsigned I = 0; I < 2; ++I) {
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  if (const auto *VecT = E->getType()->getAs<VectorType>()) {
    unsigned NumVecElements = VecT->getNumElements();
    QualType ElemQT = VecT->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);

    for (unsigned I = 0; I < NumVecElements; ++I) {
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  if (const auto *MT = E->getType()->getAs<ConstantMatrixType>()) {
    unsigned NumElems = MT->getNumElementsFlattened();
    QualType ElemQT = MT->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);

    for (unsigned I = 0; I != NumElems; ++I) {
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
  if (E->getType()->isVoidType() || E->containsErrors())
    return false;

  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  const Expr *Index = E->getIdx();
  const Expr *Base = E->getBase();

  // C++17's rules require us to evaluate the LHS first, regardless of which
  // side is the base.
  bool Success = true;
  for (const Expr *SubExpr : {LHS, RHS}) {
    if (!this->visit(SubExpr)) {
      Success = false;
      continue;
    }

    // Expand the base if this is a subscript on a
    // pointer expression.
    if (SubExpr == Base && Base->getType()->isPointerType()) {
      if (!this->emitExpandPtr(E))
        Success = false;
    }
  }

  if (!Success)
    return false;

  OptPrimType IndexT = classify(Index->getType());
  // In error-recovery cases, the index expression has a dependent type.
  if (!IndexT)
    return this->emitError(E);
  // If the index is first, we need to change that.
  if (LHS == Index) {
    if (!this->emitFlip(PT_Ptr, *IndexT, E))
      return false;
  }

  if (!this->emitArrayElemPtrPop(*IndexT, E))
    return false;
  if (DiscardResult)
    return this->emitPopPtr(E);

  if (E->isGLValue())
    return true;

  OptPrimType T = classifyPrim(E);
  return this->emitLoadPop(*T, E);
}

template <class Emitter>
bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
                                      const Expr *ArrayFiller, const Expr *E) {
  InitLinkScope<Emitter> ILS(this, InitLink::InitList());

  QualType QT = E->getType();
  if (const auto *AT = QT->getAs<AtomicType>())
    QT = AT->getValueType();

  if (QT->isVoidType()) {
    if (Inits.size() == 0)
      return true;
    return this->emitInvalid(E);
  }

  // Primitive values. A discarded one can simply discard each initializer;
  // there is no object to establish.
  if (OptPrimType T = classify(QT)) {
    if (DiscardResult) {
      for (const Expr *Init : Inits) {
        if (!this->discard(Init))
          return false;
      }
      return true;
    }
    if (Inits.size() == 0)
      return this->visitZeroInitializer(*T, QT, E);
    assert(Inits.size() == 1);
    return this->delegate(Inits[0]);
  }

  assert(!canClassify(E->getType()));

  // A composite prvalue needs somewhere to live even when it is discarded: a
  // default member initializer may read subobjects initialized earlier in this
  // same list, so those have to actually be written and `this` has to denote
  // the object. Materialize one and initialize into it.
  if (DiscardResult && !Initializing) {
    UnsignedOrNone LocalIndex = allocateLocal(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
    InitLinkScope<Emitter> ILS2(this, InitLink::Temp(*LocalIndex));
    return this->visitInitializerPop(E);
  }

  if (QT->isRecordType()) {
    const Record *R = getRecord(QT);

    if (Inits.size() == 1 && E->getType() == Inits[0]->getType())
      return this->delegate(Inits[0]);

    if (!R)
      return false;

    auto initPrimitiveField = [=](const Record::Field *FieldToInit,
                                  const Expr *Init, PrimType T,
                                  bool Activate = false) -> bool {
      bool DefaultInit = isa<CXXDefaultInitExpr>(Init);
      InitStackScope<Emitter> ISS(this, DefaultInit);

      if (DefaultInit && !this->emitStartFieldInit(FieldToInit->Offset, Init))
        return false;

      if (!this->visit(Init))
        return false;

      if (DefaultInit && !this->emitEndInit(Init))
        return false;

      bool BitField = FieldToInit->isBitField();
      if (BitField && Activate)
        return this->emitInitBitFieldActivate(T, FieldToInit->Offset,
                                              FieldToInit->bitWidth(), E);
      if (BitField)
        return this->emitInitBitField(T, FieldToInit->Offset,
                                      FieldToInit->bitWidth(), E);
      if (Activate)
        return this->emitInitFieldActivate(T, FieldToInit->Offset, E);
      return this->emitInitField(T, FieldToInit->Offset, E);
    };

    auto initCompositeField = [=](const Record::Field *FieldToInit,
                                  const Expr *Init,
                                  bool Activate = false) -> bool {
      InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Init));
      InitLinkScope<Emitter> ILS(this, InitLink::Field(FieldToInit->Offset));

      // Non-primitive case. Get a pointer to the field-to-initialize
      // on the stack and recurse into visitInitializer().
      if (!this->emitGetPtrField(FieldToInit->Offset, Init))
        return false;

      if (Activate && !this->emitActivate(E))
        return false;

      if (!this->emitStartInit(Init))
        return false;

      return this->visitInitializerPop(Init) && this->emitEndInit(Init);
    };

    if (R->isUnion()) {
      if (Inits.size() == 0) {
        if (!this->visitZeroRecordInitializer(R, E))
          return false;
      } else {
        const Expr *Init = Inits[0];
        const FieldDecl *FToInit = nullptr;
        if (const auto *ILE = dyn_cast<InitListExpr>(E))
          FToInit = ILE->getInitializedFieldInUnion();
        else
          FToInit = cast<CXXParenListInitExpr>(E)->getInitializedFieldInUnion();

        const Record::Field *FieldToInit = R->getField(FToInit);
        if (OptPrimType T = classify(Init)) {
          if (!initPrimitiveField(FieldToInit, Init, *T, /*Activate=*/true))
            return false;
        } else {
          if (!initCompositeField(FieldToInit, Init, /*Activate=*/true))
            return false;
        }
      }
      return this->emitFinishInit(E);
    }

    assert(!R->isUnion());
    for (unsigned BI = 0; BI != R->getNumBases(); ++BI) {
      const Expr *Init = Inits[BI];
      const Record::Base *B = R->getBase(BI);
      InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Init));
      InitLinkScope<Emitter> ILS(this, InitLink::Base(B->Offset));
      if (!this->emitGetPtrBase(B->Offset, Init))
        return false;
      if (!this->visitInitializerPop(Init))
        return false;
    }

    unsigned FieldIndex = 0;
    for (unsigned FI = R->getNumBases(); FI != Inits.size();) {
      const Record::Field *FieldToInit = R->getField(FieldIndex);
      if (FieldToInit->isUnnamedBitField()) {
        ++FieldIndex;
        continue;
      }

      const Expr *Init = Inits[FI];
      // If this is a child of a DesignatedInitUpdateExpr, skip elements which
      // aren't supposed to be modified.
      if (isa<NoInitExpr>(Init)) {
        ++FieldIndex;
        ++FI;
        continue;
      }

      if (OptPrimType T = classify(Init)) {
        if (!initPrimitiveField(FieldToInit, Init, *T))
          return false;
      } else if (!initCompositeField(FieldToInit, Init)) {
        return false;
      }

      ++FI;
      ++FieldIndex;
    }

    assert(R->getNumVirtualBases() == 0);

    return this->emitFinishInit(E);
  }

  if (QT->isArrayType()) {
    const ConstantArrayType *CAT =
        Ctx.getASTContext().getAsConstantArrayType(QT);
    uint64_t NumElems = CAT->getZExtSize();

    if (Initializing &&
        (!InitializingDecl || InitializingDecl->hasLocalStorage()) &&
        !this->emitCheckArrayDestSize(NumElems, E))
      return false;

    if (Inits.size() == 1 && QT == Inits[0]->getType())
      return this->delegate(Inits[0]);

    OptPrimType InitT = classify(CAT->getElementType());
    unsigned ElementIndex = 0;
    for (const Expr *Init : Inits) {
      if (const auto *EmbedS =
              dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) {
        PrimType TargetT = classifyPrim(Init->getType());

        auto Eval = [&](const IntegerLiteral *IL, unsigned ElemIndex) {
          if (TargetT == PT_Float) {
            if (!this->emitConst(IL->getValue(), classifyPrim(IL), Init))
              return false;
            const auto *Sem = &Ctx.getFloatSemantics(CAT->getElementType());
            if (!this->emitCastIntegralFloating(classifyPrim(IL), Sem,
                                                getFPOptions(E), E))
              return false;
          } else {
            if (!this->emitConst(IL->getValue(), TargetT, Init))
              return false;
          }
          return this->emitInitElem(TargetT, ElemIndex, IL);
        };
        if (!EmbedS->doForEachDataElement(Eval, ElementIndex))
          return false;
      } else if (isa<NoInitExpr>(Init)) {
        // If this is a child of a DesignatedInitUpdateExpr, skip elements which
        // aren't supposed to be modified.
        ++ElementIndex;
      } else {
        if (!this->visitArrayElemInit(ElementIndex, Init, InitT))
          return false;
        ++ElementIndex;
      }
    }

    // Expand the filler expression.
    // FIXME: This should go away.
    if (ArrayFiller && !isa<NoInitExpr>(ArrayFiller)) {
      for (; ElementIndex != NumElems; ++ElementIndex) {
        if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT))
          return false;
      }
    }

    return this->emitFinishInit(E);
  }

  if (const auto *ComplexTy = QT->getAs<ComplexType>()) {
    unsigned NumInits = Inits.size();

    if (NumInits == 1)
      return this->delegate(Inits[0]);

    QualType ElemQT = ComplexTy->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);
    if (NumInits == 0) {
      // Zero-initialize both elements.
      for (unsigned I = 0; I < 2; ++I) {
        if (!this->visitZeroInitializer(ElemT, ElemQT, E))
          return false;
        if (!this->emitInitElem(ElemT, I, E))
          return false;
      }
    } else if (NumInits == 2) {
      unsigned InitIndex = 0;
      for (const Expr *Init : Inits) {
        if (!this->visit(Init))
          return false;

        if (!this->emitInitElem(ElemT, InitIndex, E))
          return false;
        ++InitIndex;
      }
    }
    return true;
  }

  if (const auto *VecT = QT->getAs<VectorType>()) {
    unsigned NumVecElements = VecT->getNumElements();
    assert(NumVecElements >= Inits.size());

    QualType ElemQT = VecT->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);

    // All initializer elements.
    unsigned InitIndex = 0;
    for (const Expr *Init : Inits) {
      if (!this->visit(Init))
        return false;

      // If the initializer is of vector type itself, we have to deconstruct
      // that and initialize all the target fields from the initializer fields.
      if (const auto *InitVecT = Init->getType()->getAs<VectorType>()) {
        if (!this->emitCopyArray(ElemT, 0, InitIndex,
                                 InitVecT->getNumElements(), E))
          return false;
        InitIndex += InitVecT->getNumElements();
      } else {
        if (!this->emitInitElem(ElemT, InitIndex, E))
          return false;
        ++InitIndex;
      }
    }

    assert(InitIndex <= NumVecElements);

    // Fill the rest with zeroes.
    for (; InitIndex != NumVecElements; ++InitIndex) {
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, InitIndex, E))
        return false;
    }
    return true;
  }

  if (const auto *MT = QT->getAs<ConstantMatrixType>()) {
    unsigned NumElems = MT->getNumElementsFlattened();
    assert(Inits.size() == NumElems);

    QualType ElemQT = MT->getElementType();
    PrimType ElemT = classifyPrim(ElemQT);

    // Matrix initializer list elements are in row-major order, which matches
    // the matrix APValue convention and therefore no index remapping is
    // required.
    for (unsigned I = 0; I != NumElems; ++I) {
      if (!this->visit(Inits[I]))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  return false;
}

/// Pointer to the array(not the element!) must be on the stack when calling
/// this.
template <class Emitter>
bool Compiler<Emitter>::visitArrayElemInit(unsigned ElemIndex, const Expr *Init,
                                           OptPrimType InitT) {
  if (InitT) {
    // Visit the primitive element like normal.
    if (!this->visit(Init))
      return false;
    return this->emitInitElem(*InitT, ElemIndex, Init);
  }

  InitLinkScope<Emitter> ILS(this, InitLink::Elem(ElemIndex));
  // Advance the pointer currently on the stack to the given
  // dimension.
  if (!this->emitConstUint32(ElemIndex, Init))
    return false;
  if (!this->emitArrayElemPtrUint32(Init))
    return false;
  return this->visitInitializerPop(Init);
}

template <class Emitter>
bool Compiler<Emitter>::visitCallArgs(ArrayRef<const Expr *> Args,
                                      const FunctionDecl *FuncDecl,
                                      bool Activate, bool IsOperatorCall) {
  assert(VarScope->getKind() == ScopeKind::Call);
  llvm::BitVector NonNullArgs;
  if (FuncDecl && FuncDecl->hasAttr<NonNullAttr>())
    NonNullArgs = collectNonNullArgs(FuncDecl, Args);

  bool ExplicitMemberFn = false;
  if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(FuncDecl))
    ExplicitMemberFn = MD->isExplicitObjectMemberFunction();

  unsigned ArgIndex = 0;
  for (const Expr *Arg : Args) {
    if (canClassify(Arg)) {
      if (!this->visit(Arg))
        return false;
    } else {

      DeclOrExpr Source = Arg;
      if (FuncDecl) {
        // Try to use the parameter declaration instead of the argument
        // expression as a source.
        unsigned DeclIndex = ArgIndex - IsOperatorCall + ExplicitMemberFn;
        if (DeclIndex < FuncDecl->getNumParams())
          Source = FuncDecl->getParamDecl(ArgIndex - IsOperatorCall +
                                          ExplicitMemberFn);
      }

      UnsignedOrNone LocalIndex =
          allocateLocal(std::move(Source), Arg->getType(), ScopeKind::Call);
      if (!LocalIndex)
        return false;

      if (!this->emitGetPtrLocal(*LocalIndex, Arg))
        return false;
      InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalIndex));
      if (!this->visitInitializer(Arg))
        return false;
    }

    if (ArgIndex == 1 && Activate) {
      if (!this->emitActivate(Arg))
        return false;
    }

    if (!NonNullArgs.empty() && NonNullArgs[ArgIndex]) {
      PrimType ArgT = classify(Arg).value_or(PT_Ptr);
      if (ArgT == PT_Ptr) {
        if (!this->emitCheckNonNullArg(ArgT, Arg))
          return false;
      }
    }

    ++ArgIndex;
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitInitListExpr(const InitListExpr *E) {
  return this->visitInitList(E->inits(), E->getArrayFiller(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXParenListInitExpr(
    const CXXParenListInitExpr *E) {
  return this->visitInitList(E->getInitExprs(), E->getArrayFiller(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitSubstNonTypeTemplateParmExpr(
    const SubstNonTypeTemplateParmExpr *E) {
  return this->delegate(E->getReplacement());
}

template <class Emitter>
bool Compiler<Emitter>::VisitConstantExpr(const ConstantExpr *E) {
  if (!E->hasAPValueResult())
    return this->delegate(E->getSubExpr());

  if (OptPrimType T = classify(E)) {
    // Try to emit the APValue directly, without visiting the subexpr.
    // This will only fail if we can't emit the APValue, so won't emit any
    // diagnostics or any double values.
    if (DiscardResult)
      return true;
    return this->visitAPValue(E->getAPValueResult(), *T, E);
  }

  // Fall back to the subexpr for non-primitive APValues.
  return this->delegate(E->getSubExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitEmbedExpr(const EmbedExpr *E) {
  auto It = E->begin();
  return this->visit(*It);
}

static CharUnits AlignOfType(QualType T, const ASTContext &ASTCtx,
                             UnaryExprOrTypeTrait Kind) {
  bool AlignOfReturnsPreferred =
      ASTCtx.getLangOpts().isCompatibleWith(LangOptions::ClangABI::Ver7);

  // C++ [expr.alignof]p3:
  //     When alignof is applied to a reference type, the result is the
  //     alignment of the referenced type.
  if (const auto *Ref = T->getAs<ReferenceType>())
    T = Ref->getPointeeType();

  if (T.getQualifiers().hasUnaligned())
    return CharUnits::One();

  // __alignof is defined to return the preferred alignment.
  // Before 8, clang returned the preferred alignment for alignof and
  // _Alignof as well.
  if (Kind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
    return ASTCtx.toCharUnitsFromBits(ASTCtx.getPreferredTypeAlign(T));

  return ASTCtx.getTypeAlignInChars(T);
}

template <class Emitter>
bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
    const UnaryExprOrTypeTraitExpr *E) {

  UnaryExprOrTypeTrait Kind = E->getKind();
  const ASTContext &ASTCtx = Ctx.getASTContext();

  if (Kind == UETT_SizeOf || Kind == UETT_DataSizeOf) {
    QualType ArgType = E->getTypeOfArgument();

    // C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
    //   the result is the size of the referenced type."
    if (const auto *Ref = ArgType->getAs<ReferenceType>())
      ArgType = Ref->getPointeeType();

    CharUnits Size;
    if (ArgType->isVoidType() || ArgType->isFunctionType())
      Size = CharUnits::One();
    else {
      if (ArgType->isDependentType() || !ArgType->isConstantSizeType())
        return this->emitInvalid(E);

      if (Kind == UETT_SizeOf)
        Size = ASTCtx.getTypeSizeInChars(ArgType);
      else
        Size = ASTCtx.getTypeInfoDataSizeInChars(ArgType).Width;
    }

    if (DiscardResult)
      return true;

    return this->emitConst(Size.getQuantity(), E);
  }

  if (Kind == UETT_CountOf) {
    QualType Ty = E->getTypeOfArgument();
    assert(Ty->isArrayType());

    // We don't need to worry about array element qualifiers, so getting the
    // unsafe array type is fine.
    if (const auto *CAT =
            dyn_cast<ConstantArrayType>(Ty->getAsArrayTypeUnsafe())) {
      if (DiscardResult)
        return true;
      return this->emitConst(CAT->getSize(), E);
    }

    assert(!Ty->isConstantSizeType());

    // If it's a variable-length array type, we need to check whether it is a
    // multidimensional array. If so, we need to check the size expression of
    // the VLA to see if it's a constant size. If so, we can return that value.
    const auto *VAT = ASTCtx.getAsVariableArrayType(Ty);
    assert(VAT);
    if (VAT->getElementType()->isArrayType()) {
      std::optional<APSInt> Res =
          VAT->getSizeExpr()
              ? VAT->getSizeExpr()->getIntegerConstantExpr(ASTCtx)
              : std::nullopt;
      if (Res) {
        if (DiscardResult)
          return true;
        return this->emitConst(*Res, E);
      }
    }
  }

  if (Kind == UETT_AlignOf || Kind == UETT_PreferredAlignOf) {
    CharUnits Size;

    if (E->isArgumentType()) {
      QualType ArgType = E->getTypeOfArgument();

      Size = AlignOfType(ArgType, ASTCtx, Kind);
    } else {
      // Argument is an expression, not a type.
      const Expr *Arg = E->getArgumentExpr()->IgnoreParens();

      if (Arg->getType()->isDependentType())
        return false;

      // The kinds of expressions that we have special-case logic here for
      // should be kept up to date with the special checks for those
      // expressions in Sema.

      // alignof decl is always accepted, even if it doesn't make sense: we
      // default to 1 in those cases.
      if (const auto *DRE = dyn_cast<DeclRefExpr>(Arg))
        Size = ASTCtx.getDeclAlign(DRE->getDecl(),
                                   /*RefAsPointee*/ true);
      else if (const auto *ME = dyn_cast<MemberExpr>(Arg))
        Size = ASTCtx.getDeclAlign(ME->getMemberDecl(),
                                   /*RefAsPointee*/ true);
      else
        Size = AlignOfType(Arg->getType(), ASTCtx, Kind);
    }

    if (DiscardResult)
      return true;

    return this->emitConst(Size.getQuantity(), E);
  }

  if (Kind == UETT_VectorElements) {
    if (E->containsErrors())
      return false;

    if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>())
      return this->emitConst(VT->getNumElements(), E);
    assert(E->getTypeOfArgument()->isSizelessVectorType());
    return this->emitSizelessVectorElementSize(E);
  }

  if (Kind == UETT_VecStep) {
    if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>()) {
      unsigned N = VT->getNumElements();

      // The vec_step built-in functions that take a 3-component
      // vector return 4. (OpenCL 1.1 spec 6.11.12)
      if (N == 3)
        N = 4;

      return this->emitConst(N, E);
    }
    return this->emitConst(1, E);
  }

  if (Kind == UETT_OpenMPRequiredSimdAlign) {
    if (E->containsErrors())
      return false;
    assert(E->isArgumentType());
    unsigned Bits = ASTCtx.getOpenMPDefaultSimdAlign(E->getArgumentType());

    return this->emitConst(ASTCtx.toCharUnitsFromBits(Bits).getQuantity(), E);
  }

  if (Kind == UETT_PtrAuthTypeDiscriminator) {
    if (E->getArgumentType()->isDependentType())
      return this->emitInvalid(E);

    return this->emitConst(
        const_cast<ASTContext &>(ASTCtx).getPointerAuthTypeDiscriminator(
            E->getArgumentType()),
        E);
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitMemberExpr(const MemberExpr *E) {
  // 'Base.Member'
  const Expr *Base = E->getBase();
  const ValueDecl *Member = E->getMemberDecl();

  if (DiscardResult)
    return this->discard(Base);

  if (const auto *VD = dyn_cast<VarDecl>(Member)) {
    // If the member is a VarDecl, this is a static variable.
    // We need to try to lazily evaluate its initializer here since the
    // variable might've been deserialized and not registered
    // as a global variable yet.
    if (VD->getInit() && !VD->getInit()->isValueDependent())
      VD->evaluateValue();
    if (auto GlobalIndex = P.getGlobal(VD)) {
      if (!this->emitGetPtrGlobal(*GlobalIndex, E))
        return false;
      if (Member->getType()->isReferenceType())
        return this->emitLoadPopPtr(E);
      return true;
    }
    return false;
  }

  if (!isa<FieldDecl>(Member)) {
    // A non-static member function access only makes sense as part of the
    // enclosing call here. Don't try to evaluate it in isolation.
    if (const auto *MD = dyn_cast<CXXMethodDecl>(Member);
        MD && !MD->isStatic()) {
      return false;
    }

    if (!this->discard(Base) && !this->emitSideEffect(E))
      return false;

    return this->visitDeclRef(Member, E);
  }

  if (!this->visit(Base))
    return false;

  // Base above gives us a pointer on the stack.
  const auto *FD = cast<FieldDecl>(Member);
  const RecordDecl *RD = FD->getParent();
  const Record *R = getRecord(RD);
  if (!R)
    return false;
  const Record::Field *F = R->getField(FD);

  // MemberExprs are almost always lvalues, in which case we don't need to
  // do the load. But sometimes they aren't.
  const auto maybeLoadValue = [&]() -> bool {
    if (E->isGLValue())
      return true;
    if (OptPrimType T = classify(E))
      return this->emitLoadPop(*T, E);
    return false;
  };

  // Leave a pointer to the field on the stack.
  if (F->Decl->getType()->isReferenceType())
    return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue();
  return this->emitGetPtrFieldPop(F->Offset, E) && maybeLoadValue();
}

template <class Emitter>
bool Compiler<Emitter>::VisitArrayInitIndexExpr(const ArrayInitIndexExpr *E) {
  assert(!DiscardResult);
  // ArrayIndex might not be set if a ArrayInitIndexExpr is being evaluated
  // stand-alone, e.g. via EvaluateAsInt().
  if (!ArrayIndex)
    return false;
  return this->emitConst(*ArrayIndex, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) {
  assert(Initializing);
  assert(!DiscardResult);

  const Expr *Common = E->getCommonExpr();
  const Expr *SubExpr = E->getSubExpr();
  OptPrimType SubExprT = classify(SubExpr);
  size_t Size = E->getArraySize().getZExtValue();

  if (SubExprT) {
    // Unwrap the OpaqueValueExpr so we don't cache something we won't reuse.
    Common = cast<OpaqueValueExpr>(Common)->getSourceExpr();

    if (!this->visit(Common))
      return false;
    return this->emitCopyArray(*SubExprT, 0, 0, Size, E);
  }

  // We visit the common opaque expression here once so we have its value
  // cached.
  if (!this->discard(Common))
    return false;

  // TODO: This compiles to quite a lot of bytecode if the array is larger.
  //   Investigate compiling this to a loop.

  // So, every iteration, we execute an assignment here
  // where the LHS is on the stack (the target array)
  // and the RHS is our SubExpr.
  for (size_t I = 0; I != Size; ++I) {
    ArrayIndexScope<Emitter> IndexScope(this, I);
    LocalScope<Emitter> BS(this, ScopeKind::FullExpression);

    if (!this->visitArrayElemInit(I, SubExpr, SubExprT))
      return false;
    if (!BS.destroyLocals())
      return false;
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
  const Expr *SourceExpr = E->getSourceExpr();
  if (!SourceExpr)
    return false;

  if (Initializing) {
    assert(!DiscardResult);
    return this->visitInitializer(SourceExpr);
  }

  PrimType SubExprT = classify(SourceExpr).value_or(PT_Ptr);
  if (auto It = OpaqueExprs.find(E); It != OpaqueExprs.end()) {
    if (DiscardResult)
      return true;
    return this->emitGetLocal(SubExprT, It->second, E);
  }

  if (!this->visit(SourceExpr))
    return false;

  // At this point we either have the evaluated source expression or a pointer
  // to an object on the stack. We want to create a local variable that stores
  // this value.
  unsigned LocalIndex = allocateLocalPrimitive(E, SubExprT, /*IsConst=*/true);
  if (!this->emitSetLocal(SubExprT, LocalIndex, E))
    return false;

  // This is cleaned up when the local variable is destroyed.
  OpaqueExprs.insert({E, LocalIndex});

  // Here the local variable is created but the value is removed from the stack,
  // so we put it back if the caller needs it.
  if (!DiscardResult)
    return this->emitGetLocal(SubExprT, LocalIndex, E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitAbstractConditionalOperator(
    const AbstractConditionalOperator *E) {
  const Expr *Condition = E->getCond();
  const Expr *TrueExpr = E->getTrueExpr();
  const Expr *FalseExpr = E->getFalseExpr();

  if (std::optional<bool> BoolValue = getBoolValue(Condition)) {
    if (*BoolValue)
      return this->delegate(TrueExpr);
    return this->delegate(FalseExpr);
  }

  bool IsBcpCall = false;
  if (const auto *CE = dyn_cast<CallExpr>(Condition->IgnoreParenCasts());
      CE && CE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) {
    IsBcpCall = true;
  }

  LabelTy LabelEnd = this->getLabel();   // Label after the operator.
  LabelTy LabelFalse = this->getLabel(); // Label for the false expr.

  if (IsBcpCall) {
    if (!this->emitPushIgnoreDiags(E))
      return false;
  }

  if (!this->visitBool(Condition)) {
    // If the condition failed and we're checking for undefined behavior
    // (which only happens with EvalEmitter) check the TrueExpr and FalseExpr
    // as well.
    if (this->checkingForUndefinedBehavior()) {
      if (!this->discard(TrueExpr))
        return false;
      if (!this->discard(FalseExpr))
        return false;
    }
    return false;
  }

  // Force-init the scope, which creates a InitScope op. This is necessary so
  // the scope is not only initialized in one arm of the conditional operator.
  this->VarScope->forceInit();
  // The TrueExpr and FalseExpr of a conditional operator do _not_ create a
  // scope, which means the local variables created within them unconditionally
  // always exist. However, we need to later differentiate which branch was
  // taken and only destroy the varibles of the active branch. This is what the
  // "enabled" flags on local variables are used for.
  llvm::SaveAndRestore LAAA(this->VarScope->LocalsAlwaysEnabled,
                            /*NewValue=*/false);

  if (!this->jumpFalse(LabelFalse, E))
    return false;
  if (!this->delegate(TrueExpr))
    return false;

  if (!this->jump(LabelEnd, E))
    return false;
  this->emitLabel(LabelFalse);
  if (!this->delegate(FalseExpr))
    return false;

  this->fallthrough(LabelEnd);
  this->emitLabel(LabelEnd);

  if (IsBcpCall)
    return this->emitPopIgnoreDiags(E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitStringLiteral(const StringLiteral *E) {
  if (DiscardResult)
    return true;

  if (!Initializing)
    return this->emitGetStringPtr(E, E);

  // We are initializing an array on the stack.
  const ConstantArrayType *CAT =
      Ctx.getASTContext().getAsConstantArrayType(E->getType());
  assert(CAT && "a string literal that's not a constant array?");

  // If the initializer string is too long, a diagnostic has already been
  // emitted. Read only the array length from the string literal.
  unsigned ArraySize = CAT->getZExtSize();
  unsigned N = std::min(ArraySize, E->getLength());
  unsigned CharWidth = E->getCharByteWidth();

  for (unsigned I = 0; I != N; ++I) {
    uint32_t CodeUnit = E->getCodeUnit(I);

    if (CharWidth == 1) {
      this->emitConstSint8(CodeUnit, E);
      this->emitInitElemSint8(I, E);
    } else if (CharWidth == 2) {
      this->emitConstUint16(CodeUnit, E);
      this->emitInitElemUint16(I, E);
    } else if (CharWidth == 4) {
      this->emitConstUint32(CodeUnit, E);
      this->emitInitElemUint32(I, E);
    } else {
      llvm_unreachable("unsupported character width");
    }
  }

  // Fill up the rest of the char array with NUL bytes.
  for (unsigned I = N; I != ArraySize; ++I) {
    if (CharWidth == 1) {
      this->emitConstSint8(0, E);
      this->emitInitElemSint8(I, E);
    } else if (CharWidth == 2) {
      this->emitConstUint16(0, E);
      this->emitInitElemUint16(I, E);
    } else if (CharWidth == 4) {
      this->emitConstUint32(0, E);
      this->emitInitElemUint32(I, E);
    } else {
      llvm_unreachable("unsupported character width");
    }
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
  if (DiscardResult)
    return true;
  return this->emitDummyPtr(E, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
  auto &A = Ctx.getASTContext();
  std::string Str;
  A.getObjCEncodingForType(E->getEncodedType(), Str);
  StringLiteral *SL =
      StringLiteral::Create(A, Str, StringLiteralKind::Ordinary,
                            /*Pascal=*/false, E->getType(), E->getAtLoc());
  return this->delegate(SL);
}

template <class Emitter>
bool Compiler<Emitter>::VisitSYCLUniqueStableNameExpr(
    const SYCLUniqueStableNameExpr *E) {
  if (DiscardResult)
    return true;

  assert(!Initializing);

  auto &A = Ctx.getASTContext();
  std::string ResultStr = E->ComputeName(A);

  QualType CharTy = A.CharTy.withConst();
  APInt Size(A.getTypeSize(A.getSizeType()), ResultStr.size() + 1);
  QualType ArrayTy = A.getConstantArrayType(CharTy, Size, nullptr,
                                            ArraySizeModifier::Normal, 0);

  StringLiteral *SL =
      StringLiteral::Create(A, ResultStr, StringLiteralKind::Ordinary,
                            /*Pascal=*/false, ArrayTy, E->getLocation());
  return this->emitGetStringPtr(SL, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCharacterLiteral(const CharacterLiteral *E) {
  if (DiscardResult)
    return true;
  return this->emitConst(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitFloatCompoundAssignOperator(
    const CompoundAssignOperator *E) {

  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  QualType LHSType = LHS->getType();
  QualType LHSComputationType = E->getComputationLHSType();
  QualType ResultType = E->getComputationResultType();
  OptPrimType LT = classify(LHSComputationType);
  OptPrimType RT = classify(ResultType);

  assert(ResultType->isFloatingType());

  if (!LT || !RT)
    return false;

  PrimType LHST = classifyPrim(LHSType);

  if (isSideEffectFree(RHS)) {
    if (!visit(LHS))
      return false;
    if (!this->emitLoad(LHST, E))
      return false;
    // If necessary, convert LHS to its computation type.
    if (!this->emitPrimCast(LHST, classifyPrim(LHSComputationType),
                            LHSComputationType, E))
      return false;
    if (!visit(RHS))
      return false;

  } else {
    // C++17 onwards require that we evaluate the RHS first.
    // Compute RHS and save it in a temporary variable so we can
    // load it again later.
    if (!visit(RHS))
      return false;

    unsigned TempOffset =
        this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
    if (!this->emitSetLocal(*RT, TempOffset, E))
      return false;

    // First, visit LHS.
    if (!visit(LHS))
      return false;
    if (!this->emitLoad(LHST, E))
      return false;

    // If necessary, convert LHS to its computation type.
    if (!this->emitPrimCast(LHST, classifyPrim(LHSComputationType),
                            LHSComputationType, E))
      return false;

    // Now load RHS.
    if (!this->emitGetLocal(*RT, TempOffset, E))
      return false;
  }

  switch (E->getOpcode()) {
  case BO_AddAssign:
    if (!this->emitAddf(getFPOptions(E), E))
      return false;
    break;
  case BO_SubAssign:
    if (!this->emitSubf(getFPOptions(E), E))
      return false;
    break;
  case BO_MulAssign:
    if (!this->emitMulf(getFPOptions(E), E))
      return false;
    break;
  case BO_DivAssign:
    if (!this->emitDivf(getFPOptions(E), E))
      return false;
    break;
  default:
    return false;
  }

  if (!this->emitPrimCast(classifyPrim(ResultType), LHST, LHS->getType(), E))
    return false;

  if (DiscardResult)
    return this->emitStorePop(LHST, E);
  return this->emitStore(LHST, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitPointerCompoundAssignOperator(
    const CompoundAssignOperator *E) {
  BinaryOperatorKind Op = E->getOpcode();
  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  OptPrimType LT = classify(LHS->getType());
  OptPrimType RT = classify(RHS->getType());

  if (Op != BO_AddAssign && Op != BO_SubAssign)
    return false;

  if (!LT || !RT)
    return false;

  if (!visit(LHS))
    return false;

  if (!this->emitLoad(*LT, LHS))
    return false;

  if (!visit(RHS))
    return false;

  if (Op == BO_AddAssign) {
    if (!this->emitAddOffset(*RT, E))
      return false;
  } else {
    if (!this->emitSubOffset(*RT, E))
      return false;
  }

  if (DiscardResult)
    return this->emitStorePopPtr(E);
  return this->emitStorePtr(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCompoundAssignOperator(
    const CompoundAssignOperator *E) {
  if (E->getType()->isVectorType())
    return VisitVectorBinOp(E);

  const Expr *LHS = E->getLHS();
  const Expr *RHS = E->getRHS();
  OptPrimType LHSComputationT = classify(E->getComputationLHSType());
  OptPrimType LT = classify(LHS->getType());
  OptPrimType RT = classify(RHS->getType());
  OptPrimType ResultT = classify(E->getType());

  if (!Ctx.getLangOpts().CPlusPlus14)
    return this->visit(RHS) && this->visit(LHS) && this->emitError(E);

  if (!LT || !RT || !ResultT || !LHSComputationT)
    return false;

  // Handle floating point operations separately here, since they
  // require special care.
  if (ResultT == PT_Float || RT == PT_Float)
    return VisitFloatCompoundAssignOperator(E);

  if (E->getType()->isPointerType())
    return VisitPointerCompoundAssignOperator(E);

  assert(!E->getType()->isPointerType() && "Handled above");
  assert(!E->getType()->isFloatingType() && "Handled above");

  if (isSideEffectFree(RHS)) {
    if (!visit(LHS))
      return false;
    if (!this->emitLoad(*LT, E))
      return false;
    if (LT != LHSComputationT &&
        !this->emitIntegralCast(*LT, *LHSComputationT,
                                E->getComputationLHSType(), E))
      return false;
    if (!visit(RHS))
      return false;
  } else {
    // C++17 onwards require that we evaluate the RHS first.
    // Compute RHS and save it in a temporary variable so we can
    // load it again later.
    // FIXME: Compound assignments are unsequenced in C, so we might
    //   have to figure out how to reject them.
    if (!visit(RHS))
      return false;

    unsigned TempOffset =
        this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);

    if (!this->emitSetLocal(*RT, TempOffset, E))
      return false;

    // Get LHS pointer, load its value and cast it to the
    // computation type if necessary.
    if (!visit(LHS))
      return false;
    if (!this->emitLoad(*LT, E))
      return false;
    if (LT != LHSComputationT &&
        !this->emitIntegralCast(*LT, *LHSComputationT,
                                E->getComputationLHSType(), E))
      return false;

    // Get the RHS value on the stack.
    if (!this->emitGetLocal(*RT, TempOffset, E))
      return false;
  }

  // Perform operation.
  switch (E->getOpcode()) {
  case BO_AddAssign:
    if (!this->emitAdd(*LHSComputationT, E))
      return false;
    break;
  case BO_SubAssign:
    if (!this->emitSub(*LHSComputationT, E))
      return false;
    break;
  case BO_MulAssign:
    if (!this->emitMul(*LHSComputationT, E))
      return false;
    break;
  case BO_DivAssign:
    if (!this->emitDiv(*LHSComputationT, E))
      return false;
    break;
  case BO_RemAssign:
    if (!this->emitRem(*LHSComputationT, E))
      return false;
    break;
  case BO_ShlAssign:
    if (!this->emitShl(*LHSComputationT, *RT, E))
      return false;
    break;
  case BO_ShrAssign:
    if (!this->emitShr(*LHSComputationT, *RT, E))
      return false;
    break;
  case BO_AndAssign:
    if (!this->emitBitAnd(*LHSComputationT, E))
      return false;
    break;
  case BO_XorAssign:
    if (!this->emitBitXor(*LHSComputationT, E))
      return false;
    break;
  case BO_OrAssign:
    if (!this->emitBitOr(*LHSComputationT, E))
      return false;
    break;
  default:
    llvm_unreachable("Unimplemented compound assign operator");
  }

  // And now cast from LHSComputationT to ResultT.
  if (ResultT != LHSComputationT &&
      !this->emitIntegralCast(*LHSComputationT, *ResultT, E->getType(), E))
    return false;

  // And store the result in LHS.
  if (DiscardResult) {
    if (LHS->refersToBitField())
      return this->emitStoreBitFieldPop(*ResultT, E);
    return this->emitStorePop(*ResultT, E);
  }
  if (LHS->refersToBitField())
    return this->emitStoreBitField(*ResultT, E);
  return this->emitStore(*ResultT, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitExprWithCleanups(const ExprWithCleanups *E) {
  LocalScope<Emitter> ES(this, ScopeKind::FullExpression);
  const Expr *SubExpr = E->getSubExpr();

  return this->delegate(SubExpr) && ES.destroyLocals(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitMaterializeTemporaryExpr(
    const MaterializeTemporaryExpr *E) {
  if (Initializing) {
    // We already have a value, just initialize that.
    return this->delegate(E->getSubExpr());
  }
  // If we don't end up using the materialized temporary anyway, don't
  // bother creating it.
  if (DiscardResult)
    return this->discard(E->getSubExpr());

  SmallVector<const Expr *, 2> CommaLHSs;
  SmallVector<SubobjectAdjustment, 2> Adjustments;
  const Expr *Inner;
  if (!Ctx.getLangOpts().CPlusPlus11)
    Inner =
        E->getSubExpr()->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
  else
    Inner = E->getSubExpr();

  // If we passed any comma operators, evaluate their LHSs.
  for (const Expr *LHS : CommaLHSs) {
    if (!this->discard(LHS))
      return false;
  }

  // FIXME: Find a test case where Adjustments matters.

  // When we're extending a global variable *or* the storage duration of
  // the temporary is explicitly static, create a global variable.
  OptPrimType InnerT = classify(Inner);
  const ValueDecl *ExtendingDecl = E->getExtendingDecl();
  bool IsStatic = E->getStorageDuration() == SD_Static;
  if (IsStatic ||
      (ExtendingDecl && Context::shouldBeGloballyIndexed(ExtendingDecl))) {

    if (this->constantFolding())
      return false;

    UnsignedOrNone GlobalIndex = P.createGlobal(E, Inner->getType());
    if (!GlobalIndex)
      return false;

    const LifetimeExtendedTemporaryDecl *TempDecl =
        E->getLifetimeExtendedTemporaryDecl();

    if (InnerT) {
      if (!this->visit(Inner))
        return false;

      if (IsStatic) {
        assert(TempDecl);
        if (!this->emitInitGlobalTemp(*InnerT, *GlobalIndex, TempDecl, E))
          return false;
      } else {
        if (!this->emitInitGlobal(*InnerT, *GlobalIndex, E))
          return false;
      }
      return this->emitGetPtrGlobal(*GlobalIndex, E);
    }

    if (!this->checkLiteralType(Inner))
      return false;
    // Non-primitive values.
    if (!this->emitGetPtrGlobal(*GlobalIndex, E))
      return false;
    if (!this->visitInitializer(Inner))
      return false;
    if (IsStatic) {
      assert(TempDecl);
      return this->emitInitGlobalTempComp(TempDecl, E);
    }
    return true;
  }

  ScopeKind VarScope = E->getStorageDuration() == SD_FullExpression
                           ? ScopeKind::FullExpression
                           : ScopeKind::Block;

  // For everyhing else, use local variables.
  if (InnerT) {
    bool IsConst = Inner->getType().isConstQualified();
    bool IsVolatile = Inner->getType().isVolatileQualified();
    unsigned LocalIndex =
        allocateLocalPrimitive(E, *InnerT, IsConst, IsVolatile, VarScope);
    if (!this->VarScope->LocalsAlwaysEnabled &&
        !this->emitEnableLocal(LocalIndex, E))
      return false;

    if (!this->visit(Inner))
      return false;
    if (!this->emitSetLocal(*InnerT, LocalIndex, E))
      return false;

    return this->emitGetPtrLocal(LocalIndex, E);
  }

  if (!this->checkLiteralType(Inner))
    return false;

  if (UnsignedOrNone LocalIndex =
          allocateLocal(E, Inner->getType(), VarScope)) {
    InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalIndex));

    if (!this->VarScope->LocalsAlwaysEnabled &&
        !this->emitEnableLocal(*LocalIndex, E))
      return false;

    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
    return this->visitInitializer(Inner);
  }
  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXBindTemporaryExpr(
    const CXXBindTemporaryExpr *E) {
  const Expr *SubExpr = E->getSubExpr();

  if (Initializing)
    return this->delegate(SubExpr);

  // Make sure we create a temporary even if we're discarding, since that will
  // make sure we will also call the destructor.

  if (!this->visit(SubExpr))
    return false;

  if (DiscardResult)
    return this->emitPopPtr(E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
  const Expr *Init = E->getInitializer();
  if (DiscardResult)
    return this->discard(Init);

  if (Initializing) {
    // We already have a value, just initialize that.
    return this->visitInitializer(Init);
  }

  OptPrimType T = classify(E->getType());
  if (E->isFileScope()) {
    // Avoid creating a variable if this is a primitive RValue anyway.
    if (T && !E->isLValue())
      return this->delegate(Init);

    UnsignedOrNone GlobalIndex = P.createGlobal(E, E->getType());
    if (!GlobalIndex)
      return false;

    if (!this->emitGetPtrGlobal(*GlobalIndex, E))
      return false;

    // Since this is a global variable, we might've already seen,
    // don't do it again.
    if (P.isGlobalInitialized(*GlobalIndex))
      return true;

    if (T) {
      if (!this->visit(Init))
        return false;
      return this->emitInitGlobal(*T, *GlobalIndex, E);
    }

    return this->visitInitializer(Init);
  }

  // Otherwise, use a local variable.
  if (T && !E->isLValue()) {
    // For primitive types, we just visit the initializer.
    return this->delegate(Init);
  }

  unsigned LocalIndex;
  if (T)
    LocalIndex = this->allocateLocalPrimitive(Init, *T, /*IsConst=*/false);
  else if (UnsignedOrNone MaybeIndex = this->allocateLocal(Init))
    LocalIndex = *MaybeIndex;
  else
    return false;

  if (!this->emitGetPtrLocal(LocalIndex, E))
    return false;

  if (T)
    return this->visit(Init) && this->emitInit(*T, E);
  return this->visitInitializer(Init);
}

template <class Emitter>
bool Compiler<Emitter>::VisitTypeTraitExpr(const TypeTraitExpr *E) {
  if (DiscardResult)
    return true;
  if (E->isStoredAsBoolean()) {
    if (E->getType()->isBooleanType())
      return this->emitConstBool(E->getBoolValue(), E);
    return this->emitConst(E->getBoolValue(), E);
  }
  if (E->isStoredAsComparisonResult()) {
    const ComparisonCategoryInfo &CmpInfo =
        Ctx.getASTContext().CompCategories.getInfoForType(E->getType());
    const auto Result =
        ComparisonCategoryResult(E->getAPValue().getInt().getZExtValue());
    const Record *R = getRecord(E->getType());
    if (!R || R->getNumFields() == 0)
      return false;
    const Record::Field *Field = R->getField(0U);
    assert(Field->T);
    if (!this->emitConst(CmpInfo.getValueInfo(Result)->getIntValue(), *Field->T,
                         E))
      return false;
    return this->emitInitField(*Field->T, Field->Offset, E);
  }

  PrimType T = classifyPrim(E->getType());
  return this->visitAPValue(E->getAPValue(), T, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) {
  if (DiscardResult)
    return true;
  return this->emitConst(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
  if (DiscardResult)
    return true;

  assert(Initializing);
  const Record *R = P.getOrCreateRecord(E->getLambdaClass());
  if (!R)
    return false;

  auto *CaptureInitIt = E->capture_init_begin();
  // Initialize all fields (which represent lambda captures) of the
  // record with their initializers.
  for (const Record::Field &F : R->fields()) {
    const Expr *Init = *CaptureInitIt;
    if (!Init || Init->containsErrors())
      continue;
    ++CaptureInitIt;

    if (OptPrimType T = classify(Init)) {
      if (!this->visit(Init))
        return false;

      if (!this->emitInitField(*T, F.Offset, E))
        return false;
    } else {
      if (!this->emitGetPtrField(F.Offset, E))
        return false;

      if (!this->visitInitializerPop(Init))
        return false;
    }
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
  if (DiscardResult)
    return true;

  if (!Initializing)
    return this->emitGetStringPtr(E, E);
  return this->delegate(E->getFunctionName());
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXThrowExpr(const CXXThrowExpr *E) {
  if (E->getSubExpr() && !this->discard(E->getSubExpr()))
    return false;

  return this->emitInvalid(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXReinterpretCastExpr(
    const CXXReinterpretCastExpr *E) {
  const Expr *SubExpr = E->getSubExpr();

  OptPrimType FromT = classify(SubExpr);
  OptPrimType ToT = classify(E);

  if (!FromT || !ToT)
    return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, E);

  if (FromT == PT_Ptr || ToT == PT_Ptr) {
    auto CastKind = isIntegerType(*ToT) ? CastKind::ReinterpretPtrToInt
                                        : CastKind::Reinterpret;
    if (!this->emitInvalidCast(CastKind, /*Fatal=*/false, E))
      return false;
    if (E->getCastKind() == CK_LValueBitCast)
      return this->delegate(SubExpr);
    return this->VisitCastExpr(E);
  }

  // Try to actually do the cast.
  bool Fatal = (ToT != FromT);
  if (!this->emitInvalidCast(CastKind::Reinterpret, Fatal, E))
    return false;

  return this->VisitCastExpr(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXDynamicCastExpr(const CXXDynamicCastExpr *E) {
  if (!Ctx.getLangOpts().CPlusPlus20) {
    if (!this->emitInvalidCast(CastKind::Dynamic, /*Fatal=*/false, E))
      return false;
  }

  if (E->getCastKind() != CK_Dynamic)
    return this->VisitCastExpr(E);

  QualType DestType = E->getType();
  // "target type must be a reference or pointer type to a defined class"
  if (DestType->isRecordType()) {
    assert(E->isGLValue());
  } else {
    assert(DestType->isPointerOrReferenceType());
    assert(DestType->isVoidPointerType() ||
           DestType->getPointeeType()->isRecordType());
    DestType = DestType->getPointeeType();
  }

  if (!this->visit(E->getSubExpr()))
    return false;
  if (!this->emitDynamicCast(DestType.getTypePtr(),
                             /*IsReferenceCast=*/E->isGLValue(), E))
    return false;

  if (DiscardResult)
    return this->emitPopPtr(E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
  assert(E->getType()->isBooleanType());

  if (DiscardResult)
    return true;
  return this->emitConstBool(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXConstructExpr(const CXXConstructExpr *E) {
  QualType T = E->getType();
  assert(!canClassify(T));

  if (T->isRecordType()) {
    const CXXConstructorDecl *Ctor = E->getConstructor();

    // If we're discarding a construct expression, we still need
    // to allocate a variable and call the constructor and destructor.
    if (DiscardResult) {
      if (Ctor->isTrivial())
        return true;
      assert(!Initializing);
      UnsignedOrNone LocalIndex = allocateLocal(E);

      if (!LocalIndex)
        return false;

      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    // Trivial copy/move constructor. Avoid copy.
    if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
        Ctor->isTrivial() &&
        E->getArg(0)->isTemporaryObject(Ctx.getASTContext(),
                                        T->getAsCXXRecordDecl()))
      return this->visitInitializer(E->getArg(0));

    // Zero initialization.
    bool ZeroInit = E->requiresZeroInitialization();
    if (ZeroInit) {
      const Record *R = getRecord(E->getType());
      if (!R)
        return false;

      if (!this->visitZeroRecordInitializer(R, E))
        return false;

      // If the constructor is trivial anyway, we're done.
      if (Ctor->isTrivial())
        return true;
    }

    // Trivial default constructors might never be implicitly defined by the
    // AST, so we need to special-case them here.
    if (Ctor->isTrivial() && Ctor->isDefaultConstructor()) {
      if (!this->emitDefaultInit(Ctor, E))
        return false;
      if (DiscardResult)
        return this->emitPopPtr(E);
      return true;
    }

    // Avoid materializing a temporary for an elidable copy/move constructor.
    if (!ZeroInit && E->isElidable()) {
      const Expr *SrcObj = E->getArg(0);
      assert(SrcObj->isTemporaryObject(Ctx.getASTContext(), Ctor->getParent()));
      assert(Ctx.getASTContext().hasSameUnqualifiedType(E->getType(),
                                                        SrcObj->getType()));
      if (const auto *ME = dyn_cast<MaterializeTemporaryExpr>(SrcObj)) {
        if (!this->emitCheckFunctionDecl(Ctor, E))
          return false;
        return this->visitInitializer(ME->getSubExpr());
      }
    }

    const Function *Func = getFunction(Ctor);

    if (!Func)
      return false;

    assert(Func->hasThisPointer());
    assert(!Func->hasRVO());

    //  The This pointer is already on the stack because this is an initializer,
    //  but we need to dup() so the call() below has its own copy.
    if (!this->emitDupPtr(E))
      return false;

    // Constructor arguments.
    for (const auto *Arg : E->arguments()) {
      if (!this->visit(Arg))
        return false;
    }

    if (Func->isVariadic()) {
      uint32_t VarArgSize = 0;
      unsigned NumParams = Func->getNumWrittenParams();
      for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I) {
        VarArgSize +=
            align(primSize(classify(E->getArg(I)->getType()).value_or(PT_Ptr)));
      }
      if (!this->emitCallVar(Func, VarArgSize, E))
        return false;
    } else {
      if (!this->emitCall(Func, 0, E)) {
        // When discarding, we don't need the result anyway, so clean up
        // the instance dup we did earlier in case surrounding code wants
        // to keep evaluating.
        if (DiscardResult)
          (void)this->emitPopPtr(E);
        return false;
      }
    }

    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  }

  if (T->isArrayType()) {
    const Function *Func = getFunction(E->getConstructor());
    if (!Func)
      return false;

    if (!this->emitDupPtr(E))
      return false;

    std::function<bool(QualType)> initArrayDimension;
    initArrayDimension = [&](QualType T) -> bool {
      if (!T->isArrayType()) {
        // Constructor arguments.
        for (const auto *Arg : E->arguments()) {
          if (!this->visit(Arg))
            return false;
        }

        return this->emitCall(Func, 0, E);
      }

      const ConstantArrayType *CAT =
          Ctx.getASTContext().getAsConstantArrayType(T);
      if (!CAT)
        return false;
      QualType ElemTy = CAT->getElementType();
      unsigned NumElems = CAT->getZExtSize();
      for (size_t I = 0; I != NumElems; ++I) {
        if (!this->emitConstUint64(I, E))
          return false;
        if (!this->emitArrayElemPtrUint64(E))
          return false;
        if (!initArrayDimension(ElemTy))
          return false;
      }
      return this->emitPopPtr(E);
    };

    return initArrayDimension(E->getType());
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitSourceLocExpr(const SourceLocExpr *E) {
  if (DiscardResult)
    return true;

  const APValue Val =
      E->EvaluateInContext(Ctx.getASTContext(), SourceLocDefaultExpr);

  // Things like __builtin_LINE().
  if (E->getType()->isIntegerType()) {
    assert(Val.isInt());
    const APSInt &I = Val.getInt();
    return this->emitConst(I, E);
  }
  // Otherwise, the APValue is an LValue, with only one element.
  // Theoretically, we don't need the APValue at all of course.
  assert(E->getType()->isPointerType());
  assert(Val.isLValue());
  const APValue::LValueBase &Base = Val.getLValueBase();
  if (const Expr *LValueExpr = Base.dyn_cast<const Expr *>())
    return this->visit(LValueExpr);

  // Otherwise, we have a decl (which is the case for
  // __builtin_source_location).
  assert(Base.is<const ValueDecl *>());
  assert(Val.getLValuePath().size() == 0);
  const auto *BaseDecl = Base.dyn_cast<const ValueDecl *>();
  assert(BaseDecl);

  auto *UGCD = cast<UnnamedGlobalConstantDecl>(BaseDecl);

  UnsignedOrNone GlobalIndex = P.getOrCreateGlobal(UGCD);
  if (!GlobalIndex)
    return false;

  if (!this->emitGetPtrGlobal(*GlobalIndex, E))
    return false;

  const Record *R = getRecord(E->getType());
  const APValue &V = UGCD->getValue();
  for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
    const Record::Field *F = R->getField(I);
    const APValue &FieldValue = V.getStructField(I);

    if (!this->visitAPValue(FieldValue, *F->T, E))
      return false;
    if (!this->emitInitField(*F->T, F->Offset, E))
      return false;
  }

  // Leave the pointer to the global on the stack.
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
  unsigned N = E->getNumComponents();
  if (N == 0)
    return false;

  for (unsigned I = 0; I != N; ++I) {
    const OffsetOfNode &Node = E->getComponent(I);
    if (Node.getKind() == OffsetOfNode::Array) {
      const Expr *ArrayIndexExpr = E->getIndexExpr(Node.getArrayExprIndex());
      PrimType IndexT = classifyPrim(ArrayIndexExpr->getType());

      if (DiscardResult) {
        if (!this->discard(ArrayIndexExpr))
          return false;
        continue;
      }

      if (IndexT == PT_IntAP || IndexT == PT_IntAPS) {
        if (!this->visit(ArrayIndexExpr))
          return false;
        if (!this->emitCastAPToOffsetIndex(IndexT, E))
          return false;
        continue;
      }
      if (!this->visit(ArrayIndexExpr))
        return false;
      // Cast to Sint64.
      if (IndexT != PT_Sint64) {
        if (!this->emitCast(IndexT, PT_Sint64, E))
          return false;
      }
    }
  }

  if (DiscardResult)
    return true;

  PrimType T = classifyPrim(E->getType());
  return this->emitOffsetOf(T, E, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXScalarValueInitExpr(
    const CXXScalarValueInitExpr *E) {
  QualType Ty = E->getType();

  if (DiscardResult || Ty->isVoidType())
    return true;

  if (OptPrimType T = classify(Ty))
    return this->visitZeroInitializer(*T, Ty, E);

  if (Ty->isAnyComplexType() || Ty->isVectorType()) {
    if (!Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(E);
      if (!LocalIndex)
        return false;
      if (!this->emitGetPtrLocal(*LocalIndex, E))
        return false;
    }

    QualType ElemQT;
    unsigned NumElems;
    if (const auto *CT = Ty->getAs<ComplexType>()) {
      NumElems = 2;
      ElemQT = CT->getElementType();
    } else {
      const auto *VT = Ty->castAs<VectorType>();
      NumElems = VT->getNumElements();
      ElemQT = VT->getElementType();
    }

    PrimType ElemT = classifyPrim(ElemQT);

    // Initialize all fields to 0.
    for (unsigned I = 0; I != NumElems; ++I) {
      if (!this->visitZeroInitializer(ElemT, ElemQT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitSizeOfPackExpr(const SizeOfPackExpr *E) {
  return this->emitConst(E->getPackLength(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitGenericSelectionExpr(
    const GenericSelectionExpr *E) {
  return this->delegate(E->getResultExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitChooseExpr(const ChooseExpr *E) {
  return this->delegate(E->getChosenSubExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *E) {
  if (DiscardResult)
    return true;

  return this->emitConst(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXInheritedCtorInitExpr(
    const CXXInheritedCtorInitExpr *E) {
  const CXXConstructorDecl *Ctor = E->getConstructor();
  assert(!Ctor->isTrivial() &&
         "Trivial CXXInheritedCtorInitExpr, implement. (possible?)");
  const Function *F = this->getFunction(Ctor);
  if (!F)
    return false;
  assert(!F->hasRVO());
  assert(F->hasThisPointer());

  if (!this->emitDupPtr(SourceInfo{}))
    return false;

  // Forward all arguments of the current function (which should be a
  // constructor itself) to the inherited ctor.
  // This is necessary because the calling code has pushed the pointer
  // of the correct base for  us already, but the arguments need
  // to come after.
  unsigned ParamIndex = 0;
  for (const ParmVarDecl *PD : Ctor->parameters()) {
    PrimType PT = this->classify(PD->getType()).value_or(PT_Ptr);

    if (!this->emitGetParam(PT, ParamIndex, E))
      return false;
    ++ParamIndex;
  }

  return this->emitCall(F, 0, E);
}

// FIXME: This function has become rather unwieldy, especially
// the part where we initialize an array allocation of dynamic size.
template <class Emitter>
bool Compiler<Emitter>::VisitCXXNewExpr(const CXXNewExpr *E) {
  assert(classifyPrim(E->getType()) == PT_Ptr);
  const Expr *Init = E->getInitializer();
  QualType ElementType = E->getAllocatedType();
  OptPrimType ElemT = classify(ElementType);
  unsigned PlacementArgs = E->getNumPlacementArgs();
  const FunctionDecl *OperatorNew = E->getOperatorNew();
  const Expr *PlacementDest = nullptr;
  bool IsNoThrow = false;

  if (E->containsErrors())
    return false;

  if (PlacementArgs != 0) {
    // FIXME: There is no restriction on this, but it's not clear that any
    // other form makes any sense. We get here for cases such as:
    //
    //   new (std::align_val_t{N}) X(int)
    //
    // (which should presumably be valid only if N is a multiple of
    // alignof(int), and in any case can't be deallocated unless N is
    // alignof(X) and X has new-extended alignment).
    if (PlacementArgs == 1) {
      const Expr *Arg1 = E->getPlacementArg(0);
      if (OperatorNew->isReservedGlobalPlacementOperator()) {
        if (!this->emitCheckPlacementNew(E, E))
          return false;
        PlacementDest = Arg1;
      } else if (
          Arg1->getType()->isNothrowT() &&
          OperatorNew
              ->isUsableAsGlobalAllocationFunctionInConstantEvaluation()) {
        if (!this->discard(Arg1))
          return false;
        IsNoThrow = true;
      } else {
        // Any other placement list is invalid. This includes a user-declared
        // allocation function taking std::nothrow_t, e.g. by value.
        return this->emitInvalidNewDeleteExpr(E, E);
      }
    } else {
      // Always invalid.
      return this->emitInvalid(E);
    }
  } else if (!OperatorNew
                  ->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
    return this->emitInvalidNewDeleteExpr(E, E);

  const Descriptor *Desc;
  if (!PlacementDest) {
    if (ElemT) {
      if (E->isArray())
        Desc = nullptr; // We're not going to use it in this case.
      else
        Desc = P.createDescriptor(E, *ElemT);
    } else {
      Desc = P.createDescriptor(E, ElementType.getTypePtr(), /*IsConst=*/false,
                                /*IsTemporary=*/false, /*IsMutable=*/false,
                                /*IsVolatile=*/false, Init);
    }
  }

  if (E->isArray()) {
    std::optional<const Expr *> ArraySizeExpr = E->getArraySize();
    if (!ArraySizeExpr)
      return false;

    const Expr *Stripped = *ArraySizeExpr;
    for (; auto *ICE = dyn_cast<ImplicitCastExpr>(Stripped);
         Stripped = ICE->getSubExpr())
      if (ICE->getCastKind() != CK_NoOp &&
          ICE->getCastKind() != CK_IntegralCast)
        break;

    PrimType SizeT = classifyPrim(Stripped->getType());

    // Save evaluated array size to a variable.
    unsigned ArrayLen =
        allocateLocalPrimitive(Stripped, SizeT, /*IsConst=*/false);
    if (!this->visit(Stripped))
      return false;
    if (!this->emitSetLocal(SizeT, ArrayLen, E))
      return false;

    if (PlacementDest) {
      if (!this->visit(PlacementDest))
        return false;
      if (!this->emitGetLocal(SizeT, ArrayLen, E))
        return false;
      if (!this->emitCheckNewTypeMismatchArray(SizeT, E, E))
        return false;
    } else {
      if (!this->emitGetLocal(SizeT, ArrayLen, E))
        return false;

      if (ElemT) {
        // N primitive elements.
        if (!this->emitAllocN(SizeT, *ElemT, E, IsNoThrow, E))
          return false;
      } else {
        // N Composite elements.
        if (!this->emitAllocCN(SizeT, Desc, IsNoThrow, E))
          return false;
      }
    }

    if (Init) {
      QualType InitType = Init->getType();
      size_t StaticInitElems = 0;
      const Expr *DynamicInit = nullptr;
      OptPrimType ElemT;

      if (const ConstantArrayType *CAT =
              Ctx.getASTContext().getAsConstantArrayType(InitType)) {
        StaticInitElems = CAT->getZExtSize();
        // Initialize the first S element from the initializer.
        if (!this->visitInitializer(Init))
          return false;

        if (const auto *ILE = dyn_cast<InitListExpr>(Init)) {
          if (ILE->hasArrayFiller())
            DynamicInit = ILE->getArrayFiller();
          else if (StaticInitElems > 0 && isa<StringLiteral>(ILE->getInit(0)))
            ElemT = classifyPrim(CAT->getElementType());
        }
      }

      // The initializer initializes a certain number of elements, S.
      // However, the complete number of elements, N, might be larger than that.
      // In this case, we need to get an initializer for the remaining elements.
      // There are three cases:
      //   1) For the form 'new Struct[n];', the initializer is a
      //      CXXConstructExpr and its type is an IncompleteArrayType.
      //   2) For the form 'new Struct[n]{1,2,3}', the initializer is an
      //      InitListExpr and the initializer for the remaining elements
      //      is the array filler.
      //   3) StringLiterals don't have an array filler, so we need to zero
      //      the remaining elements.

      if (DynamicInit || ElemT || InitType->isIncompleteArrayType()) {
        const Function *CtorFunc = nullptr;
        if (const auto *CE = dyn_cast<CXXConstructExpr>(Init)) {
          CtorFunc = getFunction(CE->getConstructor());
          if (!CtorFunc)
            return false;
        } else if (!DynamicInit && !ElemT)
          DynamicInit = Init;

        LabelTy EndLabel = this->getLabel();
        LabelTy StartLabel = this->getLabel();

        // In the nothrow case, the alloc above might have returned nullptr.
        // Don't call any constructors that case.
        if (IsNoThrow) {
          if (!this->emitDupPtr(E))
            return false;
          if (!this->emitIsNonNullPtr(E))
            return false;
          if (!this->jumpFalse(EndLabel, E))
            return false;
        }

        // Create loop variables.
        unsigned Iter =
            allocateLocalPrimitive(Stripped, SizeT, /*IsConst=*/false);
        if (!this->emitConst(StaticInitElems, SizeT, E))
          return false;
        if (!this->emitSetLocal(SizeT, Iter, E))
          return false;

        this->fallthrough(StartLabel);
        this->emitLabel(StartLabel);
        // Condition. Iter < ArrayLen?
        if (!this->emitGetLocal(SizeT, Iter, E))
          return false;
        if (!this->emitGetLocal(SizeT, ArrayLen, E))
          return false;
        if (!this->emitLT(SizeT, E))
          return false;
        if (!this->jumpFalse(EndLabel, E))
          return false;

        // Pointer to the allocated array is already on the stack.
        if (!this->emitGetLocal(SizeT, Iter, E))
          return false;
        if (!this->emitArrayElemPtr(SizeT, E))
          return false;

        if (isa_and_nonnull<ImplicitValueInitExpr>(DynamicInit) &&
            DynamicInit->getType()->isArrayType()) {
          QualType ElemType =
              DynamicInit->getType()->getAsArrayTypeUnsafe()->getElementType();
          if (OptPrimType InitT = classify(ElemType)) {
            if (!this->visitZeroInitializer(*InitT, ElemType, E))
              return false;
            if (!this->emitStorePop(*InitT, E))
              return false;
          } else {
            assert(ElemType->isArrayType());
            if (!this->visitZeroArrayInitializer(ElemType, E))
              return false;
          }
        } else if (DynamicInit) {
          if (OptPrimType InitT = classify(DynamicInit)) {
            if (!this->visit(DynamicInit))
              return false;
            if (!this->emitStorePop(*InitT, E))
              return false;
          } else {
            if (!this->visitInitializerPop(DynamicInit))
              return false;
          }
        } else if (ElemT) {
          if (!this->visitZeroInitializer(
                  *ElemT, InitType->getAsArrayTypeUnsafe()->getElementType(),
                  Init))
            return false;
          if (!this->emitStorePop(*ElemT, E))
            return false;
        } else {
          assert(CtorFunc);
          if (!this->emitCall(CtorFunc, 0, E))
            return false;
        }

        // ++Iter;
        if (!this->emitGetPtrLocal(Iter, E))
          return false;
        if (!this->emitIncPop(SizeT, false, E))
          return false;

        if (!this->jump(StartLabel, E))
          return false;

        this->fallthrough(EndLabel);
        this->emitLabel(EndLabel);
      }
    }
  } else { // Non-array.
    if (PlacementDest) {
      if (!this->visit(PlacementDest))
        return false;
      if (!this->emitCheckNewTypeMismatch(E, E))
        return false;

    } else {
      // Allocate just one element.
      if (!this->emitAlloc(Desc, E))
        return false;
    }

    if (Init) {
      if (ElemT) {
        if (!this->visit(Init))
          return false;

        if (!this->emitInit(*ElemT, E))
          return false;
      } else {
        // Composite.
        if (!this->visitInitializer(Init))
          return false;
      }
    }
  }

  if (DiscardResult)
    return this->emitPopPtr(E);

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
  if (E->containsErrors())
    return false;
  const FunctionDecl *OperatorDelete = E->getOperatorDelete();

  if (!OperatorDelete->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
    return this->emitInvalidNewDeleteExpr(E, E);

  // Arg must be an lvalue.
  if (!this->visit(E->getArgument()))
    return false;

  return this->emitFree(E->isArrayForm(), E->isGlobalDelete(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
  if (DiscardResult)
    return true;

  const Function *Func = nullptr;
  if (const Function *F = Ctx.getOrCreateObjCBlock(E))
    Func = F;

  if (!Func)
    return false;
  return this->emitGetFnPtr(Func, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
  const Type *TypeInfoType = E->getType().getTypePtr();

  auto canonType = [](const Type *T) {
    return T->getCanonicalTypeUnqualified().getTypePtr();
  };

  if (!E->isPotentiallyEvaluated()) {
    if (DiscardResult)
      return true;

    if (E->isTypeOperand())
      return this->emitGetTypeid(
          canonType(E->getTypeOperand(Ctx.getASTContext()).getTypePtr()),
          TypeInfoType, E);

    return this->emitGetTypeid(
        canonType(E->getExprOperand()->getType().getTypePtr()), TypeInfoType,
        E);
  }

  // Otherwise, we need to evaluate the expression operand.
  assert(E->getExprOperand());
  assert(E->getExprOperand()->isLValue());

  if (!Ctx.getLangOpts().CPlusPlus20 && !this->emitDiagTypeid(E))
    return false;

  if (!this->visit(E->getExprOperand()))
    return false;

  if (!this->emitGetTypeidPtr(TypeInfoType, E))
    return false;
  if (DiscardResult)
    return this->emitPopPtr(E);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCDictionaryLiteral(
    const ObjCDictionaryLiteral *E) {
  if (E->isExpressibleAsConstantInitializer())
    return this->emitDummyPtr(E, E);
  return this->emitError(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCArrayLiteral(const ObjCArrayLiteral *E) {
  if (E->isExpressibleAsConstantInitializer())
    return this->emitDummyPtr(E, E);
  return this->emitError(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
  assert(Ctx.getLangOpts().CPlusPlus);
  return this->emitConstBool(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXUuidofExpr(const CXXUuidofExpr *E) {
  if (DiscardResult)
    return true;
  assert(!Initializing);

  const MSGuidDecl *GuidDecl = E->getGuidDecl();
  const RecordDecl *RD = GuidDecl->getType()->getAsRecordDecl();
  assert(RD);
  // If the definiton of the result type is incomplete, just return a dummy.
  // If (and when) that is read from, we will fail, but not now.
  if (!RD->isCompleteDefinition())
    return this->emitDummyPtr(GuidDecl, E);

  UnsignedOrNone GlobalIndex = P.getOrCreateGlobal(GuidDecl);
  if (!GlobalIndex)
    return false;
  if (!this->emitGetPtrGlobal(*GlobalIndex, E))
    return false;

  assert(this->getRecord(E->getType()));

  const APValue &V = GuidDecl->getAsAPValue();
  if (V.getKind() == APValue::None)
    return true;

  assert(V.isStruct());
  assert(V.getStructNumBases() == 0);
  if (!this->visitAPValueInitializer(V, E, E->getType()))
    return false;

  return this->emitFinishInit(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitRequiresExpr(const RequiresExpr *E) {
  assert(classifyPrim(E->getType()) == PT_Bool);
  if (E->isValueDependent())
    return false;
  if (DiscardResult)
    return true;
  return this->emitConstBool(E->isSatisfied(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitConceptSpecializationExpr(
    const ConceptSpecializationExpr *E) {
  assert(classifyPrim(E->getType()) == PT_Bool);
  if (DiscardResult)
    return true;
  return this->emitConstBool(E->isSatisfied(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXRewrittenBinaryOperator(
    const CXXRewrittenBinaryOperator *E) {
  return this->delegate(E->getSemanticForm());
}

template <class Emitter>
bool Compiler<Emitter>::VisitPseudoObjectExpr(const PseudoObjectExpr *E) {

  for (const Expr *SemE : E->semantics()) {
    if (auto *OVE = dyn_cast<OpaqueValueExpr>(SemE)) {
      if (SemE == E->getResultExpr())
        return false;

      if (OVE->isUnique())
        continue;

      if (!this->discard(OVE))
        return false;
    } else if (SemE == E->getResultExpr()) {
      if (!this->delegate(SemE))
        return false;
    } else {
      if (!this->discard(SemE))
        return false;
    }
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitPackIndexingExpr(const PackIndexingExpr *E) {
  return this->delegate(E->getSelectedExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitRecoveryExpr(const RecoveryExpr *E) {
  return this->emitError(E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitAddrLabelExpr(const AddrLabelExpr *E) {
  assert(E->getType()->isVoidPointerType());
  if (DiscardResult)
    return true;

  return this->emitDummyPtr(E, E);
}

template <class Emitter>
bool Compiler<Emitter>::emitVectorConversion(const Expr *Src, const Expr *E) {
  if (Src->containsErrors())
    return false;

  const auto *VT = E->getType()->castAs<VectorType>();
  QualType ElemType = VT->getElementType();
  PrimType ElemT = classifyPrim(ElemType);
  QualType SrcType = Src->getType();
  PrimType SrcElemT = classifyVectorElementType(SrcType);

  if (!Initializing) {
    UnsignedOrNone LocalIndex = allocateLocal(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  unsigned SrcOffset =
      this->allocateLocalPrimitive(Src, PT_Ptr, /*IsConst=*/true);
  if (!this->visit(Src))
    return false;
  if (!this->emitSetLocal(PT_Ptr, SrcOffset, E))
    return false;

  for (unsigned I = 0; I != VT->getNumElements(); ++I) {
    if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
      return false;
    if (!this->emitArrayElemPop(SrcElemT, I, E))
      return false;

    // Cast to the desired result element type.
    if (SrcElemT != ElemT) {
      if (!this->emitPrimCast(SrcElemT, ElemT, ElemType, E))
        return false;
    } else if (ElemType->isFloatingType() && SrcType != ElemType) {
      const auto *TargetSemantics = &Ctx.getFloatSemantics(ElemType);
      if (!this->emitCastFP(TargetSemantics, getRoundingMode(E), E))
        return false;
    }
    if (!this->emitInitElem(ElemT, I, E))
      return false;
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitConvertVectorExpr(const ConvertVectorExpr *E) {
  return emitVectorConversion(E->getSrcExpr(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitShuffleVectorExpr(const ShuffleVectorExpr *E) {
  // FIXME: Unary shuffle with mask not currently supported.
  if (E->getNumSubExprs() == 2)
    return this->emitInvalid(E);

  assert(E->getNumSubExprs() > 2);

  const Expr *Vecs[] = {E->getExpr(0), E->getExpr(1)};
  const VectorType *VT = Vecs[0]->getType()->castAs<VectorType>();
  PrimType ElemT = classifyPrim(VT->getElementType());
  unsigned NumInputElems = VT->getNumElements();
  unsigned NumOutputElems = E->getNumSubExprs() - 2;
  assert(NumOutputElems > 0);

  if (!Initializing) {
    UnsignedOrNone LocalIndex = allocateLocal(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  // Save both input vectors to a local variable.
  unsigned VectorOffsets[2];
  for (unsigned I = 0; I != 2; ++I) {
    VectorOffsets[I] =
        this->allocateLocalPrimitive(Vecs[I], PT_Ptr, /*IsConst=*/true);
    if (!this->visit(Vecs[I]))
      return false;
    if (!this->emitSetLocal(PT_Ptr, VectorOffsets[I], E))
      return false;
  }
  for (unsigned I = 0; I != NumOutputElems; ++I) {
    APSInt ShuffleIndex = E->getShuffleMaskIdx(I);
    assert(ShuffleIndex >= -1);
    if (ShuffleIndex == -1)
      return this->emitInvalidShuffleVectorIndex(I, E);

    assert(ShuffleIndex < (NumInputElems * 2));
    if (!this->emitGetLocal(PT_Ptr,
                            VectorOffsets[ShuffleIndex >= NumInputElems], E))
      return false;
    unsigned InputVectorIndex = ShuffleIndex.getZExtValue() % NumInputElems;
    if (!this->emitArrayElemPop(ElemT, InputVectorIndex, E))
      return false;

    if (!this->emitInitElem(ElemT, I, E))
      return false;
  }

  if (DiscardResult)
    return this->emitPopPtr(E);

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitExtVectorElementExpr(
    const ExtVectorElementExpr *E) {
  const Expr *Base = E->getBase();
  assert(
      Base->getType()->isVectorType() ||
      Base->getType()->getAs<PointerType>()->getPointeeType()->isVectorType());

  SmallVector<uint32_t, 4> Indices;
  E->getEncodedElementAccess(Indices);

  if (Indices.size() == 1) {
    if (!this->visit(Base))
      return false;

    if (E->isGLValue()) {
      if (!this->emitConstUint32(Indices[0], E))
        return false;
      return this->emitArrayElemPtrPop(PT_Uint32, E);
    }
    // Else, also load the value.
    return this->emitArrayElemPop(classifyPrim(E->getType()), Indices[0], E);
  }

  // Create a local variable for the base.
  unsigned BaseOffset = allocateLocalPrimitive(Base, PT_Ptr, /*IsConst=*/true);
  if (!this->visit(Base))
    return false;
  if (!this->emitSetLocal(PT_Ptr, BaseOffset, E))
    return false;

  // Now the vector variable for the return value.
  if (!Initializing) {
    UnsignedOrNone ResultIndex = allocateLocal(E);
    if (!ResultIndex)
      return false;
    if (!this->emitGetPtrLocal(*ResultIndex, E))
      return false;
  }

  assert(Indices.size() == E->getType()->getAs<VectorType>()->getNumElements());

  PrimType ElemT =
      classifyPrim(E->getType()->getAs<VectorType>()->getElementType());
  uint32_t DstIndex = 0;
  for (uint32_t I : Indices) {
    if (!this->emitGetLocal(PT_Ptr, BaseOffset, E))
      return false;
    if (!this->emitArrayElemPop(ElemT, I, E))
      return false;
    if (!this->emitInitElem(ElemT, DstIndex, E))
      return false;
    ++DstIndex;
  }

  // Leave the result pointer on the stack.
  assert(!DiscardResult);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
  const Expr *SubExpr = E->getSubExpr();
  if (!E->isExpressibleAsConstantInitializer())
    return this->discard(SubExpr) && this->emitInvalid(E);

  if (DiscardResult)
    return true;

  assert(classifyPrim(E) == PT_Ptr);
  return this->emitDummyPtr(E, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXStdInitializerListExpr(
    const CXXStdInitializerListExpr *E) {
  const Expr *SubExpr = E->getSubExpr();
  const ConstantArrayType *ArrayType =
      Ctx.getASTContext().getAsConstantArrayType(SubExpr->getType());
  const Record *R = getRecord(E->getType());
  assert(Initializing);
  assert(SubExpr->isGLValue());

  if (!this->visit(SubExpr))
    return false;
  if (!this->emitConstUint8(0, E))
    return false;
  if (!this->emitArrayElemPtrPopUint8(E))
    return false;
  if (!this->emitInitFieldPtr(R->getField(0u)->Offset, E))
    return false;

  PrimType SecondFieldT = *R->getField(1u)->T;
  if (isIntegerOrBoolType(SecondFieldT)) {
    if (!this->emitConst(ArrayType->getSize(), SecondFieldT, E))
      return false;
    return this->emitInitField(SecondFieldT, R->getField(1u)->Offset, E);
  }
  assert(SecondFieldT == PT_Ptr);

  if (!this->emitGetFieldPtr(R->getField(0u)->Offset, E))
    return false;
  if (!this->emitExpandPtr(E))
    return false;
  if (!this->emitConst(ArrayType->getSize(), PT_Uint64, E))
    return false;
  if (!this->emitArrayElemPtrPop(PT_Uint64, E))
    return false;
  return this->emitInitFieldPtr(R->getField(1u)->Offset, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitStmtExpr(const StmtExpr *E) {
  LocalScope<Emitter> BS(this);
  llvm::SaveAndRestore StmtExprSAR(this->InStmtExpr, true);

  const CompoundStmt *CS = E->getSubStmt();
  const Stmt *Result = CS->body_back();
  for (const Stmt *S : CS->body()) {
    if (S != Result) {
      if (!this->visitStmt(S))
        return false;
      continue;
    }

    assert(S == Result);
    if (const Expr *ResultExpr = dyn_cast<Expr>(S))
      return this->delegate(ResultExpr);
    if (!this->visitStmt(S))
      return false;
    return this->emitUnsupported(E);
  }

  return BS.destroyLocals();
}

template <class Emitter> bool Compiler<Emitter>::discard(const Expr *E) {
  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
                             /*NewInitializing=*/false, /*ToLValue=*/false);
  return this->Visit(E);
}

template <class Emitter> bool Compiler<Emitter>::delegate(const Expr *E) {
  // We're basically doing:
  // OptionScope<Emitter> Scope(this, DicardResult, Initializing, ToLValue);
  // but that's unnecessary of course.
  return this->Visit(E);
}

static const Expr *stripCheckedDerivedToBaseCasts(const Expr *E) {
  if (const auto *PE = dyn_cast<ParenExpr>(E))
    return stripCheckedDerivedToBaseCasts(PE->getSubExpr());

  if (const auto *CE = dyn_cast<CastExpr>(E);
      CE &&
      (CE->getCastKind() == CK_DerivedToBase || CE->getCastKind() == CK_NoOp))
    return stripCheckedDerivedToBaseCasts(CE->getSubExpr());

  return E;
}

static const Expr *stripDerivedToBaseCasts(const Expr *E) {
  if (const auto *PE = dyn_cast<ParenExpr>(E))
    return stripDerivedToBaseCasts(PE->getSubExpr());

  if (const auto *CE = dyn_cast<CastExpr>(E);
      CE && (CE->getCastKind() == CK_DerivedToBase ||
             CE->getCastKind() == CK_UncheckedDerivedToBase ||
             CE->getCastKind() == CK_NoOp))
    return stripDerivedToBaseCasts(CE->getSubExpr());

  return E;
}

template <class Emitter> bool Compiler<Emitter>::visit(const Expr *E) {
  if (E->getType().isNull())
    return false;

  if (E->getType()->isVoidType())
    return this->discard(E);

  // Create local variable to hold the return value.
  if (!E->isGLValue() && !canClassify(E->getType())) {
    UnsignedOrNone LocalIndex = allocateLocal(
        stripDerivedToBaseCasts(E), QualType(), ScopeKind::FullExpression);
    if (!LocalIndex)
      return false;

    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
    InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalIndex));
    return this->visitInitializer(E);
  }

  //  Otherwise,we have a primitive return value, produce the value directly
  //  and push it on the stack.
  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/false, /*ToLValue=*/ToLValue);
  return this->Visit(E);
}

template <class Emitter>
bool Compiler<Emitter>::visitInitializer(const Expr *E) {
  assert(!canClassify(E->getType()));

  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/true, /*ToLValue=*/false);
  return this->Visit(E) && this->emitFinishInit(E);
}

template <class Emitter>
bool Compiler<Emitter>::visitInitializerPop(const Expr *E) {
  assert(!canClassify(E->getType()));

  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/true, /*ToLValue=*/false);
  return this->Visit(E) && this->emitFinishInitPop(E);
}

template <class Emitter> bool Compiler<Emitter>::visitAsLValue(const Expr *E) {
  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/false, /*ToLValue=*/true);
  return this->Visit(E);
}

template <class Emitter> bool Compiler<Emitter>::visitBool(const Expr *E) {
  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/false, /*ToLValue=*/ToLValue);

  OptPrimType T = classify(E->getType());
  if (!T) {
    // Convert complex values to bool.
    if (E->getType()->isAnyComplexType()) {
      if (!this->visit(E))
        return false;
      return this->emitComplexBoolCast(E);
    }
    return false;
  }

  if (!this->visit(E))
    return false;

  if (T == PT_Bool)
    return true;

  // Convert pointers to bool.
  if (T == PT_Ptr)
    return this->emitIsNonNullPtr(E);

  // Or Floats.
  if (T == PT_Float)
    return this->emitCastFloatingIntegralBool(getFPOptions(E), E);

  // Or anything else we can.
  return this->emitCast(*T, PT_Bool, E);
}

template <class Emitter>
bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
                                             const Expr *E) {
  if (const auto *AT = QT->getAs<AtomicType>())
    QT = AT->getValueType();

  switch (T) {
  case PT_Bool:
    return this->emitZeroBool(E);
  case PT_Sint8:
    return this->emitZeroSint8(E);
  case PT_Uint8:
    return this->emitZeroUint8(E);
  case PT_Sint16:
    return this->emitZeroSint16(E);
  case PT_Uint16:
    return this->emitZeroUint16(E);
  case PT_Sint32:
    return this->emitZeroSint32(E);
  case PT_Uint32:
    return this->emitZeroUint32(E);
  case PT_Sint64:
    return this->emitZeroSint64(E);
  case PT_Uint64:
    return this->emitZeroUint64(E);
  case PT_IntAP:
    return this->emitZeroIntAP(Ctx.getBitWidth(QT), E);
  case PT_IntAPS:
    return this->emitZeroIntAPS(Ctx.getBitWidth(QT), E);
  case PT_Ptr:
    return this->emitNullPtr(Ctx.getASTContext().getTargetNullPointerValue(QT),
                             nullptr, E);
  case PT_MemberPtr:
    return this->emitNullMemberPtr(0, nullptr, E);
  case PT_Float: {
    APFloat F = APFloat::getZero(Ctx.getFloatSemantics(QT));
    return this->emitFloat(F, E);
  }
  case PT_FixedPoint: {
    auto Sem = Ctx.getASTContext().getFixedPointSemantics(QT);
    return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
  }
  }
  llvm_unreachable("unknown primitive type");
}

template <class Emitter>
bool Compiler<Emitter>::visitZeroRecordInitializer(const Record *R,
                                                   const Expr *E,
                                                   bool IsCompleteClass) {
  assert(E);
  assert(R);
  // Fields
  for (const Record::Field &Field : R->fields()) {
    if (Field.isUnnamedBitField())
      continue;

    const Descriptor *D = Field.Desc;
    if (D->isPrimitive()) {
      QualType QT = D->getType();
      PrimType T = D->getPrimType();
      if (!this->visitZeroInitializer(T, QT, E))
        return false;
      if (R->isUnion()) {
        if (!this->emitInitFieldActivate(T, Field.Offset, E))
          return false;
        break;
      }
      if (!this->emitInitField(T, Field.Offset, E))
        return false;
      continue;
    }

    if (!this->emitGetPtrField(Field.Offset, E))
      return false;

    if (D->isPrimitiveArray()) {
      QualType ET = D->getElemQualType();
      PrimType T = D->getPrimType();
      for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
        if (!this->visitZeroInitializer(T, ET, E))
          return false;
        if (!this->emitInitElem(T, I, E))
          return false;
      }
    } else if (D->isCompositeArray()) {
      // Can't be a vector or complex field.
      if (!this->visitZeroArrayInitializer(D->getType(), E))
        return false;
    } else if (D->isRecord()) {
      if (!this->visitZeroRecordInitializer(D->ElemRecord, E))
        return false;
    } else
      return false;

    // C++11 [dcl.init]p5: If T is a (possibly cv-qualified) union type, the
    // object's first non-static named data member is zero-initialized
    if (R->isUnion()) {
      if (!this->emitFinishInitActivatePop(E))
        return false;
      break;
    }
    if (!this->emitFinishInitPop(E))
      return false;
  }

  for (const Record::Base &B : R->bases()) {
    if (!this->emitGetPtrBase(B.Offset, E))
      return false;
    if (!this->visitZeroRecordInitializer(B.R, E, /*IsCompleteClass=*/false))
      return false;
    if (!this->emitFinishInitPop(E))
      return false;
  }

  if (IsCompleteClass) {
    for (const Record::Base &B : R->virtual_bases()) {
      if (!this->emitGetPtrVirtBase(cast<CXXRecordDecl>(B.R->getDecl()), E))
        return false;
      if (!this->visitZeroRecordInitializer(B.R, E, /*IsCompleteClass=*/false))
        return false;
      if (!this->emitFinishInitPop(E))
        return false;
    }
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::visitZeroArrayInitializer(QualType T, const Expr *E) {
  assert(T->isArrayType() || T->isAnyComplexType() || T->isVectorType());
  const ArrayType *AT = T->getAsArrayTypeUnsafe();
  QualType ElemType = AT->getElementType();
  size_t NumElems = cast<ConstantArrayType>(AT)->getZExtSize();

  if (OptPrimType ElemT = classify(ElemType)) {
    for (size_t I = 0; I != NumElems; ++I) {
      if (!this->visitZeroInitializer(*ElemT, ElemType, E))
        return false;
      if (!this->emitInitElem(*ElemT, I, E))
        return false;
    }
    return true;
  }
  if (ElemType->isRecordType()) {
    const Record *R = getRecord(ElemType);
    if (!R)
      return false;

    for (size_t I = 0; I != NumElems; ++I) {
      if (!this->emitConstUint32(I, E))
        return false;
      if (!this->emitArrayElemPtr(PT_Uint32, E))
        return false;
      if (!this->visitZeroRecordInitializer(R, E))
        return false;
      if (!this->emitPopPtr(E))
        return false;
    }
    return true;
  }
  if (ElemType->isArrayType()) {
    for (size_t I = 0; I != NumElems; ++I) {
      if (!this->emitConstUint32(I, E))
        return false;
      if (!this->emitArrayElemPtr(PT_Uint32, E))
        return false;
      if (!this->visitZeroArrayInitializer(ElemType, E))
        return false;
      if (!this->emitPopPtr(E))
        return false;
    }
    return true;
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::visitAssignment(const Expr *LHS, const Expr *RHS,
                                        const Expr *E) {
  if (!canClassify(E->getType()))
    return false;

  bool NeedsFlip = !isSideEffectFree(RHS);
  if (!NeedsFlip) {
    if (!this->visit(LHS))
      return false;
    if (!this->visit(RHS))
      return false;
  } else {
    if (!this->visit(RHS))
      return false;
    if (!this->visit(LHS))
      return false;
  }

  if (LHS->getType().isVolatileQualified())
    return this->emitInvalidStore(LHS->getType().getTypePtr(), E);

  // We don't support assignments in C.
  if (!Ctx.getLangOpts().CPlusPlus && !this->emitInvalid(E))
    return false;

  PrimType RHT = classifyPrim(RHS);
  bool Activates = refersToUnion(LHS);
  bool BitField = LHS->refersToBitField();

  if (NeedsFlip && !this->emitFlip(PT_Ptr, RHT, E))
    return false;

  if (DiscardResult) {
    if (BitField && Activates)
      return this->emitStoreBitFieldActivatePop(RHT, E);
    if (BitField)
      return this->emitStoreBitFieldPop(RHT, E);
    if (Activates)
      return this->emitStoreActivatePop(RHT, E);
    // Otherwise, regular non-activating store.
    return this->emitStorePop(RHT, E);
  }

  auto maybeLoad = [&](bool Result) -> bool {
    if (!Result)
      return false;
    // Assignments aren't necessarily lvalues in C.
    // Load from them in that case.
    if (!E->isLValue())
      return this->emitLoadPop(RHT, E);
    return true;
  };

  if (BitField && Activates)
    return maybeLoad(this->emitStoreBitFieldActivate(RHT, E));
  if (BitField)
    return maybeLoad(this->emitStoreBitField(RHT, E));
  if (Activates)
    return maybeLoad(this->emitStoreActivate(RHT, E));
  // Otherwise, regular non-activating store.
  return maybeLoad(this->emitStore(RHT, E));
}

template <class Emitter>
template <typename T>
bool Compiler<Emitter>::emitConst(T Value, PrimType Ty, SourceInfo Info) {
  switch (Ty) {
  case PT_Sint8:
    return this->emitConstSint8(Value, Info);
  case PT_Uint8:
    return this->emitConstUint8(Value, Info);
  case PT_Sint16:
    return this->emitConstSint16(Value, Info);
  case PT_Uint16:
    return this->emitConstUint16(Value, Info);
  case PT_Sint32:
    return this->emitConstSint32(Value, Info);
  case PT_Uint32:
    return this->emitConstUint32(Value, Info);
  case PT_Sint64:
    return this->emitConstSint64(Value, Info);
  case PT_Uint64:
    return this->emitConstUint64(Value, Info);
  case PT_Bool:
    return this->emitConstBool(Value, Info);
  case PT_Ptr:
  case PT_MemberPtr:
  case PT_Float:
  case PT_IntAP:
  case PT_IntAPS:
  case PT_FixedPoint:
    llvm_unreachable("Invalid integral type");
    break;
  }
  llvm_unreachable("unknown primitive type");
}

template <class Emitter>
template <typename T>
bool Compiler<Emitter>::emitConst(T Value, const Expr *E) {
  return this->emitConst(Value, classifyPrim(E->getType()), E);
}

template <class Emitter>
bool Compiler<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
                                  SourceInfo Info) {
  if (Ty == PT_IntAPS)
    return this->emitConstIntAPS(Value, Info);
  if (Ty == PT_IntAP)
    return this->emitConstIntAP(Value, Info);

  if (Value.isSigned())
    return this->emitConst(Value.getSExtValue(), Ty, Info);
  return this->emitConst(Value.getZExtValue(), Ty, Info);
}

template <class Emitter>
bool Compiler<Emitter>::emitConst(const APInt &Value, PrimType Ty,
                                  SourceInfo Info) {
  if (Ty == PT_IntAPS)
    return this->emitConstIntAPS(Value, Info);
  if (Ty == PT_IntAP)
    return this->emitConstIntAP(Value, Info);

  if (isSignedType(Ty))
    return this->emitConst(Value.getSExtValue(), Ty, Info);
  return this->emitConst(Value.getZExtValue(), Ty, Info);
}

template <class Emitter>
bool Compiler<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
  return this->emitConst(Value, classifyPrim(E->getType()), E);
}

template <class Emitter>
unsigned Compiler<Emitter>::allocateLocalPrimitive(DeclOrExpr Src, PrimType Ty,
                                                   bool IsConst,
                                                   bool IsVolatile,
                                                   ScopeKind SC) {
  // FIXME: There are cases where Src.isExpr() is wrong, e.g.
  //   (int){12} in C. Consider using Expr::isTemporaryObject() instead
  //   or isa<MaterializeTemporaryExpr>().
  Descriptor *D = P.createDescriptor(Src, Ty, nullptr, IsConst, Src.isExpr(),
                                     /*IsMutable=*/false, IsVolatile);
  D->IsConstexprUnknown = this->VariablesAreConstexprUnknown;
  Scope::Local Local = this->createLocal(D);
  if (auto *VD = Src.asValueDecl())
    Locals.insert({VD, Local});
  VarScope->addForScopeKind(Local, SC);
  return Local.Offset;
}

template <class Emitter>
UnsignedOrNone Compiler<Emitter>::allocateLocal(DeclOrExpr Src, QualType Ty,
                                                ScopeKind SC) {
  const ValueDecl *Key = nullptr;
  const Expr *Init = nullptr;
  bool IsTemporary = false;
  if (auto *VD = Src.asValueDecl()) {
    Key = VD;

    if (const auto *VarD = dyn_cast<VarDecl>(VD))
      Init = VarD->getInit();
  }
  if (const auto *E = Src.asExpr()) {
    IsTemporary = true;
    if (Ty.isNull())
      Ty = E->getType();
  }

  Descriptor *D = P.createDescriptor(
      Src, Ty.getTypePtr(), Ty.isConstQualified(), IsTemporary,
      /*IsMutable=*/false, /*IsVolatile=*/Ty.isVolatileQualified(), Init);
  if (!D)
    return std::nullopt;
  D->IsConstexprUnknown = this->VariablesAreConstexprUnknown;

  Scope::Local Local = this->createLocal(D);
  if (Key)
    Locals.insert({Key, Local});
  VarScope->addForScopeKind(Local, SC);
  return Local.Offset;
}

template <class Emitter>
UnsignedOrNone Compiler<Emitter>::allocateTemporary(const Expr *E) {
  QualType Ty = E->getType();
  assert(!Ty->isRecordType());

  Descriptor *D = P.createDescriptor(E, Ty.getTypePtr(), Ty.isConstQualified(),
                                     /*IsTemporary=*/true);

  if (!D)
    return std::nullopt;

  Scope::Local Local = this->createLocal(D);
  VariableScope<Emitter> *S = VarScope;
  assert(S);
  // Attach to topmost scope.
  while (S->getParent())
    S = S->getParent();
  assert(S && !S->getParent());
  S->addLocal(Local);
  return Local.Offset;
}

template <class Emitter>
const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
  if (const PointerType *PT = dyn_cast<PointerType>(Ty))
    return PT->getPointeeType()->getAsCanonical<RecordType>();
  return Ty->getAsCanonical<RecordType>();
}

template <class Emitter> Record *Compiler<Emitter>::getRecord(QualType Ty) {
  if (const auto *RecordTy = getRecordTy(Ty))
    return getRecord(RecordTy->getDecl()->getDefinitionOrSelf());
  return nullptr;
}

template <class Emitter>
Record *Compiler<Emitter>::getRecord(const RecordDecl *RD) {
  return P.getOrCreateRecord(RD);
}

template <class Emitter>
const Function *Compiler<Emitter>::getFunction(const FunctionDecl *FD) {
  return Ctx.getOrCreateFunction(FD);
}

template <class Emitter>
bool Compiler<Emitter>::visitExpr(const Expr *E, bool DestroyToplevelScope) {
  assert(E);
  assert(!E->getType().isNull());
  LocalScope<Emitter> RootScope(this, ScopeKind::FullExpression);

  auto maybeDestroyLocals = [&]() -> bool {
    if (DestroyToplevelScope)
      return RootScope.destroyLocals() && this->emitCheckAllocations(E);
    return this->emitCheckAllocations(E);
  };

  // Void expressions.
  if (E->getType()->isVoidType()) {
    if (!visit(E))
      return false;
    return this->emitRetVoid(E) && maybeDestroyLocals();
  }

  // Expressions with a primitive return type.
  if (OptPrimType T = classify(E)) {
    if (!visit(E))
      return false;

    return this->emitRet(*T, E) && maybeDestroyLocals();
  }

  // Expressions with a composite return type.
  // For us, that means everything we don't
  // have a PrimType for.
  if (UnsignedOrNone LocalOffset = this->allocateLocal(E)) {
    InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalOffset));
    if (!this->emitGetPtrLocal(*LocalOffset, E))
      return false;

    if (!visitInitializer(E))
      return false;
    // We are destroying the locals AFTER the Ret op.
    // The Ret op needs to copy the (alive) values, but the
    // destructors may still turn the entire expression invalid.
    return this->emitRetValue(E) && maybeDestroyLocals();
  }

  return maybeDestroyLocals() && false;
}

template <class Emitter>
bool Compiler<Emitter>::visitLValueExpr(const Expr *E,
                                        bool DestroyToplevelScope) {
  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
                             /*NewInitializing=*/false, /*ToLValue=*/true);

  return this->visitExpr(E, DestroyToplevelScope);
}

template <class Emitter>
VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD) {

  auto R = this->visitVarDecl(VD, VD->getInit(), /*Toplevel=*/true);

  if (R.notCreated())
    return R;

  if (R)
    return true;

  if (!R && Context::shouldBeGloballyIndexed(VD)) {
    if (auto GlobalIndex = P.getGlobal(VD)) {
      Block *GlobalBlock = P.getGlobal(*GlobalIndex);
      auto &GD = GlobalBlock->getBlockDesc<GlobalInlineDescriptor>();

      GD.InitState = GlobalInitState::InitializerFailed;
      GlobalBlock->invokeDtor();
    }
  }

  return R;
}

/// Toplevel visitDeclAndReturn().
/// We get here from evaluateAsInitializer().
/// We need to evaluate the initializer and return its value.
template <class Emitter>
bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD, const Expr *Init,
                                           bool ConstantContext) {
  // We only create variables if we're evaluating in a constant context.
  // Otherwise, just evaluate the initializer and return it.
  if (!ConstantContext) {
    DeclScope<Emitter> LS(this, VD);
    if (!this->visit(Init))
      return false;
    return this->emitRet(classify(Init).value_or(PT_Ptr), VD) &&
           LS.destroyLocals() && this->emitCheckAllocations(VD);
  }

  LocalScope<Emitter> VDScope(this);
  if (!this->visitVarDecl(VD, Init, /*Toplevel=*/true))
    return false;

  OptPrimType VarT = classify(VD->getType());
  bool IsReference = VD->getType()->isReferenceType();
  if (Context::shouldBeGloballyIndexed(VD)) {
    auto GlobalIndex = P.getGlobal(VD);
    assert(GlobalIndex); // visitVarDecl() didn't return false.
    if (VarT) {
      if (!this->emitGetGlobalUnchecked(*VarT, *GlobalIndex, VD))
        return false;
    } else {
      if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
        return false;
    }
  } else {
    auto Local = Locals.find(VD);
    assert(Local != Locals.end()); // Same here.
    if (VarT) {
      if (IsReference) {
        if (!this->emitGetRefLocal(Local->second.Offset, VD))
          return false;
      } else if (!this->emitGetLocal(*VarT, Local->second.Offset, VD))
        return false;
    } else {
      if (!this->emitGetPtrLocal(Local->second.Offset, VD))
        return false;
    }
  }

  // Return the value.
  if (!this->emitRet(VarT.value_or(PT_Ptr), VD)) {
    // If the Ret above failed and this is a global variable. Mark it as
    // uninitialized, even if everything else succeeded.
    if (Context::shouldBeGloballyIndexed(VD)) {
      auto GlobalIndex = P.getGlobal(VD);
      assert(GlobalIndex);
      Block *GlobalBlock = P.getGlobal(*GlobalIndex);
      auto &GD = GlobalBlock->getBlockDesc<GlobalInlineDescriptor>();

      GD.InitState = GlobalInitState::InitializerFailed;
      GlobalBlock->invokeDtor();
    }
    return false;
  }

  return VDScope.destroyLocals() && this->emitCheckAllocations(VD);
}

template <class Emitter>
VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
                                                 const Expr *Init,
                                                 bool Toplevel) {
  QualType VarTy = VD->getType();
  // We don't know what to do with these, so just return false.
  if (VarTy.isNull())
    return false;

  // This case is EvalEmitter-only. If we won't create any instructions for the
  // initializer anyway, don't bother creating the variable in the first place.
  if (!this->isActive())
    return VarCreationState::NotCreated();

  OptPrimType VarT = classify(VD->getType());

  if (Init && Init->isValueDependent())
    return false;

  if (Context::shouldBeGloballyIndexed(VD)) {
    auto checkDecl = [&]() -> bool {
      bool NeedsOp = !Toplevel && VD->isLocalVarDecl() && VD->isStaticLocal();
      return !NeedsOp || this->emitCheckDecl(VD, VD);
    };

    DeclScope<Emitter> LocalScope(this, VD);
    UnsignedOrNone GlobalIndex = P.getGlobal(VD);
    if (GlobalIndex) {
      // The global was previously created but the initializer failed.
      if (!P.getGlobal(*GlobalIndex)->isInitialized())
        return false;
      // We've already seen and initialized this global.
      if (P.isGlobalInitialized(*GlobalIndex))
        return checkDecl();
      // The previous attempt at initialization might've been unsuccessful,
      // so let's try this one.
    } else if ((GlobalIndex =
                    P.createGlobal(VD, Init, VariablesAreConstexprUnknown))) {
    } else {
      return false;
    }
    if (!Init)
      return true;

    if (!checkDecl())
      return false;

    if (VarT) {
      if (!this->visit(Init))
        return false;

      return this->emitInitGlobal(*VarT, *GlobalIndex, VD);
    }

    if (!this->emitGetPtrGlobal(*GlobalIndex, Init))
      return false;

    if (!this->emitStartInit(Init))
      return false;

    if (!visitInitializer(Init))
      return false;

    if (!this->emitEndInit(Init))
      return false;

    return this->emitFinishInitGlobal(Init);
  }
  // Local variables.
  InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));

  if (VarT) {
    unsigned Offset = this->allocateLocalPrimitive(
        VD, *VarT, VarTy.isConstQualified(), VarTy.isVolatileQualified(),
        ScopeKind::Block);

    if (!Init || Init->getType()->isVoidType())
      return true;

    // If this is a toplevel declaration, create a scope for the
    // initializer.
    if (Toplevel) {
      LocalScope<Emitter> Scope(this);
      if (!this->visit(Init))
        return false;
      return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
    }
    if (!this->visit(Init))
      return false;

    if (VarTy->isReferenceType()) {
      // [C++26][decl.ref]
      // The object designated by such a glvalue can be outside its lifetime
      // Because a null pointer value or a pointer past the end of an object
      // does not point to an object, a reference in a well-defined program
      // cannot refer to such things;
      assert(classifyPrim(VarTy) == PT_Ptr);
      if (!this->emitCheckRefInit(Init))
        return false;
    }

    return this->emitSetLocal(*VarT, Offset, VD);
  }
  // Local composite variables.
  if (UnsignedOrNone Offset =
          this->allocateLocal(VD, VarTy, ScopeKind::Block)) {
    if (!Init)
      return true;

    if (!this->emitGetPtrLocal(*Offset, Init))
      return false;

    return visitInitializerPop(Init);
  }
  return false;
}

template <class Emitter>
bool Compiler<Emitter>::visitDtorCall(const VarDecl *VD, const APValue &Value) {
  assert(!canClassify(VD->getType()));

  DeclScope<Emitter> LocalScope(this, VD);
  // Create a local variable to use as the instance.
  QualType Ty = VD->getType();
  Descriptor *D =
      P.createDescriptor(VD, Ty.getTypePtr(), /*IsConst=*/Ty.isConstQualified(),
                         /*IsTemporary=*/false, /*IsMutable=*/false,
                         /*IsVolatile=*/Ty.isVolatileQualified(), nullptr);
  if (!D)
    return false;

  // FIXME: Would be nice if we didn't allocate the descriptor at all in this
  // case.
  if (D->hasTrivialDtor())
    return true;

  Scope::Local Local = this->createLocal(D);
  Locals.insert({VD, Local});
  VarScope->addForScopeKind(Local, ScopeKind::Block);

  if (!this->emitGetPtrLocal(Local.Offset, VD))
    return false;

  if (!this->visitAPValueInitializer(Value, VD, Ty))
    return false;

  return this->emitDestructionPop(D, VD);
}

class ParamFinder : public ConstDynamicRecursiveASTVisitor {
public:
  llvm::SmallPtrSet<const ParmVarDecl *, 1> FoundParams;
  explicit ParamFinder() {}

  bool VisitDeclRefExpr(const DeclRefExpr *E) override {
    if (const auto *P = dyn_cast<ParmVarDecl>(E->getDecl()))
      FoundParams.insert(P);
    return true;
  }
};

/// Evaluate the \p Condition as if it was in the body of \p Callee.
/// Specifically, all the parameters of the callee are available to use
/// for the condition, and their values are given by \p Args (and \p This).
///
// Since this is a somewhat niche feature, we're abusing a few other mechanisms
// to implement this.
//
// We don't create an actual function frame but instead register the parameters
// as local variables.
//
// So we evaluate something like:
//
// bool thisfunc() {
//    auto Arg0 = Args[0];
//    ...
//    return Condition;
// }
//
template <class Emitter>
bool Compiler<Emitter>::visitWithSubstitutions(const FunctionDecl *Callee,
                                               ArrayRef<const Expr *> Args,
                                               const Expr *This,
                                               const Expr *Condition) {
  // Instead of evaluating all parameters and trying to ignore failure,
  // we collect all the parameters used in the condition and only evaluate
  // those. Note that we still ignore failure in the loop below because the
  // failure might be inconsequential in the end,
  // e.g. in the case of `true || x`.
  ParamFinder PF;
  PF.TraverseStmt(Condition);

  LocalScope<Emitter> ArgScope(this);
  for (const ParmVarDecl *PVD : PF.FoundParams) {
    unsigned ParamIndex = 0;
    for (const ParmVarDecl *P : Callee->parameters()) {
      if (P == PVD)
        break;
      ++ParamIndex;
    }

    const Expr *Arg = Args[ParamIndex];
    const ParmVarDecl *Param = Callee->getParamDecl(ParamIndex);
    if (OptPrimType ParamT = classify(Param->getType())) {
      unsigned ArgOffset =
          allocateLocalPrimitive(Param, *ParamT, /*IsConst=*/true);
      if (!this->visit(Arg))
        continue;
      if (!this->emitSetLocal(*ParamT, ArgOffset, Arg))
        return false;
    } else {
      UnsignedOrNone ArgOffset = this->allocateLocal(Param, Param->getType());
      if (!ArgOffset)
        return false;
      if (!this->emitGetPtrLocal(*ArgOffset, Arg))
        return false;
      if (!this->visitInitializerPop(Arg))
        continue;
    }
  }

  if (This) {
    // We abuse the init stack for this and tell it to use
    // either a local variable or another decl for the This pointer.
    this->InitStackActive = true;

    if (This->getType()->isPointerType()) {
      // Nothing to do here, the evaluation will fail if the instance
      // pointer is used.
    } else if (const auto *DRE = dyn_cast<DeclRefExpr>(This)) {
      InitStack.push_back(InitLink::Decl(DRE->getDecl()));
    } else {
      assert(!canClassify(This->getType()));
      UnsignedOrNone ArgOffset = this->allocateLocal(This, This->getType());
      if (!ArgOffset)
        return false;
      if (!this->emitGetPtrLocal(*ArgOffset, This))
        return false;
      if (!this->visitInitializerPop(This))
        return false;
      this->InitStack.push_back(InitLink::Temp(*ArgOffset));
    }
  }

  // Destruction of the argument values is part of the callee frame,
  // so we simply ignore them here.
  this->VarScope = nullptr;

  LocalScope<Emitter> RetScope(this);
  if (!this->visit(Condition))
    return false;
  if (!RetScope.destroyLocals())
    return false;

  // Result of the condition should be on the stack.
  return this->emitRet(PT_Bool, Condition);
}

template <class Emitter>
bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
                                     SourceInfo Info) {
  assert(!Val.isIndeterminate() && "Needs to be checked before");
  assert(!DiscardResult);
  if (Val.isInt())
    return this->emitConst(Val.getInt(), ValType, Info);
  if (Val.isFloat())
    return this->emitFloat(Val.getFloat(), Info);

  if (Val.isMemberPointer()) {
    if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl()) {
      if (!this->emitGetMemberPtr(MemberDecl, Info))
        return false;

      bool IsDerived = Val.isMemberPointerToDerivedMember();
      // Apply the member pointer path.
      for (const CXXRecordDecl *PathEntry : Val.getMemberPointerPath()) {
        if (!this->emitCopyMemberPtrPath(PathEntry, IsDerived, Info))
          return false;
      }

      return true;
    }
    return this->emitNullMemberPtr(0, nullptr, Info);
  }

  if (Val.isLValue()) {
    if (Val.isNullPointer())
      return this->emitNull(ValType, 0, nullptr, Info);

    APValue::LValueBase Base = Val.getLValueBase();

    if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
      return this->visit(BaseExpr);
    if (const auto *VD = Base.dyn_cast<const ValueDecl *>()) {
      if (!this->visitDeclRef(VD, Info.asExpr()))
        return false;

      QualType EntryType = VD->getType();
      if (Val.hasLValuePath()) {
        ArrayRef<APValue::LValuePathEntry> Path = Val.getLValuePath();
        for (auto &Entry : Path) {
          if (EntryType->isArrayType()) {
            uint64_t Index = Entry.getAsArrayIndex();
            QualType ElemType =
                EntryType->getAsArrayTypeUnsafe()->getElementType();
            if (!this->emitConst(Index, PT_Uint64, Info))
              return false;
            if (!this->emitArrayElemPtrPop(PT_Uint64, Info))
              return false;
            EntryType = ElemType;
          } else {
            assert(EntryType->isRecordType());
            const Record *EntryRecord = getRecord(EntryType);
            if (!EntryRecord)
              return false;

            const Decl *BaseOrMember = Entry.getAsBaseOrMember().getPointer();
            if (const auto *FD = dyn_cast<FieldDecl>(BaseOrMember)) {
              unsigned EntryOffset = EntryRecord->getField(FD)->Offset;
              if (!this->emitGetPtrFieldPop(EntryOffset, Info))
                return false;
              EntryType = FD->getType();
            } else {
              const auto *Base = cast<CXXRecordDecl>(BaseOrMember);
              if (const Record::Base *B = EntryRecord->getBaseOrNull(Base)) {
                if (!this->emitGetPtrBasePop(B->Offset, /*NullOK=*/false, Info))
                  return false;
              } else {
                // Must be a virtual base.
                assert(EntryRecord->findVirtualBase(Base));
                if (!this->emitGetPtrVirtBasePop(Base, Info))
                  return false;
              }
              EntryType = Ctx.getASTContext().getCanonicalTagType(Base);
            }
          }
        }
      }

      return true;
    }
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::visitAPValueInitializer(const APValue &Val,
                                                SourceInfo Info, QualType T,
                                                bool IsCompleteClass) {
  if (Val.isStruct()) {
    const Record *R = this->getRecord(T);
    assert(R);

    assert(R->getNumBases() == Val.getStructNumBases());
    if (IsCompleteClass)
      assert(R->getNumVirtualBases() == Val.getStructNumVirtualBases());

    for (unsigned I = 0, N = Val.getStructNumBases(); I != N; ++I) {
      const APValue &B = Val.getStructBase(I);
      if (B.isIndeterminate())
        continue;
      const Record::Base *RB = R->getBase(I);
      QualType BaseType = Ctx.getASTContext().getCanonicalTagType(RB->Decl);

      if (!this->emitGetPtrBase(RB->Offset, Info))
        return false;
      if (!this->visitAPValueInitializer(B, Info, BaseType,
                                         /*IsCompleteClass=*/false))
        return false;
      if (!this->emitFinishInitPop(Info))
        return false;
    }

    for (unsigned I = 0, N = Val.getStructNumFields(); I != N; ++I) {
      const APValue &F = Val.getStructField(I);
      if (F.isIndeterminate())
        continue;
      const Record::Field *RF = R->getField(I);
      QualType FieldType = RF->Decl->getType();
      // Fields.
      if (OptPrimType PT = RF->T) {
        if (!this->visitAPValue(F, *PT, Info))
          return false;
        if (!this->emitInitField(*PT, RF->Offset, Info))
          return false;
      } else {
        if (!this->emitGetPtrField(RF->Offset, Info))
          return false;
        if (!this->visitAPValueInitializer(F, Info, FieldType))
          return false;
        if (!this->emitFinishInitPop(Info))
          return false;
      }
    }

    // Virtual Bases.
    if (IsCompleteClass) {
      for (unsigned I = 0, N = Val.getStructNumVirtualBases(); I != N; ++I) {
        const APValue &B = Val.getStructVirtualBase(I);
        if (B.isIndeterminate())
          continue;
        const Record::Base *RB = R->getVirtualBase(I);
        QualType BaseType = Ctx.getASTContext().getCanonicalTagType(RB->Decl);

        if (!this->emitGetPtrVirtBase(cast<CXXRecordDecl>(RB->R->getDecl()),
                                      Info))
          return false;
        if (!this->visitAPValueInitializer(B, Info, BaseType,
                                           /*IsCompleteClass=*/false))
          return false;
        if (!this->emitFinishInitPop(Info))
          return false;
      }
    }

    return true;
  }
  if (Val.isUnion()) {
    const FieldDecl *UnionField = Val.getUnionField();
    if (!UnionField)
      return true;
    const Record *R = this->getRecord(T);
    assert(R);
    const APValue &F = Val.getUnionValue();
    if (F.isIndeterminate())
      return true;
    const Record::Field *RF = R->getField(UnionField);
    QualType FieldType = RF->Decl->getType();

    if (OptPrimType PT = RF->T) {
      if (!this->visitAPValue(F, *PT, Info))
        return false;
      if (RF->isBitField())
        return this->emitInitBitFieldActivate(*PT, RF->Offset, RF->bitWidth(),
                                              Info);
      return this->emitInitFieldActivate(*PT, RF->Offset, Info);
    }

    if (!this->emitGetPtrField(RF->Offset, Info))
      return false;
    if (!this->emitActivate(Info))
      return false;
    if (!this->visitAPValueInitializer(F, Info, FieldType))
      return false;
    return this->emitPopPtr(Info);
  }
  if (Val.isArray()) {
    unsigned InitializedElems = Val.getArrayInitializedElts();
    const auto *ArrType = T->getAsArrayTypeUnsafe();
    QualType ElemType = ArrType->getElementType();
    OptPrimType ElemT = classify(ElemType);

    for (unsigned A = 0, AN = Val.getArraySize(); A != AN; ++A) {
      const APValue &Elem = A >= InitializedElems
                                ? Val.getArrayFiller()
                                : Val.getArrayInitializedElt(A);
      if (Elem.isIndeterminate())
        continue;

      if (ElemT) {
        if (!this->visitAPValue(Elem, *ElemT, Info))
          return false;
        if (!this->emitInitElem(*ElemT, A, Info))
          return false;
      } else {
        if (!this->emitConstUint32(A, Info))
          return false;
        if (!this->emitArrayElemPtrUint32(Info))
          return false;
        if (!this->visitAPValueInitializer(Elem, Info, ElemType))
          return false;
        if (!this->emitPopPtr(Info))
          return false;
      }
    }
    return true;
  }
  // TODO: Other types.

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::registerRedecl(const VarDecl *VD, const APValue &Val) {
  if (P.getGlobal(VD))
    return true;

  UnsignedOrNone GlobalIndex = P.createGlobal(VD, /*Init=*/nullptr);
  if (!GlobalIndex) {
    llvm_unreachable("Why didn't that work?");
  }

  assert(canClassify(VD->getType()) &&
         "registerRedecl should only be called with primitive values");

  PrimType T = classifyPrim(VD->getType());
  if (!visitAPValue(Val, T, VD))
    return false;
  return this->emitInitGlobal(T, *GlobalIndex, {});
}

template <class Emitter>
bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
                                             unsigned BuiltinID) {
  if (BuiltinID == Builtin::BI__builtin_constant_p) {
    // Void argument is always invalid and harder to handle later.
    if (E->getArg(0)->getType()->isVoidType()) {
      if (DiscardResult)
        return true;
      return this->emitConst(0, E);
    }

    if (!this->emitStartSpeculation(E))
      return false;
    LabelTy EndLabel = this->getLabel();
    if (!this->speculate(E, EndLabel))
      return false;
    if (!this->emitEndSpeculation(E))
      return false;
    this->fallthrough(EndLabel);
    if (DiscardResult)
      return this->emitPop(classifyPrim(E), E);
    return true;
  }

  // For these, we're expected to ultimately return an APValue pointing
  // to the CallExpr. This is needed to get the correct codegen.
  if (BuiltinID == Builtin::BI__builtin___CFStringMakeConstantString ||
      BuiltinID == Builtin::BI__builtin___NSStringMakeConstantString ||
      BuiltinID == Builtin::BI__builtin_ptrauth_sign_constant ||
      BuiltinID == Builtin::BI__builtin_function_start) {
    if (DiscardResult)
      return true;
    return this->emitDummyPtr(E, E);
  }

  QualType ReturnType = E->getType();
  OptPrimType ReturnT = classify(E);

  // Non-primitive return type. Prepare storage.
  if (!Initializing && !ReturnT && !ReturnType->isVoidType()) {
    UnsignedOrNone LocalIndex = allocateLocal(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  // Prepare function arguments including special cases.
  switch (BuiltinID) {
  case Builtin::BI__builtin_object_size:
  case Builtin::BI__builtin_dynamic_object_size: {
    assert(E->getNumArgs() == 2);
    const Expr *Arg0 = E->getArg(0);
    if (Arg0->isGLValue()) {
      if (!this->visit(Arg0))
        return false;

    } else {
      if (!this->visitAsLValue(ignorePointerCastsAndParens(Arg0)))
        return false;
    }
    if (!this->visit(E->getArg(1)))
      return false;

  } break;
  case Builtin::BI__assume:
  case Builtin::BI__builtin_assume:
    // Argument is not evaluated.
    break;
  case Builtin::BI__atomic_is_lock_free:
  case Builtin::BI__atomic_always_lock_free: {
    assert(E->getNumArgs() == 2);
    if (!this->visit(E->getArg(0)))
      return false;
    if (!this->visitAsLValue(E->getArg(1)))
      return false;
  } break;

  default:
    if (!Context::isUnevaluatedBuiltin(BuiltinID)) {
      // Put arguments on the stack.
      for (const auto *Arg : E->arguments()) {
        if (!this->visit(Arg))
          return false;
      }
    }
  }

  if (!this->emitCallBI(E, BuiltinID, E))
    return false;

  if (DiscardResult && !ReturnType->isVoidType())
    return this->emitPop(ReturnT.value_or(PT_Ptr), E);

  return true;
}

static bool isTrivialMemoryOperation(const CXXMethodDecl *MD) {
  if (!MD || !MD->isDefaulted())
    return false;
  if (!MD->isCopyAssignmentOperator() && !MD->isMoveAssignmentOperator())
    return false;
  return MD->getParent()->isUnion() ||
         (MD->isTrivial() && isReadByLvalueToRvalueConversion(MD->getParent()));
}

template <class Emitter>
bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
  if (E->containsErrors())
    return false;
  const FunctionDecl *FuncDecl = E->getDirectCallee();

  if (FuncDecl) {
    if (unsigned BuiltinID = FuncDecl->getBuiltinID())
      return VisitBuiltinCallExpr(E, BuiltinID);

    // Calls to replaceable operator new/operator delete.
    if (FuncDecl->isUsableAsGlobalAllocationFunctionInConstantEvaluation()) {
      if (FuncDecl->getDeclName().isAnyOperatorNew())
        return VisitBuiltinCallExpr(E, Builtin::BI__builtin_operator_new);
      assert(FuncDecl->getDeclName().getCXXOverloadedOperator() == OO_Delete ||
             FuncDecl->getDeclName().getCXXOverloadedOperator() ==
                 OO_Array_Delete);
      return VisitBuiltinCallExpr(E, Builtin::BI__builtin_operator_delete);
    }

    // Explicit calls to trivial destructors
    if (const auto *DD = dyn_cast<CXXDestructorDecl>(FuncDecl);
        DD && DD->isTrivial()) {
      const auto *MemberCall = cast<CXXMemberCallExpr>(E);
      if (!this->visit(MemberCall->getImplicitObjectArgument()))
        return false;
      return this->emitCheckDestruction(E) && this->emitEndLifetime(E) &&
             this->emitPopPtr(E);
    }
  }

  LocalScope<Emitter> CallScope(this, ScopeKind::Call);
  ArrayRef<const Expr *> Args(E->getArgs(), E->getNumArgs());
  bool ActivateLHS = false;

  // Emit a special op for trivial copy/move operators.
  if (isTrivialMemoryOperation(dyn_cast_if_present<CXXMethodDecl>(FuncDecl))) {
    const Function *Func = getFunction(FuncDecl);
    if (!Func)
      return false;

    if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(E);
        OCE && OCE->isAssignmentOp()) {
      const CXXRecordDecl *LHSRecord = Args[0]->getType()->getAsCXXRecordDecl();
      ActivateLHS = LHSRecord && LHSRecord->hasTrivialDefaultConstructor();
    }
    if (const auto *MCE = dyn_cast<CXXMemberCallExpr>(E))
      if (!this->visit(MCE->getImplicitObjectArgument()))
        return false;

    if (!this->visitCallArgs(Args, FuncDecl, /*ActivateLHS=*/ActivateLHS,
                             isa<CXXOperatorCallExpr>(E)))
      return false;

    if (!this->emitTrivialCopy(ActivateLHS, Func, E))
      return false;

    if (!DiscardResult)
      return CallScope.destroyLocals();
    return this->emitPopPtr(E) && CallScope.destroyLocals();
  }

  QualType ReturnType = E->getCallReturnType(Ctx.getASTContext());
  OptPrimType T = classify(ReturnType);
  bool HasRVO = !ReturnType->isVoidType() && !T;

  if (HasRVO) {
    if (DiscardResult) {
      // If we need to discard the return value but the function returns its
      // value via an RVO pointer, we need to create one such pointer just
      // for this call.
      if (UnsignedOrNone LocalIndex = allocateLocal(E)) {
        if (!this->emitGetPtrLocal(*LocalIndex, E))
          return false;
      }
    } else {
      // We need the result. Prepare a pointer to return or
      // dup the current one.
      if (!Initializing) {
        if (UnsignedOrNone LocalIndex = allocateLocal(E)) {
          if (!this->emitGetPtrLocal(*LocalIndex, E))
            return false;
        }
      }
      if (!this->emitDupPtr(E))
        return false;
    }
  }

  const Expr *ReversedArgs[2];
  bool IsAssignmentOperatorCall = false;
  if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(E);
      OCE && OCE->isAssignmentOp()) {
    // Just like with regular assignments, we need to special-case assignment
    // operators here and evaluate the RHS (the second arg) before the LHS (the
    // first arg). We fix this by using a Flip op later.
    assert(Args.size() == 2);
    const CXXRecordDecl *LHSRecord = Args[0]->getType()->getAsCXXRecordDecl();
    ActivateLHS = LHSRecord && LHSRecord->hasTrivialDefaultConstructor();
    IsAssignmentOperatorCall = true;
    ReversedArgs[0] = Args[1];
    ReversedArgs[1] = Args[0];
    Args = ReversedArgs;
  }

  // Calling a static operator will still
  // pass the instance, but we don't need it.
  // Discard it here.
  if (isa<CXXOperatorCallExpr>(E)) {
    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(FuncDecl);
        MD && MD->isStatic()) {
      if (!this->discard(E->getArg(0)))
        return false;
      // Drop first arg.
      Args = Args.drop_front();
    }
  }

  bool Devirtualized = false;
  UnsignedOrNone CalleeOffset = std::nullopt;
  // Add the (optional, implicit) This pointer.
  if (const auto *MC = dyn_cast<CXXMemberCallExpr>(E)) {
    if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) {
      // If we end up creating a CallPtr op for this, we need the base of the
      // member pointer as the instance pointer, and later extract the function
      // decl as the function pointer.
      const Expr *Callee = E->getCallee();
      CalleeOffset =
          this->allocateLocalPrimitive(Callee, PT_MemberPtr, /*IsConst=*/true);
      if (!this->visit(Callee))
        return false;
      if (!this->emitSetLocal(PT_MemberPtr, *CalleeOffset, E))
        return false;
      if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
        return false;
      if (!this->emitGetMemberPtrBase(E))
        return false;
    } else {
      const auto *InstancePtr = MC->getImplicitObjectArgument();
      if (isa_and_nonnull<CXXDestructorDecl>(CompilingFunction) ||
          isa_and_nonnull<CXXConstructorDecl>(CompilingFunction)) {
        const auto *Stripped = stripCheckedDerivedToBaseCasts(InstancePtr);
        if (isa<CXXThisExpr>(Stripped)) {
          FuncDecl =
              cast<CXXMethodDecl>(FuncDecl)->getCorrespondingMethodInClass(
                  Stripped->getType()->getPointeeType()->getAsCXXRecordDecl());
          Devirtualized = true;
          if (!this->visit(Stripped))
            return false;
        } else {
          if (!this->visit(InstancePtr))
            return false;
        }
      } else {
        if (!this->visit(InstancePtr))
          return false;
      }
    }
  } else if (const auto *PD =
                 dyn_cast<CXXPseudoDestructorExpr>(E->getCallee())) {
    if (!this->emitCheckPseudoDtor(E))
      return false;
    const Expr *Base = PD->getBase();
    // E.g. `using T = int; 0.~T();`.
    if (OptPrimType BaseT = classify(Base); !BaseT || BaseT != PT_Ptr)
      return this->discard(Base);
    if (!this->visit(Base))
      return false;
    return this->emitPseudoDtor(E);
  } else if (!FuncDecl) {
    const Expr *Callee = E->getCallee();
    CalleeOffset =
        this->allocateLocalPrimitive(Callee, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(Callee))
      return false;
    if (!this->emitSetLocal(PT_Ptr, *CalleeOffset, E))
      return false;
  }

  if (!this->visitCallArgs(Args, FuncDecl, ActivateLHS,
                           isa<CXXOperatorCallExpr>(E)))
    return false;

  // Undo the argument reversal we did earlier.
  if (IsAssignmentOperatorCall) {
    assert(Args.size() == 2);
    PrimType Arg1T = classify(Args[0]).value_or(PT_Ptr);
    PrimType Arg2T = classify(Args[1]).value_or(PT_Ptr);
    if (!this->emitFlip(Arg2T, Arg1T, E))
      return false;
  }

  if (FuncDecl) {
    const Function *Func = getFunction(FuncDecl);
    if (!Func)
      return false;

    // In error cases, the function may be called with fewer arguments than
    // parameters.
    if (E->getNumArgs() < Func->getNumWrittenParams())
      return false;

    assert(HasRVO == Func->hasRVO());

    bool HasQualifier = false;
    if (const auto *ME = dyn_cast<MemberExpr>(E->getCallee()))
      HasQualifier = ME->hasQualifier();

    bool IsVirtual = false;
    if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl))
      IsVirtual = !Devirtualized && MD->isVirtual();

    // In any case call the function. The return value will end up on the stack
    // and if the function has RVO, we already have the pointer on the stack to
    // write the result into.
    if (IsVirtual && !HasQualifier) {
      uint32_t VarArgSize = 0;
      unsigned NumParams =
          Func->getNumWrittenParams() +
          (isa<CXXOperatorCallExpr>(E) && Func->hasImplicitThisPointer());
      for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
        VarArgSize += align(primSize(classify(E->getArg(I)).value_or(PT_Ptr)));

      if (!this->emitCallVirt(Func, VarArgSize, E))
        return false;
    } else if (Func->isVariadic()) {
      uint32_t VarArgSize = 0;
      unsigned NumParams =
          Func->getNumWrittenParams() +
          (isa<CXXOperatorCallExpr>(E) && Func->hasImplicitThisPointer());
      for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
        VarArgSize += align(primSize(classify(E->getArg(I)).value_or(PT_Ptr)));
      if (!this->emitCallVar(Func, VarArgSize, E))
        return false;
    } else {
      if (!this->emitCall(Func, 0, E))
        return false;
    }
  } else {
    // Indirect call. Visit the callee, which will leave a FunctionPointer on
    // the stack. Cleanup of the returned value if necessary will be done after
    // the function call completed.

    // Sum the size of all args from the call expr.
    uint32_t ArgSize = 0;
    for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
      ArgSize += align(primSize(classify(E->getArg(I)).value_or(PT_Ptr)));

    // Get the callee, either from a member pointer or function pointer saved in
    // CalleeOffset.
    if (isa<CXXMemberCallExpr>(E) && CalleeOffset) {
      if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
        return false;
      if (!this->emitGetMemberPtrDecl(E))
        return false;
    } else {
      if (!this->emitGetLocal(PT_Ptr, *CalleeOffset, E))
        return false;
    }
    if (!this->emitCallPtr(ArgSize, E, E))
      return false;
  }

  // Cleanup for discarded return values.
  if (DiscardResult && !ReturnType->isVoidType() && T)
    return this->emitPop(*T, E) && CallScope.destroyLocals();

  return CallScope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *E) {
  SourceLocScope<Emitter> SLS(this, E);

  return this->delegate(E->getExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) {
  SourceLocScope<Emitter> SLS(this, E);

  return this->delegate(E->getExpr());
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *E) {
  if (DiscardResult)
    return true;

  return this->emitConstBool(E->getValue(), E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXNullPtrLiteralExpr(
    const CXXNullPtrLiteralExpr *E) {
  if (DiscardResult)
    return true;

  uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(E->getType());
  return this->emitNullPtr(Val, nullptr, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitGNUNullExpr(const GNUNullExpr *E) {
  if (DiscardResult)
    return true;

  assert(E->getType()->isIntegerType());

  PrimType T = classifyPrim(E->getType());
  return this->emitZero(T, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
  if (DiscardResult)
    return true;

  if constexpr (!std::is_same_v<Emitter, EvalEmitter>) {
    if (this->LambdaThisCapture.Offset > 0) {
      if (this->LambdaThisCapture.IsPtr)
        return this->emitGetThisFieldPtr(this->LambdaThisCapture.Offset, E);
      return this->emitGetPtrThisField(this->LambdaThisCapture.Offset, E);
    }
  }

  // In some circumstances, the 'this' pointer does not actually refer to the
  // instance pointer of the current function frame, but e.g. to the declaration
  // currently being initialized. Here we emit the necessary instruction(s) for
  // this scenario.
  if (!InitStackActive || InitStack.empty())
    return this->emitThis(E);

  // If our init stack is, for example:
  // 0 Stack: 3 (decl)
  // 1 Stack: 6 (init list)
  // 2 Stack: 1 (field)
  // 3 Stack: 6 (init list)
  // 4 Stack: 1 (field)
  //
  // We want to find the LAST element in it that's an init list,
  // which is marked with the K_InitList marker. The index right
  // before that points to an init list. We need to find the
  // elements before the K_InitList element that point to a base
  // (e.g. a decl or This), optionally followed by field, elem, etc.
  // In the example above, we want to emit elements [0..2].
  unsigned StartIndex = 0;
  unsigned EndIndex = 0;
  // Find the init list.
  for (StartIndex = InitStack.size() - 1; StartIndex > 0; --StartIndex) {
    if (InitStack[StartIndex].Kind == InitLink::K_DIE) {
      EndIndex = StartIndex;
      --StartIndex;
      break;
    }
  }

  // Walk backwards to find the base.
  for (; StartIndex > 0; --StartIndex) {
    if (InitStack[StartIndex].Kind == InitLink::K_InitList)
      continue;

    if (InitStack[StartIndex].Kind != InitLink::K_Field &&
        InitStack[StartIndex].Kind != InitLink::K_Elem &&
        InitStack[StartIndex].Kind != InitLink::K_Base &&
        InitStack[StartIndex].Kind != InitLink::K_DIE)
      break;
  }

  if (StartIndex == 0 && EndIndex == 0)
    EndIndex = InitStack.size() - 1;

  assert(InitStack[StartIndex].Kind == InitLink::K_Decl ||
         InitStack[StartIndex].Kind == InitLink::K_This ||
         InitStack[StartIndex].Kind == InitLink::K_Temp ||
         InitStack[StartIndex].Kind == InitLink::K_RVO);

  // NOTE: This could be StartIndex < EndIndex, but we're also abusing the
  // InitStack mechanism in visitWithSubstitutions to have the This pointer
  // _just_ be a local variable.
  assert(StartIndex <= EndIndex);

  // Emit the instructions.
  for (unsigned I = StartIndex; I != (EndIndex + 1); ++I) {
    if (InitStack[I].Kind == InitLink::K_InitList ||
        InitStack[I].Kind == InitLink::K_DIE)
      continue;
    if (!InitStack[I].template emit<Emitter>(this, E))
      return false;
  }
  return true;
}

template <class Emitter> bool Compiler<Emitter>::visitStmt(const Stmt *S) {
  switch (S->getStmtClass()) {
  case Stmt::CompoundStmtClass:
    return visitCompoundStmt(cast<CompoundStmt>(S));
  case Stmt::DeclStmtClass:
    return visitDeclStmt(cast<DeclStmt>(S), /*EvaluateConditionDecl=*/true);
  case Stmt::ReturnStmtClass:
    return visitReturnStmt(cast<ReturnStmt>(S));
  case Stmt::IfStmtClass:
    return visitIfStmt(cast<IfStmt>(S));
  case Stmt::WhileStmtClass:
    return visitWhileStmt(cast<WhileStmt>(S));
  case Stmt::DoStmtClass:
    return visitDoStmt(cast<DoStmt>(S));
  case Stmt::ForStmtClass:
    return visitForStmt(cast<ForStmt>(S));
  case Stmt::CXXForRangeStmtClass:
    return visitCXXForRangeStmt(cast<CXXForRangeStmt>(S));
  case Stmt::BreakStmtClass:
    return visitBreakStmt(cast<BreakStmt>(S));
  case Stmt::ContinueStmtClass:
    return visitContinueStmt(cast<ContinueStmt>(S));
  case Stmt::SwitchStmtClass:
    return visitSwitchStmt(cast<SwitchStmt>(S));
  case Stmt::CaseStmtClass:
    return visitCaseStmt(cast<CaseStmt>(S));
  case Stmt::DefaultStmtClass:
    return visitDefaultStmt(cast<DefaultStmt>(S));
  case Stmt::AttributedStmtClass:
    return visitAttributedStmt(cast<AttributedStmt>(S));
  case Stmt::CXXTryStmtClass:
    return visitCXXTryStmt(cast<CXXTryStmt>(S));
  case Stmt::NullStmtClass:
    return true;
  // Always invalid statements.
  case Stmt::GCCAsmStmtClass:
  case Stmt::MSAsmStmtClass:
  case Stmt::GotoStmtClass:
    return this->emitInvalid(S);
  case Stmt::LabelStmtClass:
    return this->visitStmt(cast<LabelStmt>(S)->getSubStmt());
  case Stmt::CXXExpansionStmtInstantiationClass:
    return this->visitCXXExpansionStmtInstantiation(
        cast<CXXExpansionStmtInstantiation>(S));
  default: {
    if (const auto *E = dyn_cast<Expr>(S))
      return this->discard(E);
    return false;
  }
  }
}

template <class Emitter>
bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
  LocalScope<Emitter> Scope(this);
  for (const auto *InnerStmt : S->body())
    if (!visitStmt(InnerStmt))
      return false;
  return Scope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
  if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
    for (auto *BD : DD->flat_bindings())
      if (auto *KD = BD->getHoldingVar();
          KD && !this->visitVarDecl(KD, KD->getInit()))
        return false;
  }
  return true;
}

static bool hasTrivialDefaultCtorParent(const FieldDecl *FD) {
  assert(FD);
  assert(FD->getParent()->isUnion());
  const CXXRecordDecl *CXXRD =
      FD->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
  return !CXXRD || CXXRD->hasTrivialDefaultConstructor();
}

template <class Emitter> bool Compiler<Emitter>::refersToUnion(const Expr *E) {
  for (;;) {
    if (const auto *ME = dyn_cast<MemberExpr>(E)) {
      if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
          FD && FD->getParent()->isUnion() && hasTrivialDefaultCtorParent(FD))
        return true;
      E = ME->getBase();
      continue;
    }

    if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E)) {
      E = ASE->getBase()->IgnoreImplicit();
      continue;
    }

    if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E);
        ICE && (ICE->getCastKind() == CK_NoOp ||
                ICE->getCastKind() == CK_DerivedToBase ||
                ICE->getCastKind() == CK_UncheckedDerivedToBase)) {
      E = ICE->getSubExpr();
      continue;
    }

    if (const auto *This = dyn_cast<CXXThisExpr>(E)) {
      const auto *ThisRecord =
          This->getType()->getPointeeType()->getAsRecordDecl();
      if (!ThisRecord->isUnion())
        return false;
      // Otherwise, always activate if we're in the ctor.
      if (const auto *Ctor =
              dyn_cast_if_present<CXXConstructorDecl>(CompilingFunction))
        return Ctor->getParent() == ThisRecord;
      return false;
    }

    break;
  }
  return false;
}

template <class Emitter>
bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS,
                                      bool EvaluateConditionDecl) {
  for (const auto *D : DS->decls()) {
    if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, BaseUsingDecl,
            FunctionDecl, NamespaceAliasDecl, UsingDirectiveDecl>(D))
      continue;

    if (const auto *ESD = dyn_cast<CXXExpansionStmtDecl>(D)) {
      assert(ESD->getInstantiations() && "not expanded?");
      if (!this->visitStmt(ESD->getInstantiations()))
        return false;
      continue;
    }

    const auto *VD = dyn_cast<VarDecl>(D);
    if (!VD)
      return false;
    if (!this->visitVarDecl(VD, VD->getInit()))
      return false;

    // Register decomposition decl holding vars.
    if (EvaluateConditionDecl && !this->maybeEmitDeferredVarInit(VD))
      return false;
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::visitReturnStmt(const ReturnStmt *RS) {
  if (this->InStmtExpr)
    return this->emitUnsupported(RS);

  if (const Expr *RE = RS->getRetValue()) {
    LocalScope<Emitter> RetScope(this);
    if (ReturnType) {
      // Primitive types are simply returned.
      if (!this->visit(RE))
        return false;
      this->emitCleanup();
      return this->emitRet(*ReturnType, RS);
    }

    if (RE->getType()->isVoidType()) {
      if (!this->visit(RE))
        return false;
    } else {
      if (RE->containsErrors())
        return false;

      InitLinkScope<Emitter> ILS(this, InitLink::RVO());
      // RVO - construct the value in the return location.
      if (!this->emitRVOPtr(RE))
        return false;
      if (!this->visitInitializerPop(RE))
        return false;

      this->emitCleanup();
      return this->emitRetVoid(RS);
    }
  }

  // Void return.
  this->emitCleanup();
  return this->emitRetVoid(RS);
}

template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
  LocalScope<Emitter> IfScope(this);

  auto visitChildStmt = [&](const Stmt *S) -> bool {
    LocalScope<Emitter> SScope(this);
    if (!visitStmt(S))
      return false;
    return SScope.destroyLocals();
  };

  if (auto *CondInit = IS->getInit()) {
    if (!visitStmt(CondInit))
      return false;
  }

  if (const DeclStmt *CondDecl = IS->getConditionVariableDeclStmt()) {
    if (!visitDeclStmt(CondDecl))
      return false;
  }

  // Save ourselves compiling some code and the jumps, etc. if the condition is
  // stataically known to be either true or false. We could look at more cases
  // here, but I think all the ones that actually happen are using a
  // ConstantExpr.
  if (std::optional<bool> BoolValue = getBoolValue(IS->getCond())) {
    if (*BoolValue)
      return visitChildStmt(IS->getThen());
    if (const Stmt *Else = IS->getElse())
      return visitChildStmt(Else);
    return true;
  }

  // Otherwise, compile the condition.
  if (IS->isNonNegatedConsteval()) {
    if (!this->emitIsConstantContext(IS))
      return false;
  } else if (IS->isNegatedConsteval()) {
    if (!this->emitIsConstantContext(IS))
      return false;
    if (!this->emitInv(IS))
      return false;
  } else {
    LocalScope<Emitter> CondScope(this, ScopeKind::FullExpression);
    if (!this->visitBool(IS->getCond()))
      return false;
    if (!CondScope.destroyLocals())
      return false;
  }

  if (!this->maybeEmitDeferredVarInit(IS->getConditionVariable()))
    return false;

  if (const Stmt *Else = IS->getElse()) {
    LabelTy LabelElse = this->getLabel();
    LabelTy LabelEnd = this->getLabel();
    if (!this->jumpFalse(LabelElse, IS))
      return false;
    if (!visitChildStmt(IS->getThen()))
      return false;
    if (!this->jump(LabelEnd, IS))
      return false;
    this->emitLabel(LabelElse);
    if (!visitChildStmt(Else))
      return false;
    this->emitLabel(LabelEnd);
  } else {
    LabelTy LabelEnd = this->getLabel();
    if (!this->jumpFalse(LabelEnd, IS))
      return false;
    if (!visitChildStmt(IS->getThen()))
      return false;
    this->emitLabel(LabelEnd);
  }

  if (!IfScope.destroyLocals())
    return false;

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::visitWhileStmt(const WhileStmt *S) {
  const Expr *Cond = S->getCond();
  const Stmt *Body = S->getBody();

  LabelTy CondLabel = this->getLabel(); // Label before the condition.
  LabelTy EndLabel = this->getLabel();  // Label after the loop.
  LocalScope<Emitter> WholeLoopScope(this);
  LoopScope<Emitter> LS(this, S, EndLabel, CondLabel);

  this->fallthrough(CondLabel);
  this->emitLabel(CondLabel);

  // Start of the loop body {
  LocalScope<Emitter> CondScope(this);

  if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt()) {
    if (!visitDeclStmt(CondDecl))
      return false;
  }

  if (!this->visitBool(Cond))
    return false;

  if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
    return false;

  if (!this->jumpFalse(EndLabel, S))
    return false;

  if (!this->visitStmt(Body))
    return false;

  if (!CondScope.destroyLocals())
    return false;
  // } End of loop body.

  if (!this->jump(CondLabel, S))
    return false;
  this->fallthrough(EndLabel);
  this->emitLabel(EndLabel);

  return CondScope.destroyLocals() && WholeLoopScope.destroyLocals();
}

template <class Emitter> bool Compiler<Emitter>::visitDoStmt(const DoStmt *S) {
  const Expr *Cond = S->getCond();
  const Stmt *Body = S->getBody();

  LabelTy StartLabel = this->getLabel();
  LabelTy EndLabel = this->getLabel();
  LabelTy CondLabel = this->getLabel();
  LocalScope<Emitter> WholeLoopScope(this);
  LoopScope<Emitter> LS(this, S, EndLabel, CondLabel);

  this->fallthrough(StartLabel);
  this->emitLabel(StartLabel);

  {
    LocalScope<Emitter> CondScope(this);
    if (!this->visitStmt(Body))
      return false;
    this->fallthrough(CondLabel);
    this->emitLabel(CondLabel);
    if (!this->visitBool(Cond))
      return false;

    if (!CondScope.destroyLocals())
      return false;
  }
  if (!this->jumpTrue(StartLabel, S))
    return false;

  this->fallthrough(EndLabel);
  this->emitLabel(EndLabel);
  return WholeLoopScope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::visitForStmt(const ForStmt *S) {
  // for (Init; Cond; Inc) { Body }
  const Stmt *Init = S->getInit();
  const Expr *Cond = S->getCond();
  const Expr *Inc = S->getInc();
  const Stmt *Body = S->getBody();

  LabelTy EndLabel = this->getLabel();
  LabelTy CondLabel = this->getLabel();
  LabelTy IncLabel = this->getLabel();

  LocalScope<Emitter> WholeLoopScope(this);
  if (Init && !this->visitStmt(Init))
    return false;

  // Start of the loop body {
  this->fallthrough(CondLabel);
  this->emitLabel(CondLabel);

  LocalScope<Emitter> CondScope(this);
  LoopScope<Emitter> LS(this, S, EndLabel, IncLabel);
  if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt()) {
    if (!visitDeclStmt(CondDecl))
      return false;
  }

  if (Cond) {
    if (!this->visitBool(Cond))
      return false;
    if (!this->jumpFalse(EndLabel, S))
      return false;
  }
  if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
    return false;

  if (Body && !this->visitStmt(Body))
    return false;

  this->fallthrough(IncLabel);
  this->emitLabel(IncLabel);
  if (Inc && !this->discard(Inc))
    return false;

  if (!CondScope.destroyLocals())
    return false;
  if (!this->jump(CondLabel, S))
    return false;
  // } End of loop body.

  this->emitLabel(EndLabel);
  // If we jumped out of the loop above, we still need to clean up the condition
  // scope.
  return CondScope.destroyLocals() && WholeLoopScope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
  const Stmt *Init = S->getInit();
  const Expr *Cond = S->getCond();
  const Expr *Inc = S->getInc();
  const Stmt *Body = S->getBody();
  const Stmt *BeginStmt = S->getBeginStmt();
  const Stmt *RangeStmt = S->getRangeStmt();
  const Stmt *EndStmt = S->getEndStmt();

  LabelTy EndLabel = this->getLabel();
  LabelTy CondLabel = this->getLabel();
  LabelTy IncLabel = this->getLabel();
  LocalScope<Emitter> WholeLoopScope(this);
  LoopScope<Emitter> LS(this, S, EndLabel, IncLabel);

  // Emit declarations needed in the loop.
  if (Init && !this->visitStmt(Init))
    return false;
  if (!this->visitStmt(RangeStmt))
    return false;
  if (!this->visitStmt(BeginStmt))
    return false;
  if (!this->visitStmt(EndStmt))
    return false;

  LocalScope<Emitter> CondScope(this);
  // Now the condition as well as the loop variable assignment.
  this->fallthrough(CondLabel);
  this->emitLabel(CondLabel);
  if (!this->visitBool(Cond))
    return false;
  if (!this->jumpFalse(EndLabel, S))
    return false;

  if (!this->visitDeclStmt(S->getLoopVarStmt(), /*EvaluateConditionDecl=*/true))
    return false;

  // Body.
  {
    if (!this->visitStmt(Body))
      return false;

    this->fallthrough(IncLabel);
    this->emitLabel(IncLabel);
    if (!this->discard(Inc))
      return false;
  }

  if (!CondScope.destroyLocals())
    return false;
  if (!this->jump(CondLabel, S))
    return false;

  this->fallthrough(EndLabel);
  this->emitLabel(EndLabel);
  return WholeLoopScope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::visitBreakStmt(const BreakStmt *S) {
  if (LabelInfoStack.empty())
    return false;

  OptLabelTy TargetLabel = std::nullopt;
  const Stmt *TargetLoop = S->getNamedLoopOrSwitch();
  const VariableScope<Emitter> *BreakScope = nullptr;

  if (!TargetLoop) {
    for (const auto &LI : llvm::reverse(LabelInfoStack)) {
      if (LI.BreakLabel) {
        TargetLabel = *LI.BreakLabel;
        BreakScope = LI.BreakOrContinueScope;
        break;
      }
    }
  } else {
    for (const auto &LI : LabelInfoStack) {
      if (LI.Name == TargetLoop) {
        TargetLabel = *LI.BreakLabel;
        BreakScope = LI.BreakOrContinueScope;
        break;
      }
    }
  }

  // Faulty break statement (e.g. label redefined or named loops disabled).
  if (!TargetLabel)
    return false;

  for (VariableScope<Emitter> *C = this->VarScope; C != BreakScope;
       C = C->getParent()) {
    if (!C->destroyLocals())
      return false;
  }

  return this->jump(*TargetLabel, S);
}

template <class Emitter>
bool Compiler<Emitter>::visitContinueStmt(const ContinueStmt *S) {
  if (LabelInfoStack.empty())
    return false;

  OptLabelTy TargetLabel = std::nullopt;
  const Stmt *TargetLoop = S->getNamedLoopOrSwitch();
  const VariableScope<Emitter> *ContinueScope = nullptr;

  if (!TargetLoop) {
    for (const auto &LI : llvm::reverse(LabelInfoStack)) {
      if (LI.ContinueLabel) {
        TargetLabel = *LI.ContinueLabel;
        ContinueScope = LI.BreakOrContinueScope;
        break;
      }
    }
  } else {
    for (auto LI : LabelInfoStack) {
      if (LI.Name == TargetLoop) {
        TargetLabel = *LI.ContinueLabel;
        ContinueScope = LI.BreakOrContinueScope;
        break;
      }
    }
  }

  if (!TargetLabel)
    return false;

  for (VariableScope<Emitter> *C = VarScope; C != ContinueScope;
       C = C->getParent()) {
    if (!C->destroyLocals())
      return false;
  }

  return this->jump(*TargetLabel, S);
}

template <class Emitter>
bool Compiler<Emitter>::visitSwitchStmt(const SwitchStmt *S) {
  const Expr *Cond = S->getCond();
  if (Cond->containsErrors())
    return false;

  PrimType CondT = this->classifyPrim(Cond->getType());
  LocalScope<Emitter> LS(this);
  llvm::SaveAndRestore StmtExprSAR(this->SwitchInStmtExpr, this->InStmtExpr);

  LabelTy EndLabel = this->getLabel();
  UnsignedOrNone DefaultLabel = std::nullopt;
  unsigned CondVar =
      this->allocateLocalPrimitive(Cond, CondT, /*IsConst=*/true);

  if (const auto *CondInit = S->getInit())
    if (!visitStmt(CondInit))
      return false;

  if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
    if (!visitDeclStmt(CondDecl))
      return false;

  // Initialize condition variable.
  if (!this->visit(Cond))
    return false;
  if (!this->emitSetLocal(CondT, CondVar, S))
    return false;

  if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
    return false;

  CaseMap CaseLabels;
  // Create labels and comparison ops for all case statements.
  for (const SwitchCase *SC = S->getSwitchCaseList(); SC;
       SC = SC->getNextSwitchCase()) {
    if (const auto *CS = dyn_cast<CaseStmt>(SC)) {
      CaseLabels[SC] = this->getLabel();

      if (CS->caseStmtIsGNURange()) {
        LabelTy EndOfRangeCheck = this->getLabel();
        const Expr *Low = CS->getLHS();
        const Expr *High = CS->getRHS();
        if (Low->isValueDependent() || High->isValueDependent())
          return false;

        if (!this->emitGetLocal(CondT, CondVar, CS))
          return false;
        if (!this->visit(Low))
          return false;
        PrimType LT = this->classifyPrim(Low->getType());
        if (!this->emitGE(LT, S))
          return false;
        if (!this->jumpFalse(EndOfRangeCheck, S))
          return false;

        if (!this->emitGetLocal(CondT, CondVar, CS))
          return false;
        if (!this->visit(High))
          return false;
        PrimType HT = this->classifyPrim(High->getType());
        if (!this->emitLE(HT, S))
          return false;
        if (!this->jumpTrue(CaseLabels[CS], S))
          return false;
        this->emitLabel(EndOfRangeCheck);
        continue;
      }

      const Expr *Value = CS->getLHS();
      if (Value->isValueDependent())
        return false;
      PrimType ValueT = this->classifyPrim(Value->getType());

      // Compare the case statement's value to the switch condition.
      if (!this->emitGetLocal(CondT, CondVar, CS))
        return false;
      if (!this->visit(Value))
        return false;

      // Compare and jump to the case label.
      if (!this->emitEQ(ValueT, S))
        return false;
      if (!this->jumpTrue(CaseLabels[CS], S))
        return false;
    } else {
      assert(!DefaultLabel);
      DefaultLabel = this->getLabel();
    }
  }

  // If none of the conditions above were true, fall through to the default
  // statement or jump after the switch statement.
  if (DefaultLabel) {
    if (!this->jump(*DefaultLabel, S))
      return false;
  } else {
    if (!this->jump(EndLabel, S))
      return false;
  }

  SwitchScope<Emitter> SS(this, S, std::move(CaseLabels), EndLabel,
                          DefaultLabel);
  if (!this->visitStmt(S->getBody()))
    return false;
  this->fallthrough(EndLabel);
  this->emitLabel(EndLabel);

  return LS.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::visitCaseStmt(const CaseStmt *S) {
  this->fallthrough(CaseLabels[S]);
  this->emitLabel(CaseLabels[S]);

  // We can't jump from an outer switch statement to a case label
  // that's inside a StmtExpr.
  if (this->InStmtExpr && !this->SwitchInStmtExpr)
    return this->emitUnsupported(S);

  return this->visitStmt(S->getSubStmt());
}

template <class Emitter>
bool Compiler<Emitter>::visitDefaultStmt(const DefaultStmt *S) {
  if (LabelInfoStack.empty())
    return false;

  LabelTy DefaultLabel;
  for (const LabelInfo &LI : llvm::reverse(LabelInfoStack)) {
    if (LI.DefaultLabel) {
      DefaultLabel = *LI.DefaultLabel;
      break;
    }
  }

  this->emitLabel(DefaultLabel);
  return this->visitStmt(S->getSubStmt());
}

template <class Emitter>
bool Compiler<Emitter>::visitAttributedStmt(const AttributedStmt *S) {
  const Stmt *SubStmt = S->getSubStmt();

  bool IsMSVCConstexprAttr = isa<ReturnStmt>(SubStmt) &&
                             hasSpecificAttr<MSConstexprAttr>(S->getAttrs());

  if (IsMSVCConstexprAttr && !this->emitPushMSVCCE(S))
    return false;

  if (this->Ctx.getLangOpts().CXXAssumptions &&
      !this->Ctx.getLangOpts().MSVCCompat) {
    for (const Attr *A : S->getAttrs()) {
      auto *AA = dyn_cast<CXXAssumeAttr>(A);
      if (!AA)
        continue;

      assert(isa<NullStmt>(SubStmt));

      const Expr *Assumption = AA->getAssumption();
      if (Assumption->isValueDependent())
        return false;

      if (Assumption->HasSideEffects(this->Ctx.getASTContext()))
        continue;

      // Evaluate assumption.
      if (!this->visitBool(Assumption))
        return false;

      if (!this->emitAssume(Assumption))
        return false;
    }
  }

  // Ignore other attributes.
  if (!this->visitStmt(SubStmt))
    return false;

  if (IsMSVCConstexprAttr)
    return this->emitPopMSVCCE(S);
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::visitCXXTryStmt(const CXXTryStmt *S) {
  // Ignore all handlers.
  return this->visitStmt(S->getTryBlock());
}

/// template for (auto x : {1, 2}) {}
///
/// This is not a loop from an AST perspective at all since it has already
/// been instantiated to a list of compound statements.
///
/// Since we can have control flow in those compound statements, we need to
/// handle it mostly like a loop though.
template <class Emitter>
bool Compiler<Emitter>::visitCXXExpansionStmtInstantiation(
    const CXXExpansionStmtInstantiation *S) {
  LocalScope<Emitter> WholeLoopScope(this, ScopeKind::Block);

  for (const Stmt *PreambleStmt : S->getPreambleStmts()) {
    if (!this->visitDeclStmt(cast<DeclStmt>(PreambleStmt), true))
      return false;
  }

  LabelTy EndLabel = this->getLabel();
  for (const Stmt *Instantiation : S->getInstantiations()) {
    LabelTy ContinueLabel = this->getLabel();
    LoopScope<Emitter> LS(this, S, EndLabel, ContinueLabel);

    if (!this->visitStmt(Instantiation))
      return false;
    this->emitLabel(ContinueLabel);
  }

  this->emitLabel(EndLabel);

  return WholeLoopScope.destroyLocals();
}

template <class Emitter>
bool Compiler<Emitter>::emitLambdaStaticInvokerBody(const CXXMethodDecl *MD) {
  assert(MD->isLambdaStaticInvoker());
  assert(MD->hasBody());
  assert(cast<CompoundStmt>(MD->getBody())->body_empty());

  const CXXRecordDecl *ClosureClass = MD->getParent();
  const FunctionDecl *LambdaCallOp;
  assert(ClosureClass->captures().empty());
  if (ClosureClass->isGenericLambda()) {
    LambdaCallOp = ClosureClass->getLambdaCallOperator();
    assert(MD->isFunctionTemplateSpecialization() &&
           "A generic lambda's static-invoker function must be a "
           "template specialization");
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
    FunctionTemplateDecl *CallOpTemplate =
        LambdaCallOp->getDescribedFunctionTemplate();
    llvm::FoldingSetInsertToken InsertToken;
    const FunctionDecl *CorrespondingCallOpSpecialization =
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertToken);
    assert(CorrespondingCallOpSpecialization);
    LambdaCallOp = CorrespondingCallOpSpecialization;
  } else {
    LambdaCallOp = ClosureClass->getLambdaCallOperator();
  }
  assert(ClosureClass->captures().empty());
  const Function *Func = this->getFunction(LambdaCallOp);
  if (!Func)
    return false;
  assert(Func->hasThisPointer());
  assert(Func->getNumParams() == (MD->getNumParams() + 1 + Func->hasRVO()));

  if (Func->hasRVO()) {
    if (!this->emitRVOPtr(MD))
      return false;
  }

  // The lambda call operator needs an instance pointer, but we don't have
  // one here, and we don't need one either because the lambda cannot have
  // any captures, as verified above. Emit a null pointer. This is then
  // special-cased when interpreting to not emit any misleading diagnostics.
  if (!this->emitNullPtr(0, nullptr, MD))
    return false;

  // Forward all arguments from the static invoker to the lambda call operator.
  for (const ParmVarDecl *PVD : MD->parameters()) {
    auto It = this->Params.find(PVD);
    assert(It != this->Params.end());

    // We do the lvalue-to-rvalue conversion manually here, so no need
    // to care about references.
    PrimType ParamType = this->classify(PVD->getType()).value_or(PT_Ptr);
    if (!this->emitGetParam(ParamType, It->second.Index, MD))
      return false;
  }

  if (!this->emitCall(Func, 0, LambdaCallOp))
    return false;

  this->emitCleanup();
  if (ReturnType)
    return this->emitRet(*ReturnType, MD);

  // Nothing to do, since we emitted the RVO pointer above.
  return this->emitRetVoid(MD);
}

template <class Emitter>
bool Compiler<Emitter>::checkLiteralType(const Expr *E) {
  if (Ctx.getLangOpts().CPlusPlus23)
    return true;

  if (!E->isPRValue() || E->getType()->isLiteralType(Ctx.getASTContext()))
    return true;

  return this->emitCheckLiteralType(E->getType().getTypePtr(), E);
}

static bool initNeedsOverridenLoc(const CXXCtorInitializer *Init) {
  const Expr *InitExpr = Init->getInit();

  if (!Init->isWritten() && !Init->isInClassMemberInitializer() &&
      !isa<CXXConstructExpr>(InitExpr))
    return true;

  if (const auto *CE = dyn_cast<CXXConstructExpr>(InitExpr)) {
    const CXXConstructorDecl *Ctor = CE->getConstructor();
    if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
        Ctor->isTrivial())
      return true;
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
  assert(!ReturnType);

  // Only start the lifetime of the instance pointer.
  if (!this->emitStartThisLifetime1(Ctor))
    return false;

  auto emitFieldInitializer = [&](const Record::Field *F, unsigned FieldOffset,
                                  const Expr *InitExpr,
                                  bool Activate = false) -> bool {
    // We don't know what to do with these, so just return false.
    if (InitExpr->getType().isNull())
      return false;

    if (OptPrimType T = this->classify(InitExpr)) {
      if (Activate && !this->emitActivateThisField(FieldOffset, InitExpr))
        return false;

      if (!this->visit(InitExpr))
        return false;

      if (F->isBitField())
        return this->emitInitThisBitField(*T, FieldOffset, F->bitWidth(),
                                          InitExpr);
      return this->emitInitThisField(*T, FieldOffset, InitExpr);
    }
    // Non-primitive case. Get a pointer to the field-to-initialize
    // on the stack and call visitInitialzer() for it.
    InitLinkScope<Emitter> FieldScope(this, InitLink::Field(F->Offset));
    if (!this->emitGetPtrThisField(FieldOffset, InitExpr))
      return false;

    if (Activate && !this->emitActivate(InitExpr))
      return false;

    return this->visitInitializerPop(InitExpr);
  };

  const RecordDecl *RD = Ctor->getParent();
  const Record *R = this->getRecord(RD);
  if (!R)
    return false;
  bool IsUnion = R->isUnion();

  // Default union copy and move ctors are special.
  if (IsUnion && Ctor->isCopyOrMoveConstructor() && Ctor->isDefaulted()) {
    LocOverrideScope<Emitter> LOS(this, SourceInfo{});

    // No special case for NumFields == 0 here, so the Memcpy op
    // below also does its checks in those cases.

    assert(cast<CompoundStmt>(Ctor->getBody())->body_empty());
    if (!this->emitThis(Ctor))
      return false;

    if (!this->emitGetParam(PT_Ptr, /*ParamIndex=*/0, Ctor))
      return false;

    return this->emitMemcpy(Ctor) && this->emitPopPtr(Ctor) &&
           this->emitRetVoid(Ctor);
  }

  unsigned FieldInits = 0;
  InitLinkScope<Emitter> InitScope(this, InitLink::This());
  // First, initialize virtual bases if the records has them.
  if (R->getNumVirtualBases() > 0) {
    if (!this->emitThis(Ctor))
      return false;
    LabelTy AfterVirtBasesLabel = this->getLabel();

    // If the instance pointer is a base class, skip the virtual bases.
    if (!this->emitIsBaseClass({}))
      return false;
    if (!this->jumpTrue(AfterVirtBasesLabel, {}))
      return false;

    for (const auto *Init : Ctor->inits()) {
      if (const Type *Base = Init->getBaseClass();
          Base && Init->isBaseVirtual()) {
        const auto *BaseDecl = Base->getAsCXXRecordDecl();
        assert(BaseDecl);
        assert(R->findVirtualBase(BaseDecl));
        if (!this->emitGetPtrThisVirtBase(BaseDecl, Ctor))
          return false;
        if (!this->visitInitializerPop(Init->getInit()))
          return false;
      }
    }

    this->fallthrough(AfterVirtBasesLabel);
    this->emitLabel(AfterVirtBasesLabel);

    if (!this->emitPopPtr(Ctor))
      return false;
  }

  for (const auto *Init : Ctor->inits()) {
    // Scope needed for the initializers.
    LocalScope<Emitter> Scope(this, ScopeKind::FullExpression);

    const Expr *InitExpr = Init->getInit();
    if (const FieldDecl *Member = Init->getMember()) {
      const Record::Field *F = R->getField(Member);

      LocOverrideScope<Emitter> LOS(this, SourceInfo{},
                                    initNeedsOverridenLoc(Init));
      if (!emitFieldInitializer(F, F->Offset, InitExpr, IsUnion))
        return false;
      ++FieldInits;
    } else if (const Type *Base = Init->getBaseClass()) {
      const auto *BaseDecl = Base->getAsCXXRecordDecl();
      assert(BaseDecl);

      if (Init->isBaseVirtual()) {
        // See above.
        continue;
      } else {
        // Base class initializer.
        // Get This Base and call initializer on it.
        const Record::Base *B = R->getBase(BaseDecl);
        assert(B);
        if (!this->emitGetPtrThisBase(B->Offset, InitExpr))
          return false;
      }

      if (!this->visitInitializerPop(InitExpr))
        return false;
    } else if (const IndirectFieldDecl *IFD = Init->getIndirectMember()) {
      LocOverrideScope<Emitter> LOS(this, SourceInfo{},
                                    initNeedsOverridenLoc(Init));
      unsigned ChainSize = IFD->getChainingSize();
      assert(ChainSize >= 2);

      unsigned NestedFieldOffset = 0;
      const Record::Field *NestedField = nullptr;
      for (unsigned I = 0; I != ChainSize; ++I) {
        const auto *FD = cast<FieldDecl>(IFD->chain()[I]);
        const Record *FieldRecord = this->P.getOrCreateRecord(FD->getParent());
        assert(FieldRecord);

        NestedField = FieldRecord->getField(FD);
        assert(NestedField);
        IsUnion = IsUnion || FieldRecord->isUnion();

        NestedFieldOffset += NestedField->Offset;

        // Add a new InitChainLink for the record, but not for the final field.
        if (I != ChainSize - 1)
          InitStack.push_back(InitLink::Field(NestedField->Offset));
      }
      assert(NestedField);

      InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(InitExpr));
      if (!emitFieldInitializer(NestedField, NestedFieldOffset, InitExpr,
                                IsUnion))
        return false;

      // Mark all chain links as initialized.
      unsigned InitFieldOffset = 0;
      for (const NamedDecl *ND : IFD->chain().drop_back()) {
        const auto *FD = cast<FieldDecl>(ND);
        const Record *FieldRecord = this->P.getOrCreateRecord(FD->getParent());
        assert(FieldRecord);
        NestedField = FieldRecord->getField(FD);
        InitFieldOffset += NestedField->Offset;
        assert(NestedField);
        if (!this->emitGetPtrThisField(InitFieldOffset, InitExpr))
          return false;
        if (!this->emitFinishInitPop(InitExpr))
          return false;
      }

      InitStack.pop_back_n(ChainSize - 1);

    } else {
      assert(Init->isDelegatingInitializer());
      if (!this->emitThis(InitExpr))
        return false;
      if (!this->visitInitializerPop(Init->getInit()))
        return false;
    }

    if (!Scope.destroyLocals())
      return false;
  }

  if (FieldInits != R->getNumFields()) {
    assert(FieldInits < R->getNumFields());
    // Start the lifetime of all members.
    if (!this->emitStartThisLifetime(Ctor))
      return false;
  }

  if (const Stmt *Body = Ctor->getBody()) {
    // Only emit the CtorCheck op for non-empty CompoundStmt bodies.
    // For non-CompoundStmts, always assume they are non-empty and emit it.
    if (const auto *CS = dyn_cast<CompoundStmt>(Body)) {
      if (!CS->body_empty() && !this->emitCtorCheck(SourceInfo{}))
        return false;
    } else {
      if (!this->emitCtorCheck(SourceInfo{}))
        return false;
    }

    if (!visitStmt(Body))
      return false;
  }

  return this->emitRetVoid(SourceInfo{});
}

template <class Emitter>
bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
  const RecordDecl *RD = Dtor->getParent();
  const Record *R = this->getRecord(RD);
  if (!R)
    return false;

  if (!Dtor->isTrivial() && Dtor->getBody()) {
    if (!this->visitStmt(Dtor->getBody()))
      return false;
  }

  if (!this->emitThis(Dtor))
    return false;

  if (!this->emitCheckDestruction(Dtor))
    return false;

  assert(R);
  if (!R->isUnion()) {

    LocOverrideScope<Emitter> LOS(this, SourceInfo{});
    // First, destroy all fields.
    for (const Record::Field &Field : llvm::reverse(R->fields())) {
      const Descriptor *D = Field.Desc;
      if (D->hasTrivialDtor())
        continue;
      if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
        return false;
      if (!this->emitDestructionPop(D, SourceInfo{}))
        return false;
    }
  }

  for (const Record::Base &Base : llvm::reverse(R->bases())) {
    if (Base.R->hasTrivialDtor())
      continue;
    if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
      return false;
    if (!this->emitRecordDestructionPop(Base.R, {}))
      return false;
  }

  if (R->getNumVirtualBases() > 0) {
    LabelTy EndLabel = this->getLabel();
    // If this is a base class, skip the virtual bases.
    if (!this->emitIsBaseClass({}))
      return false;
    if (!this->jumpTrue(EndLabel, {}))
      return false;

    for (const Record::Base &Base : llvm::reverse(R->virtual_bases())) {
      if (Base.R->hasTrivialDtor())
        continue;
      if (!this->emitGetPtrVirtBase(cast<CXXRecordDecl>(Base.R->getDecl()),
                                    SourceInfo{}))
        return false;
      if (!this->emitRecordDestructionPop(Base.R, {}))
        return false;
    }

    this->fallthrough(EndLabel);
    this->emitLabel(EndLabel);
  }

  if (!this->emitMarkDestroyed(Dtor))
    return false;

  return this->emitPopPtr(Dtor) && this->emitRetVoid(Dtor);
}

template <class Emitter>
bool Compiler<Emitter>::compileUnionAssignmentOperator(
    const CXXMethodDecl *MD) {
  if (!this->emitThis(MD))
    return false;

  if (!this->emitGetParam(PT_Ptr, /*ParamIndex=*/0, MD))
    return false;

  return this->emitMemcpy(MD) && this->emitRet(PT_Ptr, MD);
}

template <class Emitter>
bool Compiler<Emitter>::visitFunc(const FunctionDecl *F) {
  if (F->getReturnType()->isDependentType())
    return false;

  // Classify the return type.
  ReturnType = this->classify(F->getReturnType());

  this->CompilingFunction = F;

  if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(F))
    return this->compileConstructor(Ctor);
  if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(F))
    return this->compileDestructor(Dtor);

  // Emit custom code if this is a lambda static invoker.
  if (const auto *MD = dyn_cast<CXXMethodDecl>(F)) {
    const RecordDecl *RD = MD->getParent();

    if (RD->isUnion() &&
        (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()))
      return this->compileUnionAssignmentOperator(MD);

    if (MD->isLambdaStaticInvoker())
      return this->emitLambdaStaticInvokerBody(MD);
  }

  // Regular functions.
  if (const auto *Body = F->getBody())
    if (!visitStmt(Body))
      return false;

  // Emit a guard return to protect against a code path missing one.
  if (F->getReturnType()->isVoidType())
    return this->emitRetVoid(SourceInfo{});
  return this->emitNoRet(SourceInfo{});
}

static uint32_t getBitWidth(const Expr *E) {
  assert(E->refersToBitField());
  const auto *ME = cast<MemberExpr>(E);
  const auto *FD = cast<FieldDecl>(ME->getMemberDecl());
  return FD->getBitWidthValue();
}

template <class Emitter>
bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
  if (E->containsErrors())
    return false;

  const Expr *SubExpr = E->getSubExpr();
  if (SubExpr->getType()->isAnyComplexType())
    return this->VisitComplexUnaryOperator(E);
  if (SubExpr->getType()->isVectorType())
    return this->VisitVectorUnaryOperator(E);
  if (SubExpr->getType()->isFixedPointType())
    return this->VisitFixedPointUnaryOperator(E);
  OptPrimType T = classify(SubExpr->getType());

  switch (E->getOpcode()) {
  case UO_PostInc: { // x++
    if (!Ctx.getLangOpts().CPlusPlus14)
      return this->emitInvalid(E);
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;

    if (T == PT_Ptr) {
      if (!this->emitIncPtr(E))
        return false;

      return DiscardResult ? this->emitPopPtr(E) : true;
    }

    if (T == PT_Float)
      return DiscardResult ? this->emitIncfPop(getFPOptions(E), E)
                           : this->emitIncf(getFPOptions(E), E);

    if (SubExpr->refersToBitField())
      return DiscardResult ? this->emitIncPopBitfield(*T, E->canOverflow(),
                                                      getBitWidth(SubExpr), E)
                           : this->emitIncBitfield(*T, E->canOverflow(),
                                                   getBitWidth(SubExpr), E);

    return DiscardResult ? this->emitIncPop(*T, E->canOverflow(), E)
                         : this->emitInc(*T, E->canOverflow(), E);
  }
  case UO_PostDec: { // x--
    if (!Ctx.getLangOpts().CPlusPlus14)
      return this->emitInvalid(E);
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;

    if (T == PT_Ptr) {
      if (!this->emitDecPtr(E))
        return false;

      return DiscardResult ? this->emitPopPtr(E) : true;
    }

    if (T == PT_Float)
      return DiscardResult ? this->emitDecfPop(getFPOptions(E), E)
                           : this->emitDecf(getFPOptions(E), E);

    if (SubExpr->refersToBitField()) {
      return DiscardResult ? this->emitDecPopBitfield(*T, E->canOverflow(),
                                                      getBitWidth(SubExpr), E)
                           : this->emitDecBitfield(*T, E->canOverflow(),
                                                   getBitWidth(SubExpr), E);
    }

    return DiscardResult ? this->emitDecPop(*T, E->canOverflow(), E)
                         : this->emitDec(*T, E->canOverflow(), E);
  }
  case UO_PreInc: { // ++x
    if (!Ctx.getLangOpts().CPlusPlus14)
      return this->emitInvalid(E);
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;

    if (T == PT_Ptr) {
      if (!this->emitLoadPtr(E))
        return false;
      if (!this->emitConstUint8(1, E))
        return false;
      if (!this->emitAddOffsetUint8(E))
        return false;
      return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
    }

    // Post-inc and pre-inc are the same if the value is to be discarded.
    if (DiscardResult) {
      if (T == PT_Float)
        return this->emitIncfPop(getFPOptions(E), E);
      if (SubExpr->refersToBitField())
        return DiscardResult ? this->emitIncPopBitfield(*T, E->canOverflow(),
                                                        getBitWidth(SubExpr), E)
                             : this->emitIncBitfield(*T, E->canOverflow(),
                                                     getBitWidth(SubExpr), E);
      return this->emitIncPop(*T, E->canOverflow(), E);
    }

    if (T == PT_Float) {
      const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
      if (!this->emitLoadFloat(E))
        return false;
      APFloat F(TargetSemantics, 1);
      if (!this->emitFloat(F, E))
        return false;

      if (!this->emitAddf(getFPOptions(E), E))
        return false;
      if (!this->emitStoreFloat(E))
        return false;
    } else if (SubExpr->refersToBitField()) {
      assert(isIntegerOrBoolType(*T));
      if (!this->emitPreIncBitfield(*T, E->canOverflow(), getBitWidth(SubExpr),
                                    E))
        return false;
    } else {
      assert(isIntegerOrBoolType(*T));
      if (!this->emitPreInc(*T, E->canOverflow(), E))
        return false;
    }
    return E->isGLValue() || this->emitLoadPop(*T, E);
  }
  case UO_PreDec: { // --x
    if (!Ctx.getLangOpts().CPlusPlus14)
      return this->emitInvalid(E);
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;

    if (T == PT_Ptr) {
      if (!this->emitLoadPtr(E))
        return false;
      if (!this->emitConstUint8(1, E))
        return false;
      if (!this->emitSubOffsetUint8(E))
        return false;
      return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
    }

    // Post-dec and pre-dec are the same if the value is to be discarded.
    if (DiscardResult) {
      if (T == PT_Float)
        return this->emitDecfPop(getFPOptions(E), E);
      if (SubExpr->refersToBitField())
        return DiscardResult ? this->emitDecPopBitfield(*T, E->canOverflow(),
                                                        getBitWidth(SubExpr), E)
                             : this->emitDecBitfield(*T, E->canOverflow(),
                                                     getBitWidth(SubExpr), E);
      return this->emitDecPop(*T, E->canOverflow(), E);
    }

    if (T == PT_Float) {
      const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
      if (!this->emitLoadFloat(E))
        return false;
      APFloat F(TargetSemantics, 1);
      if (!this->emitFloat(F, E))
        return false;

      if (!this->emitSubf(getFPOptions(E), E))
        return false;
      if (!this->emitStoreFloat(E))
        return false;
    } else if (SubExpr->refersToBitField()) {
      assert(isIntegerOrBoolType(*T));
      if (!this->emitPreDecBitfield(*T, E->canOverflow(), getBitWidth(SubExpr),
                                    E))
        return false;
    } else {
      assert(isIntegerOrBoolType(*T));
      if (!this->emitPreDec(*T, E->canOverflow(), E))
        return false;
    }
    return E->isGLValue() || this->emitLoadPop(*T, E);
  }
  case UO_LNot: // !x
    if (!T)
      return this->emitError(E);

    if (DiscardResult)
      return this->discard(SubExpr);

    if (!this->visitBool(SubExpr))
      return false;

    if (!this->emitInv(E))
      return false;

    if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
      return this->emitCast(PT_Bool, ET, E);
    return true;
  case UO_Minus: // -x
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;
    return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
  case UO_Plus: // +x
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr)) // noop
      return false;
    return DiscardResult ? this->emitPop(*T, E) : true;
  case UO_AddrOf: // &x
    if (E->getType()->isMemberPointerType()) {
      // C++11 [expr.unary.op]p3 has very strict rules on how the address of a
      // member can be formed.
      if (DiscardResult)
        return true;
      return this->emitGetMemberPtr(cast<DeclRefExpr>(SubExpr)->getDecl(), E);
    }
    // [C11 6.5.3.2p3]: if the operand of '&' is the result of a unary '*'
    // operator, neither operator is evaluated and the result is as if both
    // were omitted. So '&*q' is just 'q' with no dereference; delegate to the
    // pointer operand directly instead of to the '*' (which would emit a null
    // check), so that e.g. '&*(int *)0' is not rejected.
    if (!Ctx.getLangOpts().CPlusPlus) {
      const Expr *Sub = SubExpr->IgnoreParens();

      if (const auto *Deref = dyn_cast<UnaryOperator>(Sub);
          Deref && Deref->getOpcode() == UO_Deref) {
        if (DiscardResult)
          return this->discard(Deref->getSubExpr());
        return this->visit(Deref->getSubExpr()) && this->emitAddrOf(E);
      }
    }
    // We should already have a pointer when we get here.
    if (DiscardResult)
      return this->discard(SubExpr);
    return this->delegate(SubExpr) && this->emitAddrOf(E);
  case UO_Deref: // *x
    if (DiscardResult)
      return this->discard(SubExpr);

    if (!this->visit(SubExpr))
      return false;

    if (!SubExpr->getType()->isFunctionPointerType() && !this->emitCheckNull(E))
      return false;

    if (classifyPrim(SubExpr) == PT_Ptr)
      return this->emitNarrowPtr(E);
    return true;

  case UO_Not: // ~x
    if (!T)
      return this->emitError(E);

    if (!this->visit(SubExpr))
      return false;
    return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
  case UO_Real: // __real x
    if (!T)
      return false;
    return this->delegate(SubExpr);
  case UO_Imag: { // __imag x
    if (!T)
      return false;
    if (!this->discard(SubExpr))
      return false;
    return DiscardResult
               ? true
               : this->visitZeroInitializer(*T, SubExpr->getType(), SubExpr);
  }
  case UO_Extension:
    return this->delegate(SubExpr);
  case UO_Coawait:
    assert(false && "Unhandled opcode");
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
  const Expr *SubExpr = E->getSubExpr();
  assert(SubExpr->getType()->isAnyComplexType());

  if (DiscardResult)
    return this->discard(SubExpr);

  OptPrimType ResT = classify(E);
  auto prepareResult = [=]() -> bool {
    if (!ResT && !Initializing) {
      UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
      if (!LocalIndex)
        return false;
      return this->emitGetPtrLocal(*LocalIndex, E);
    }

    return true;
  };

  // The offset of the temporary, if we created one.
  unsigned SubExprOffset = ~0u;
  auto createTemp = [=, &SubExprOffset]() -> bool {
    SubExprOffset =
        this->allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(SubExpr))
      return false;
    return this->emitSetLocal(PT_Ptr, SubExprOffset, E);
  };

  PrimType ElemT = classifyComplexElementType(SubExpr->getType());
  auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
    if (!this->emitGetLocal(PT_Ptr, Offset, E))
      return false;
    return this->emitArrayElemPop(ElemT, Index, E);
  };

  switch (E->getOpcode()) {
  case UO_Minus: // -x
    if (!prepareResult())
      return false;
    if (!createTemp())
      return false;
    for (unsigned I = 0; I != 2; ++I) {
      if (!getElem(SubExprOffset, I))
        return false;
      if (!this->emitNeg(ElemT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    break;

  case UO_Plus:   // +x
  case UO_AddrOf: // &x
  case UO_Deref:  // *x
    return this->delegate(SubExpr);

  case UO_LNot:
    if (!this->visit(SubExpr))
      return false;
    if (!this->emitComplexBoolCast(SubExpr))
      return false;
    if (!this->emitInv(E))
      return false;
    if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
      return this->emitCast(PT_Bool, ET, E);
    return true;

  case UO_Real:
    return this->emitComplexReal(SubExpr);

  case UO_Imag:
    if (!this->visit(SubExpr))
      return false;

    if (SubExpr->isLValue()) {
      if (!this->emitConstUint8(1, E))
        return false;
      return this->emitArrayElemPtrPopUint8(E);
    }

    // Since our _Complex implementation does not map to a primitive type,
    // we sometimes have to do the lvalue-to-rvalue conversion here manually.
    return this->emitArrayElemPop(classifyPrim(E->getType()), 1, E);

  case UO_Not: // ~x
    if (!this->delegate(SubExpr))
      return false;
    // Negate the imaginary component.
    if (!this->emitArrayElem(ElemT, 1, E))
      return false;
    if (!this->emitNeg(ElemT, E))
      return false;
    if (!this->emitInitElem(ElemT, 1, E))
      return false;
    return DiscardResult ? this->emitPopPtr(E) : true;

  case UO_Extension:
    return this->delegate(SubExpr);

  default:
    return this->emitInvalid(E);
  }

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
  const Expr *SubExpr = E->getSubExpr();
  assert(SubExpr->getType()->isVectorType());

  if (DiscardResult)
    return this->discard(SubExpr);

  auto UnaryOp = E->getOpcode();
  if (UnaryOp == UO_Extension)
    return this->delegate(SubExpr);

  if (UnaryOp != UO_Plus && UnaryOp != UO_Minus && UnaryOp != UO_LNot &&
      UnaryOp != UO_Not && UnaryOp != UO_AddrOf)
    return this->emitInvalid(E);

  // Nothing to do here.
  if (UnaryOp == UO_Plus || UnaryOp == UO_AddrOf)
    return this->delegate(SubExpr);

  if (!Initializing) {
    UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  // The offset of the temporary, if we created one.
  unsigned SubExprOffset =
      this->allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
  if (!this->visit(SubExpr))
    return false;
  if (!this->emitSetLocal(PT_Ptr, SubExprOffset, E))
    return false;

  const auto *VecTy = SubExpr->getType()->getAs<VectorType>();
  PrimType ElemT = classifyVectorElementType(SubExpr->getType());
  auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
    if (!this->emitGetLocal(PT_Ptr, Offset, E))
      return false;
    return this->emitArrayElemPop(ElemT, Index, E);
  };

  switch (UnaryOp) {
  case UO_Minus:
    for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
      if (!getElem(SubExprOffset, I))
        return false;
      if (!this->emitNeg(ElemT, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    break;
  case UO_LNot: { // !x
    // In C++, the logic operators !, &&, || are available for vectors. !v is
    // equivalent to v == 0.
    //
    // The result of the comparison is a vector of the same width and number of
    // elements as the comparison operands with a signed integral element type.
    //
    // https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
    QualType ResultVecTy = E->getType();
    PrimType ResultVecElemT =
        classifyPrim(ResultVecTy->getAs<VectorType>()->getElementType());
    for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
      if (!getElem(SubExprOffset, I))
        return false;
      // operator ! on vectors returns -1 for 'truth', so negate it.
      if (!this->emitPrimCast(ElemT, PT_Bool, Ctx.getASTContext().BoolTy, E))
        return false;
      if (!this->emitInv(E))
        return false;
      if (!this->emitPrimCast(PT_Bool, ElemT, VecTy->getElementType(), E))
        return false;
      if (!this->emitNeg(ElemT, E))
        return false;
      if (ElemT != ResultVecElemT &&
          !this->emitPrimCast(ElemT, ResultVecElemT, ResultVecTy, E))
        return false;
      if (!this->emitInitElem(ResultVecElemT, I, E))
        return false;
    }
    break;
  }
  case UO_Not: // ~x
    for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
      if (!getElem(SubExprOffset, I))
        return false;
      if (ElemT == PT_Bool) {
        if (!this->emitInv(E))
          return false;
      } else {
        if (!this->emitComp(ElemT, E))
          return false;
      }
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    break;
  default:
    llvm_unreachable("Unsupported unary operators should be handled up front");
  }
  return true;
}

template <class Emitter>
bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
  if (const auto *ECD = dyn_cast<EnumConstantDecl>(D)) {
    if (DiscardResult)
      return true;
    return this->emitConst(ECD->getInitVal(), E);
  }
  if (const auto *FuncDecl = dyn_cast<FunctionDecl>(D)) {
    if (DiscardResult)
      return true;
    const Function *F = getFunction(FuncDecl);
    return F && this->emitGetFnPtr(F, E);
  }
  if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(D)) {
    TPOD = TPOD->getFirstDecl();
    if (DiscardResult)
      return true;
    if (UnsignedOrNone GlobalIndex = P.getGlobal(TPOD))
      return this->emitGetPtrGlobal(*GlobalIndex, E);

    if (UnsignedOrNone Index = P.getOrCreateGlobal(TPOD)) {
      if (OptPrimType T = classify(TPOD->getType())) {
        if (!this->visitAPValue(TPOD->getValue(), *T, E))
          return false;
        return this->emitInitGlobal(*T, *Index, E);
      }

      if (!this->emitGetPtrGlobal(*Index, E))
        return false;
      if (!this->visitAPValueInitializer(TPOD->getValue(), E, TPOD->getType()))
        return false;
      return this->emitFinishInit(E);
    }
    return false;
  }

  // References are implemented via pointers, so when we see a DeclRefExpr
  // pointing to a reference, we need to get its value directly (i.e. the
  // pointer to the actual value) instead of a pointer to the pointer to the
  // value.
  QualType DeclType = D->getType();
  bool IsReference = DeclType->isReferenceType();

  auto maybePopPtr = [&]() -> bool {
    if (DiscardResult)
      return this->emitPopPtr(E);
    return true;
  };

  // Function parameters.
  // Note that it's important to check them first since we might have a local
  // variable created for a ParmVarDecl as well.
  if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
    if (DiscardResult)
      return true;

    if (Ctx.getLangOpts().CPlusPlus && !Ctx.getLangOpts().CPlusPlus11 &&
        !DeclType->isIntegralOrEnumerationType()) {
      return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
                                      /*InitializerFailed=*/false, E);
    }
    if (auto It = this->Params.find(PVD); It != this->Params.end()) {
      if (IsReference || !It->second.IsPtr)
        return this->emitGetParam(classifyPrim(E), It->second.Index, E);

      return this->emitGetPtrParam(It->second.Index, E);
    }

    if (!Ctx.getLangOpts().CPlusPlus23 && IsReference && !Locals.contains(D))
      return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
                                      /*InitializerFailed=*/false, E);
  }

  // Local variables.
  if (auto It = Locals.find(D); It != Locals.end()) {
    const unsigned Offset = It->second.Offset;
    if (IsReference) {
      assert(classifyPrim(E) == PT_Ptr);
      return this->emitGetRefLocal(Offset, E) && maybePopPtr();
    }
    return this->emitGetPtrLocal(Offset, E) && maybePopPtr();
  }
  // Global variables.
  if (auto GlobalIndex = P.getGlobal(D)) {
    if (IsReference) {
      if (!Ctx.getLangOpts().CPlusPlus11)
        return this->emitGetGlobal(classifyPrim(E), *GlobalIndex, E);
      if (!Ctx.getLangOpts().CPlusPlus23)
        return this->emitGetGlobalUnchecked(classifyPrim(E), *GlobalIndex, E);

      return this->emitGetRefGlobal(*GlobalIndex, E) && maybePopPtr();
    }

    return this->emitGetPtrGlobal(*GlobalIndex, E) && maybePopPtr();
  }

  // In case we need to re-visit a declaration.
  auto revisit = [&](const VarDecl *VD,
                     bool IsConstexprUnknown = true) -> bool {
    llvm::SaveAndRestore CURS(this->VariablesAreConstexprUnknown,
                              IsConstexprUnknown);
    if constexpr (std::is_same_v<Emitter, EvalEmitter>) {
      if (!this->emitPushCC(VD->hasConstantInitialization(), E))
        return false;
    }
    auto VarState = this->visitDecl(VD);

    if constexpr (std::is_same_v<Emitter, EvalEmitter>) {
      if (!this->emitPopCC(E))
        return false;
    }

    if (VarState.notCreated())
      return true;
    if (!VarState)
      return false;
    // Retry.
    return this->visitDeclRef(D, E);
  };

  if constexpr (!std::is_same_v<Emitter, EvalEmitter>) {
    // Lambda captures.
    if (auto It = this->LambdaCaptures.find(D);
        It != this->LambdaCaptures.end()) {
      auto [Offset, IsPtr] = It->second;

      if (IsPtr)
        return this->emitGetThisFieldPtr(Offset, E) && maybePopPtr();
      return this->emitGetPtrThisField(Offset, E) && maybePopPtr();
    }
  }

  if (const auto *DRE = dyn_cast<DeclRefExpr>(E);
      DRE && DRE->refersToEnclosingVariableOrCapture()) {
    if (const auto *VD = dyn_cast<VarDecl>(D); VD && VD->isInitCapture())
      return revisit(VD);
  }

  if (const auto *BD = dyn_cast<BindingDecl>(D))
    return this->delegate(BD->getBinding());

  // Avoid infinite recursion.
  if (D == InitializingDecl) {
    if (DiscardResult)
      return true;
    return this->emitDummyPtr(D, E);
  }

  // Try to lazily visit (or emit dummy pointers for) declarations
  // we haven't seen yet.
  const auto *VD = dyn_cast<VarDecl>(D);
  if (!VD)
    return this->emitError(E);

  // For C.
  if (!Ctx.getLangOpts().CPlusPlus) {
    if (VD->getInit() && !VD->getInit()->isValueDependent() &&
        DeclType.isConstant(Ctx.getASTContext()) && !VD->isWeak() &&
        VD->evaluateValue())
      return revisit(VD, /*IsConstexprUnknown=*/false);

    if (DiscardResult)
      return true;
    return this->emitDummyPtr(D, E);
  }

  // ... and C++.
  const auto typeShouldBeVisited = [&](QualType T) -> bool {
    if (T.isConstant(Ctx.getASTContext()))
      return true;
    return T->isReferenceType();
  };

  if ((VD->hasGlobalStorage() || VD->isStaticDataMember()) &&
      typeShouldBeVisited(DeclType)) {
    if (const Expr *Init = VD->getAnyInitializer();
        Init && !Init->isValueDependent()) {
      // Whether or not the evaluation is successul doesn't really matter
      // here -- we will create a global variable in any case, and that
      // will have the state of initializer evaluation attached.
      Expr::EvalResult Result;
      (void)Init->EvaluateAsInitializer(Ctx.getASTContext(), VD, Result, true);
      return this->visitDeclRef(D, E);
    }
    return revisit(VD, !VD->isConstexpr() && DeclType->isReferenceType());
  }

  // FIXME: The evaluateValue() check here is a little ridiculous, since
  // it will ultimately call into Context::evaluateAsInitializer(). In
  // other words, we're evaluating the initializer, just to know if we can
  // evaluate the initializer.
  if (VD->isLocalVarDecl() && typeShouldBeVisited(DeclType) && VD->getInit() &&
      !VD->getInit()->isValueDependent()) {
    if (VD->evaluateValue()) {
      bool IsConstexprUnknown = !DeclType.isConstant(Ctx.getASTContext()) &&
                                !DeclType->isReferenceType();
      // Revisit the variable declaration, but make sure it's associated with a
      // different evaluation, so e.g. mutable reads don't work on it.
      EvalIDScope _(Ctx);
      return revisit(VD, IsConstexprUnknown);
    } else if (Ctx.getLangOpts().CPlusPlus23 && IsReference)
      return revisit(VD, /*IsConstexprUnknown=*/true);

    if (IsReference)
      return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
                                      /*InitializerFailed=*/true, E);
  }

  if (DiscardResult)
    return true;
  return this->emitDummyPtr(
      D, E, Ctx.getLangOpts().CPlusPlus23 && DeclType->isReferenceType());
}

template <class Emitter>
bool Compiler<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
  const auto *D = E->getDecl();
  return this->visitDeclRef(D, E);
}

template <class Emitter>
bool Compiler<Emitter>::VisitDesignatedInitUpdateExpr(
    const DesignatedInitUpdateExpr *E) {
  if (!this->visitInitializer(E->getBase()))
    return false;
  return this->visitInitializer(E->getUpdater());
}

template <class Emitter> bool Compiler<Emitter>::emitCleanup() {
  for (VariableScope<Emitter> *C = VarScope; C; C = C->getParent()) {
    if (!C->destroyLocals())
      return false;
  }
  return true;
}

template <class Emitter>
unsigned Compiler<Emitter>::collectBaseOffset(const QualType BaseType,
                                              const QualType DerivedType) {
  const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
    if (const auto *R = Ty->getPointeeCXXRecordDecl())
      return R;
    return Ty->getAsCXXRecordDecl();
  };
  const CXXRecordDecl *BaseDecl = extractRecordDecl(BaseType);
  const CXXRecordDecl *DerivedDecl = extractRecordDecl(DerivedType);

  return Ctx.collectBaseOffset(BaseDecl, DerivedDecl);
}

/// Emit casts from a PrimType to another PrimType.
template <class Emitter>
bool Compiler<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
                                     QualType ToQT, const Expr *E) {

  if (FromT == PT_Float) {
    // Floating to floating.
    if (ToT == PT_Float) {
      const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
      return this->emitCastFP(ToSem, getRoundingMode(E), E);
    }

    if (ToT == PT_IntAP)
      return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(ToQT),
                                              getFPOptions(E), E);
    if (ToT == PT_IntAPS)
      return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(ToQT),
                                               getFPOptions(E), E);

    // Float to integral.
    if (isIntegerOrBoolType(ToT) || ToT == PT_Bool)
      return this->emitCastFloatingIntegral(ToT, getFPOptions(E), E);
  }

  if (isIntegerOrBoolType(FromT) || FromT == PT_Bool) {
    if (ToT == PT_IntAP)
      return this->emitCastAP(FromT, Ctx.getBitWidth(ToQT), E);
    if (ToT == PT_IntAPS)
      return this->emitCastAPS(FromT, Ctx.getBitWidth(ToQT), E);

    // Integral to integral.
    if (isIntegerOrBoolType(ToT) || ToT == PT_Bool)
      return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;

    if (ToT == PT_Float) {
      // Integral to floating.
      const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(ToQT);
      return this->emitCastIntegralFloating(FromT, ToSem, getFPOptions(E), E);
    }
  }

  return false;
}

template <class Emitter>
bool Compiler<Emitter>::emitIntegralCast(PrimType FromT, PrimType ToT,
                                         QualType ToQT, const Expr *E) {
  assert(FromT != ToT);

  if (ToT == PT_IntAP)
    return this->emitCastAP(FromT, Ctx.getBitWidth(ToQT), E);
  if (ToT == PT_IntAPS)
    return this->emitCastAPS(FromT, Ctx.getBitWidth(ToQT), E);

  return this->emitCast(FromT, ToT, E);
}

/// Emits __real(SubExpr)
template <class Emitter>
bool Compiler<Emitter>::emitComplexReal(const Expr *SubExpr) {
  assert(SubExpr->getType()->isAnyComplexType());

  if (DiscardResult)
    return this->discard(SubExpr);

  if (!this->visit(SubExpr))
    return false;
  if (SubExpr->isLValue()) {
    if (!this->emitConstUint8(0, SubExpr))
      return false;
    return this->emitArrayElemPtrPopUint8(SubExpr);
  }

  // Rvalue, load the actual element.
  return this->emitArrayElemPop(classifyComplexElementType(SubExpr->getType()),
                                0, SubExpr);
}

template <class Emitter>
bool Compiler<Emitter>::emitComplexBoolCast(const Expr *E) {
  assert(!DiscardResult);
  PrimType ElemT = classifyComplexElementType(E->getType());
  // We emit the expression (__real(E) != 0 || __imag(E) != 0)
  // for us, that means (bool)E[0] || (bool)E[1]
  if (!this->emitArrayElem(ElemT, 0, E))
    return false;
  if (ElemT == PT_Float) {
    if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
      return false;
  } else {
    if (!this->emitCast(ElemT, PT_Bool, E))
      return false;
  }

  // We now have the bool value of E[0] on the stack.
  LabelTy LabelTrue = this->getLabel();
  if (!this->jumpTrue(LabelTrue, E))
    return false;

  if (!this->emitArrayElemPop(ElemT, 1, E))
    return false;
  if (ElemT == PT_Float) {
    if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
      return false;
  } else {
    if (!this->emitCast(ElemT, PT_Bool, E))
      return false;
  }
  // Leave the boolean value of E[1] on the stack.
  LabelTy EndLabel = this->getLabel();
  this->jump(EndLabel, E);

  this->emitLabel(LabelTrue);
  if (!this->emitPopPtr(E))
    return false;
  if (!this->emitConstBool(true, E))
    return false;

  this->fallthrough(EndLabel);
  this->emitLabel(EndLabel);

  return true;
}

template <class Emitter>
bool Compiler<Emitter>::emitComplexComparison(const Expr *LHS, const Expr *RHS,
                                              const BinaryOperator *E) {
  assert(E->isComparisonOp());
  assert(!Initializing);
  if (DiscardResult)
    return this->discard(LHS) && this->discard(RHS);

  PrimType ElemT;
  bool LHSIsComplex;
  unsigned LHSOffset;
  if (LHS->getType()->isAnyComplexType()) {
    LHSIsComplex = true;
    ElemT = classifyComplexElementType(LHS->getType());
    LHSOffset = allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(LHS))
      return false;
    if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
      return false;
  } else {
    LHSIsComplex = false;
    PrimType LHST = classifyPrim(LHS->getType());
    LHSOffset = this->allocateLocalPrimitive(LHS, LHST, /*IsConst=*/true);
    if (!this->visit(LHS))
      return false;
    if (!this->emitSetLocal(LHST, LHSOffset, E))
      return false;
  }

  bool RHSIsComplex;
  unsigned RHSOffset;
  if (RHS->getType()->isAnyComplexType()) {
    RHSIsComplex = true;
    ElemT = classifyComplexElementType(RHS->getType());
    RHSOffset = allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true);
    if (!this->visit(RHS))
      return false;
    if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
      return false;
  } else {
    RHSIsComplex = false;
    PrimType RHST = classifyPrim(RHS->getType());
    RHSOffset = this->allocateLocalPrimitive(RHS, RHST, /*IsConst=*/true);
    if (!this->visit(RHS))
      return false;
    if (!this->emitSetLocal(RHST, RHSOffset, E))
      return false;
  }

  auto getElem = [&](unsigned LocalOffset, unsigned Index,
                     bool IsComplex) -> bool {
    if (IsComplex) {
      if (!this->emitGetLocal(PT_Ptr, LocalOffset, E))
        return false;
      return this->emitArrayElemPop(ElemT, Index, E);
    }
    return this->emitGetLocal(ElemT, LocalOffset, E);
  };

  for (unsigned I = 0; I != 2; ++I) {
    // Get both values.
    if (!getElem(LHSOffset, I, LHSIsComplex))
      return false;
    if (!getElem(RHSOffset, I, RHSIsComplex))
      return false;
    // And compare them.
    if (!this->emitEQ(ElemT, E))
      return false;

    if (!this->emitCastBoolUint8(E))
      return false;
  }

  // We now have two bool values on the stack. Compare those.
  if (!this->emitAddUint8(E))
    return false;
  if (!this->emitConstUint8(2, E))
    return false;

  if (E->getOpcode() == BO_EQ) {
    if (!this->emitEQUint8(E))
      return false;
  } else if (E->getOpcode() == BO_NE) {
    if (!this->emitNEUint8(E))
      return false;
  } else
    return false;

  // In C, this returns an int.
  if (PrimType ResT = classifyPrim(E->getType()); ResT != PT_Bool)
    return this->emitCast(PT_Bool, ResT, E);
  return true;
}

/// When calling this, we have a pointer of the local-to-destroy
/// on the stack.
/// Emit destruction of record types (or arrays of record types).
template <class Emitter>
bool Compiler<Emitter>::emitRecordDestructionPop(const Record *R,
                                                 SourceInfo Loc) {
  assert(R);
  assert(!R->hasTrivialDtor());
  const CXXDestructorDecl *Dtor = R->getDestructor();
  assert(Dtor);
  const Function *DtorFunc = getFunction(Dtor);
  if (!DtorFunc)
    return false;
  assert(DtorFunc->hasThisPointer());
  assert(DtorFunc->getNumParams() == 1);
  return this->emitCall(DtorFunc, 0, Loc);
}
/// When calling this, we have a pointer of the local-to-destroy
/// on the stack.
/// Emit destruction of record types (or arrays of record types).
template <class Emitter>
bool Compiler<Emitter>::emitDestructionPop(const Descriptor *Desc,
                                           SourceInfo Loc) {
  assert(Desc);
  assert(!Desc->hasTrivialDtor());

  // Arrays.
  if (Desc->isArray()) {
    const Descriptor *ElemDesc = Desc->ElemDesc;
    assert(ElemDesc);

    unsigned N = Desc->getNumElems();
    if (N == 0)
      return this->emitPopPtr(Loc);

    for (ssize_t I = N - 1; I >= 1; --I) {
      if (!this->emitConstUint64(I, Loc))
        return false;
      if (!this->emitArrayElemPtrUint64(Loc))
        return false;
      if (!this->emitDestructionPop(ElemDesc, Loc))
        return false;
    }
    // Last iteration, removes the instance pointer from the stack.
    if (!this->emitConstUint64(0, Loc))
      return false;
    if (!this->emitArrayElemPtrPopUint64(Loc))
      return false;
    return this->emitDestructionPop(ElemDesc, Loc);
  }

  assert(Desc->ElemRecord);
  assert(!Desc->ElemRecord->hasTrivialDtor());
  return this->emitRecordDestructionPop(Desc->ElemRecord, Loc);
}

/// Create a dummy pointer for the given decl (or expr) and
/// push a pointer to it on the stack.
template <class Emitter>
bool Compiler<Emitter>::emitDummyPtr(DeclOrExpr D, const Expr *E, bool CU) {
  assert(!DiscardResult && "Should've been checked before");
  return this->emitGetOpaquePtr(D, CU, E);
}

template <class Emitter>
bool Compiler<Emitter>::emitFloat(const APFloat &F, SourceInfo Info) {
  if (Floating::singleWord(F.getSemantics()))
    return this->emitConstFloat(Floating(F), Info);

  APInt I = F.bitcastToAPInt();
  return this->emitConstFloat(
      Floating(const_cast<uint64_t *>(I.getRawData()),
               llvm::APFloatBase::SemanticsToEnum(F.getSemantics())),
      Info);
}

//  This function is constexpr if and only if To, From, and the types of
//  all subobjects of To and From are types T such that...
//  (3.1) - is_union_v<T> is false;
//  (3.2) - is_pointer_v<T> is false;
//  (3.3) - is_member_pointer_v<T> is false;
//  (3.4) - is_volatile_v<T> is false; and
//  (3.5) - T has no non-static data members of reference type
template <class Emitter>
bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
  const Expr *SubExpr = E->getSubExpr();
  QualType FromType = SubExpr->getType();
  QualType ToType = E->getType();
  OptPrimType ToT = classify(ToType);

  assert(!ToType->isReferenceType());

  // Prepare storage for the result in case we discard.
  if (DiscardResult && !Initializing && !ToT) {
    UnsignedOrNone LocalIndex = allocateLocal(E);
    if (!LocalIndex)
      return false;
    if (!this->emitGetPtrLocal(*LocalIndex, E))
      return false;
  }

  // Get a pointer to the value-to-cast on the stack.
  // For CK_LValueToRValueBitCast, this is always an lvalue and
  // we later assume it to be one (i.e. a PT_Ptr). However,
  // we call this function for other utility methods where
  // a bitcast might be useful, so convert it to a PT_Ptr in that case.
  if (SubExpr->isGLValue() || FromType->isVectorType()) {
    if (!this->visit(SubExpr))
      return false;
  } else if (OptPrimType FromT = classify(SubExpr)) {
    unsigned TempOffset =
        allocateLocalPrimitive(SubExpr, *FromT, /*IsConst=*/true);
    if (!this->visit(SubExpr))
      return false;
    if (!this->emitSetLocal(*FromT, TempOffset, E))
      return false;
    if (!this->emitGetPtrLocal(TempOffset, E))
      return false;
  } else {
    return false;
  }

  if (!ToT) {
    if (!this->emitBitCast(E))
      return false;
    return DiscardResult ? this->emitPopPtr(E) : true;
  }
  assert(ToT);

  const llvm::fltSemantics *TargetSemantics = nullptr;
  if (ToT == PT_Float)
    TargetSemantics = &Ctx.getFloatSemantics(ToType);

  // Conversion to a primitive type. FromType can be another
  // primitive type, or a record/array.
  bool ToTypeIsUChar = (ToType->isSpecificBuiltinType(BuiltinType::UChar) ||
                        ToType->isSpecificBuiltinType(BuiltinType::Char_U));
  uint32_t ResultBitWidth = std::max(Ctx.getBitWidth(ToType), 8u);

  if (!this->emitBitCastPrim(*ToT, ToTypeIsUChar || ToType->isStdByteType(),
                             ResultBitWidth, TargetSemantics,
                             ToType.getTypePtr(), E))
    return false;

  if (DiscardResult)
    return this->emitPop(*ToT, E);

  return true;
}

/// Replicate a scalar value into every scalar element of an aggregate.
/// The scalar is stored in a local at \p SrcOffset and a pointer to the
/// destination must be on top of the interpreter stack. Each element receives
/// the scalar, cast to its own type.
template <class Emitter>
bool Compiler<Emitter>::emitHLSLAggregateSplat(PrimType SrcT,
                                               unsigned SrcOffset,
                                               QualType DestType,
                                               const Expr *E) {
  // Vectors and matrices are treated as flat sequences of elements.
  unsigned NumElems = 0;
  QualType ElemType;
  if (const auto *VT = DestType->getAs<VectorType>()) {
    NumElems = VT->getNumElements();
    ElemType = VT->getElementType();
  } else if (const auto *MT = DestType->getAs<ConstantMatrixType>()) {
    NumElems = MT->getNumElementsFlattened();
    ElemType = MT->getElementType();
  }
  if (NumElems > 0) {
    PrimType ElemT = classifyPrim(ElemType);
    for (unsigned I = 0; I != NumElems; ++I) {
      if (!this->emitGetLocal(SrcT, SrcOffset, E))
        return false;
      if (!this->emitPrimCast(SrcT, ElemT, ElemType, E))
        return false;
      if (!this->emitInitElem(ElemT, I, E))
        return false;
    }
    return true;
  }

  // Arrays: primitive elements are filled directly; composite elements
  // require recursion into each sub-aggregate.
  if (const auto *AT = DestType->getAsArrayTypeUnsafe()) {
    const auto *CAT = cast<ConstantArrayType>(AT);
    QualType ArrElemType = CAT->getElementType();
    unsigned ArrSize = CAT->getZExtSize();

    if (OptPrimType ElemT = classify(ArrElemType)) {
      for (unsigned I = 0; I != ArrSize; ++I) {
        if (!this->emitGetLocal(SrcT, SrcOffset, E))
          return false;
        if (!this->emitPrimCast(SrcT, *ElemT, ArrElemType, E))
          return false;
        if (!this->emitInitElem(*ElemT, I, E))
          return false;
      }
    } else {
      for (unsigned I = 0; I != ArrSize; ++I) {
        if (!this->emitConstUint32(I, E))
          return false;
        if (!this->emitArrayElemPtrUint32(E))
          return false;
        if (!emitHLSLAggregateSplat(SrcT, SrcOffset, ArrElemType, E))
          return false;
        if (!this->emitFinishInitPop(E))
          return false;
      }
    }
    return true;
  }

  // Records: fill base classes first, then named fields in declaration
  // order.
  if (DestType->isRecordType()) {
    const Record *R = getRecord(DestType);
    if (!R)
      return false;

    if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(R->getDecl())) {
      for (const CXXBaseSpecifier &BS : CXXRD->bases()) {
        const Record::Base *B = R->getBase(BS.getType());
        assert(B);
        if (!this->emitGetPtrBase(B->Offset, E))
          return false;
        if (!emitHLSLAggregateSplat(SrcT, SrcOffset, BS.getType(), E))
          return false;
        if (!this->emitFinishInitPop(E))
          return false;
      }
    }

    for (const Record::Field &F : R->fields()) {
      if (F.isUnnamedBitField())
        continue;

      QualType FieldType = F.Decl->getType();
      if (OptPrimType FieldT = F.T) {
        if (!this->emitGetLocal(SrcT, SrcOffset, E))
          return false;
        if (!this->emitPrimCast(SrcT, *FieldT, FieldType, E))
          return false;
        if (F.isBitField()) {
          if (!this->emitInitBitField(*FieldT, F.Offset, F.bitWidth(), E))
            return false;
        } else {
          if (!this->emitInitField(*FieldT, F.Offset, E))
            return false;
        }
      } else {
        if (!this->emitGetPtrField(F.Offset, E))
          return false;
        if (!emitHLSLAggregateSplat(SrcT, SrcOffset, FieldType, E))
          return false;
        if (!this->emitPopPtr(E))
          return false;
      }
    }
    return true;
  }

  return false;
}

/// Return the total number of scalar elements in a type. This is used
/// to cap how many source elements are extracted during an elementwise cast,
/// so we never flatten more than the destination can hold.
template <class Emitter>
unsigned Compiler<Emitter>::countHLSLFlatElements(QualType Ty) {
  // Vector and matrix types are treated as flat sequences of elements.
  if (const auto *VT = Ty->getAs<VectorType>())
    return VT->getNumElements();
  if (const auto *MT = Ty->getAs<ConstantMatrixType>())
    return MT->getNumElementsFlattened();
  // Arrays: total count is array size * scalar elements per element.
  if (const auto *AT = Ty->getAsArrayTypeUnsafe()) {
    const auto *CAT = cast<ConstantArrayType>(AT);
    return CAT->getZExtSize() * countHLSLFlatElements(CAT->getElementType());
  }
  // Records: sum scalar element counts of base classes and named fields.
  if (Ty->isRecordType()) {
    const Record *R = getRecord(Ty);
    if (!R)
      return 0;
    unsigned Count = 0;
    if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(R->getDecl())) {
      for (const CXXBaseSpecifier &BS : CXXRD->bases())
        Count += countHLSLFlatElements(BS.getType());
    }
    for (const Record::Field &F : R->fields()) {
      if (F.isUnnamedBitField())
        continue;
      Count += countHLSLFlatElements(F.Decl->getType());
    }
    return Count;
  }
  // Scalar primitive types contribute one element.
  if (canClassify(Ty))
    return 1;
  return 0;
}

/// Walk a source aggregate and extract every scalar element into its own local
/// variable. The results are appended to \p Elements in declaration order,
/// stopping once \p MaxElements have been collected. A pointer to the
/// source aggregate must be stored in the local at \p SrcOffset.
template <class Emitter>
bool Compiler<Emitter>::emitHLSLFlattenAggregate(
    QualType SrcType, unsigned SrcOffset,
    SmallVectorImpl<HLSLFlatElement> &Elements, unsigned MaxElements,
    const Expr *E) {

  // Save a scalar value from the stack into a new local and record it.
  auto saveToLocal = [&](PrimType T) -> bool {
    unsigned Offset = allocateLocalPrimitive(E, T, /*IsConst=*/true);
    if (!this->emitSetLocal(T, Offset, E))
      return false;
    Elements.push_back({Offset, T});
    return true;
  };

  // Save a pointer from the stack into a new local for later use.
  auto savePtrToLocal = [&]() -> UnsignedOrNone {
    unsigned Offset = allocateLocalPrimitive(E, PT_Ptr, /*IsConst=*/true);
    if (!this->emitSetLocal(PT_Ptr, Offset, E))
      return std::nullopt;
    return Offset;
  };

  // Vectors and matrices are flat sequences of elements.
  unsigned NumElems = 0;
  QualType ElemType;
  if (const auto *VT = SrcType->getAs<VectorType>()) {
    NumElems = VT->getNumElements();
    ElemType = VT->getElementType();
  } else if (const auto *MT = SrcType->getAs<ConstantMatrixType>()) {
    NumElems = MT->getNumElementsFlattened();
    ElemType = MT->getElementType();
  }
  if (NumElems > 0) {
    PrimType ElemT = classifyPrim(ElemType);
    for (unsigned I = 0; I != NumElems && Elements.size() < MaxElements; ++I) {
      if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
        return false;
      if (!this->emitArrayElemPop(ElemT, I, E))
        return false;
      if (!saveToLocal(ElemT))
        return false;
    }
    return true;
  }

  // Arrays: primitive elements are extracted directly; composite elements
  // require recursion into each sub-aggregate.
  if (const auto *AT = SrcType->getAsArrayTypeUnsafe()) {
    const auto *CAT = cast<ConstantArrayType>(AT);
    QualType ArrElemType = CAT->getElementType();
    unsigned ArrSize = CAT->getZExtSize();

    if (OptPrimType ElemT = classify(ArrElemType)) {
      for (unsigned I = 0; I != ArrSize && Elements.size() < MaxElements; ++I) {
        if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
          return false;
        if (!this->emitArrayElemPop(*ElemT, I, E))
          return false;
        if (!saveToLocal(*ElemT))
          return false;
      }
    } else {
      for (unsigned I = 0; I != ArrSize && Elements.size() < MaxElements; ++I) {
        if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
          return false;
        if (!this->emitConstUint32(I, E))
          return false;
        if (!this->emitArrayElemPtrPopUint32(E))
          return false;
        UnsignedOrNone ElemPtrOffset = savePtrToLocal();
        if (!ElemPtrOffset)
          return false;
        if (!emitHLSLFlattenAggregate(ArrElemType, *ElemPtrOffset, Elements,
                                      MaxElements, E))
          return false;
      }
    }
    return true;
  }

  // Records: base classes come first, then named fields in declaration
  // order.
  if (SrcType->isRecordType()) {
    const Record *R = getRecord(SrcType);
    if (!R)
      return false;

    if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(R->getDecl())) {
      for (const CXXBaseSpecifier &BS : CXXRD->bases()) {
        if (Elements.size() >= MaxElements)
          break;
        const Record::Base *B = R->getBase(BS.getType());
        assert(B);
        if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
          return false;
        if (!this->emitGetPtrBasePop(B->Offset, /*NullOK=*/false, E))
          return false;
        UnsignedOrNone BasePtrOffset = savePtrToLocal();
        if (!BasePtrOffset)
          return false;
        if (!emitHLSLFlattenAggregate(BS.getType(), *BasePtrOffset, Elements,
                                      MaxElements, E))
          return false;
      }
    }

    for (const Record::Field &F : R->fields()) {
      if (Elements.size() >= MaxElements)
        break;
      if (F.isUnnamedBitField())
        continue;

      QualType FieldType = F.Decl->getType();
      if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
        return false;
      if (!this->emitGetPtrFieldPop(F.Offset, E))
        return false;

      if (OptPrimType FieldT = F.T) {
        if (!this->emitLoadPop(*FieldT, E))
          return false;
        if (!saveToLocal(*FieldT))
          return false;
      } else {
        UnsignedOrNone FieldPtrOffset = savePtrToLocal();
        if (!FieldPtrOffset)
          return false;
        if (!emitHLSLFlattenAggregate(FieldType, *FieldPtrOffset, Elements,
                                      MaxElements, E))
          return false;
      }
    }
    return true;
  }

  return false;
}

/// Populate an HLSL aggregate from a flat list of previously extracted source
/// elements, casting each to the corresponding destination element type.
/// \p ElemIdx tracks the current position in \p Elements and is advanced as
/// elements are consumed. A pointer to the destination must be on top of the
/// interpreter stack.
template <class Emitter>
bool Compiler<Emitter>::emitHLSLConstructAggregate(
    QualType DestType, ArrayRef<HLSLFlatElement> Elements, unsigned &ElemIdx,
    const Expr *E) {

  // Consume the next source element, cast it, and leave it on the stack.
  auto loadAndCast = [&](PrimType DestT, QualType DestQT) -> bool {
    const auto &Src = Elements[ElemIdx++];
    if (!this->emitGetLocal(Src.Type, Src.LocalOffset, E))
      return false;
    return this->emitPrimCast(Src.Type, DestT, DestQT, E);
  };

  // Vectors and matrices are flat sequences of elements.
  unsigned NumElems = 0;
  QualType ElemType;
  if (const auto *VT = DestType->getAs<VectorType>()) {
    NumElems = VT->getNumElements();
    ElemType = VT->getElementType();
  } else if (const auto *MT = DestType->getAs<ConstantMatrixType>()) {
    NumElems = MT->getNumElementsFlattened();
    ElemType = MT->getElementType();
  }
  if (NumElems > 0) {
    PrimType DestElemT = classifyPrim(ElemType);
    for (unsigned I = 0; I != NumElems; ++I) {
      if (!loadAndCast(DestElemT, ElemType))
        return false;
      if (!this->emitInitElem(DestElemT, I, E))
        return false;
    }
    return true;
  }

  // Arrays: primitive elements are filled directly; composite elements
  // require recursion into each sub-aggregate.
  if (const auto *AT = DestType->getAsArrayTypeUnsafe()) {
    const auto *CAT = cast<ConstantArrayType>(AT);
    QualType ArrElemType = CAT->getElementType();
    unsigned ArrSize = CAT->getZExtSize();

    if (OptPrimType ElemT = classify(ArrElemType)) {
      for (unsigned I = 0; I != ArrSize; ++I) {
        if (!loadAndCast(*ElemT, ArrElemType))
          return false;
        if (!this->emitInitElem(*ElemT, I, E))
          return false;
      }
    } else {
      for (unsigned I = 0; I != ArrSize; ++I) {
        if (!this->emitConstUint32(I, E))
          return false;
        if (!this->emitArrayElemPtrUint32(E))
          return false;
        if (!emitHLSLConstructAggregate(ArrElemType, Elements, ElemIdx, E))
          return false;
        if (!this->emitFinishInitPop(E))
          return false;
      }
    }
    return true;
  }

  // Records: base classes come first, then named fields in declaration
  // order.
  if (DestType->isRecordType()) {
    const Record *R = getRecord(DestType);
    if (!R)
      return false;

    if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(R->getDecl())) {
      for (const CXXBaseSpecifier &BS : CXXRD->bases()) {
        const Record::Base *B = R->getBase(BS.getType());
        assert(B);
        if (!this->emitGetPtrBase(B->Offset, E))
          return false;
        if (!emitHLSLConstructAggregate(BS.getType(), Elements, ElemIdx, E))
          return false;
        if (!this->emitFinishInitPop(E))
          return false;
      }
    }

    for (const Record::Field &F : R->fields()) {
      if (F.isUnnamedBitField())
        continue;

      QualType FieldType = F.Decl->getType();
      if (OptPrimType FieldT = F.T) {
        if (!loadAndCast(*FieldT, FieldType))
          return false;
        if (F.isBitField()) {
          if (!this->emitInitBitField(*FieldT, F.Offset, F.bitWidth(), E))
            return false;
        } else {
          if (!this->emitInitField(*FieldT, F.Offset, E))
            return false;
        }
      } else {
        if (!this->emitGetPtrField(F.Offset, E))
          return false;
        if (!emitHLSLConstructAggregate(FieldType, Elements, ElemIdx, E))
          return false;
        if (!this->emitPopPtr(E))
          return false;
      }
    }
    return true;
  }

  return false;
}

namespace clang {
namespace interp {

template class Compiler<ByteCodeEmitter>;
template class Compiler<EvalEmitter>;

} // namespace interp
} // namespace clang