| //===- Interpreter.cpp - Interpreter Loop for llubi -----------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file implements the evaluation loop for each kind of instruction. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "Context.h" | |
| #include "ExecutorBase.h" | |
| #include "Library.h" | |
| #include "Value.h" | |
| #include "llvm/ADT/STLExtras.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| #include "llvm/Analysis/VectorUtils.h" | |
| #include "llvm/IR/BundleAttributes.h" | |
| #include "llvm/IR/InlineAsm.h" | |
| #include "llvm/IR/InstVisitor.h" | |
| #include "llvm/IR/Intrinsics.h" | |
| #include "llvm/IR/Operator.h" | |
| #include "llvm/IR/PatternMatch.h" | |
| #include "llvm/TargetParser/Triple.h" | |
| #include <cassert> | |
| #include <cstring> | |
| #include <limits> | |
| #include <list> | |
| namespace llvm::ubi { | |
| using namespace PatternMatch; | |
| /// Visit the scalar values recursively. The callback function may modify the | |
| /// value in-place. | |
| static void forEachScalarValue(AnyValue &V, | |
| function_ref<void(AnyValue &)> Visit) { | |
| if (V.isNone()) | |
| return; | |
| if (V.isAggregate()) { | |
| for (auto &SubValue : V.asAggregate()) | |
| forEachScalarValue(SubValue, Visit); | |
| return; | |
| } | |
| Visit(V); | |
| } | |
| static void applyRangeAttr(AnyValue &V, const ConstantRange &CR) { | |
| forEachScalarValue(V, [&](AnyValue &Scalar) { | |
| if (Scalar.isInteger() && !CR.contains(Scalar.asInteger())) | |
| Scalar = AnyValue::poison(); | |
| }); | |
| } | |
| static void applyNoFPClassAttr(AnyValue &V, FPClassTest NoFPClass) { | |
| forEachScalarValue(V, [NoFPClass](AnyValue &Scalar) { | |
| if (Scalar.isFloat() && (Scalar.asFloat().classify() & NoFPClass)) | |
| Scalar = AnyValue::poison(); | |
| }); | |
| } | |
| static void applyNonNullAttr(AnyValue &V, unsigned AS, const DataLayout &DL) { | |
| if (V.isPointer() && V.asPointer().isNullPtr(AS, DL)) | |
| V = AnyValue::poison(); | |
| } | |
| static void applyAlignAttr(AnyValue &V, Align Alignment) { | |
| forEachScalarValue(V, [Alignment](AnyValue &Scalar) { | |
| if (Scalar.isPointer() && | |
| Scalar.asPointer().address().countr_zero() < Log2(Alignment)) | |
| Scalar = AnyValue::poison(); | |
| }); | |
| } | |
| static bool violatesNoUndefAttr(const AnyValue &V) { | |
| bool ContainsPoison = false; | |
| forEachScalarValue(const_cast<AnyValue &>(V), [&](const AnyValue &Scalar) { | |
| if (Scalar.isPoison()) { | |
| ContainsPoison = true; | |
| return; | |
| } | |
| if (Scalar.isByte() && !ContainsPoison) { | |
| // For non-byte-sized values, padding bits are poison if and only if all | |
| // non-padding bits in the padding byte are poison. | |
| ContainsPoison = any_of(Scalar.asByte().bytes(), [](const Byte &V) { | |
| return V.ConcreteMask != 255; | |
| }); | |
| } | |
| }); | |
| return ContainsPoison; | |
| } | |
| /// Assumes V is either a poison or a pointer. | |
| static bool violatesDereferenceableBytesAttr(const AnyValue &V, uint64_t Bytes, | |
| bool OrNull, unsigned AS, | |
| Context &Ctx) { | |
| if (V.isPoison()) | |
| return true; | |
| auto &Ptr = V.asPointer(); | |
| if (Ptr.isNullPtr(AS, Ctx.getDataLayout())) { | |
| if (OrNull) | |
| return false; | |
| return true; | |
| } | |
| auto *MO = Ctx.checkProvenance(Ptr, [&](const Provenance &) { | |
| // TODO: check read_provenance | |
| // TODO: check nofree for attributes/metadata. | |
| return true; | |
| }); | |
| if (!MO) | |
| return true; | |
| if (MO->getState() == MemoryObjectState::Freed) | |
| return true; | |
| const APInt &PtrAddr = Ptr.address(); | |
| return Bytes > MO->getSize() || PtrAddr.ult(MO->getAddress()) || | |
| PtrAddr.ugt(MO->getAddress() + MO->getSize() - Bytes); | |
| } | |
| /// Instruction executor using the visitor pattern. | |
| /// Unlike the Context class that manages the global state, | |
| /// InstExecutor only maintains the state for call frames. | |
| class InstExecutor : public InstVisitor<InstExecutor, void>, | |
| public ExecutorBase { | |
| const DataLayout &DL; | |
| std::list<Frame> CallStack; | |
| AnyValue None; | |
| std::list<AnyValue> UnsupportedConstantValues; | |
| Library Lib; | |
| const AnyValue &getValue(Value *V) { | |
| if (auto *C = dyn_cast<Constant>(V)) { | |
| if (const AnyValue *Val = Ctx.getConstantValue(C)) | |
| return *Val; | |
| reportError() << "Unsupported constant: " << *C << "."; | |
| UnsupportedConstantValues.push_back( | |
| AnyValue::getPoisonValue(Ctx, C->getType())); | |
| return UnsupportedConstantValues.back(); | |
| } | |
| if (isa<MetadataAsValue>(V)) | |
| return None; | |
| return CurrentFrame->ValueMap.at(V); | |
| } | |
| void setResult(Instruction &I, AnyValue V) { | |
| if (!hasProgramExited() && !Handler.onInstructionExecuted(I, V)) | |
| setFailed(); | |
| if (hasProgramExited()) | |
| return; | |
| assert(V.isCompatibleWith(I.getType()) && "Unexpected value storage kind."); | |
| if (!V.isNone()) | |
| CurrentFrame->ValueMap.insert_or_assign(&I, std::move(V)); | |
| } | |
| APFloat handleDenormal(APFloat Val, DenormalMode::DenormalModeKind Mode, | |
| bool IsInput) { | |
| if (!Val.isDenormal()) | |
| return Val; | |
| if (IsInput) { | |
| // Non-deterministically choose between flushing or preserving the | |
| // denormal value. | |
| if (Ctx.getRandomBool()) | |
| return Val; | |
| } | |
| if (Mode == DenormalMode::PositiveZero) | |
| return APFloat::getZero(Val.getSemantics(), false); | |
| if (Mode == DenormalMode::PreserveSign) | |
| return APFloat::getZero(Val.getSemantics(), Val.isNegative()); | |
| // Default case for IEEE, Dynamic, and Invalid | |
| // Currently we treat Dynamic the same as IEEE, since we don't support | |
| // changing the mode at this point. | |
| return Val; | |
| } | |
| AnyValue handleFMFFlags(AnyValue Val, FastMathFlags FMF, bool IsInput) { | |
| if (Val.isPoison()) | |
| return AnyValue::poison(); | |
| if (Val.isAggregate()) { | |
| std::vector<AnyValue> ResVec; | |
| ResVec.reserve(Val.asAggregate().size()); | |
| for (const auto &A : Val.asAggregate()) | |
| ResVec.push_back(handleFMFFlags(A, FMF, IsInput)); | |
| return AnyValue(ResVec); | |
| } | |
| const APFloat &APVal = Val.asFloat(); | |
| if (FMF.noNaNs() && APVal.isNaN()) | |
| return AnyValue::poison(); | |
| if (FMF.noInfs() && APVal.isInfinity()) | |
| return AnyValue::poison(); | |
| if (IsInput && FMF.noSignedZeros() && APVal.isZero()) | |
| return AnyValue(APFloat::getZero( | |
| APVal.getSemantics(), APVal.isNegative() ^ Ctx.getRandomBool())); | |
| return Val; | |
| } | |
| void addNaNCandidate(SmallVectorImpl<APFloat> &Candidates, | |
| APFloat Candidate) { | |
| if (any_of(Candidates, [&](const APFloat &Existing) { | |
| return Existing.bitwiseIsEqual(Candidate); | |
| })) | |
| return; | |
| Candidates.push_back(std::move(Candidate)); | |
| } | |
| APFloat pickNaNCandidate(ArrayRef<APFloat> Candidates) { | |
| assert(!Candidates.empty() && "Need at least one NaN candidate."); | |
| return Candidates[Ctx.getRandomUInt64() % Candidates.size()]; | |
| } | |
| APInt getRandomNaNPayload(const fltSemantics &Sem) { | |
| const unsigned NumBits = Sem.precision - 1; | |
| SmallVector<APInt::WordType, 2> RandomWords; | |
| const unsigned NumWords = APInt::getNumWords(NumBits); | |
| RandomWords.reserve(NumWords); | |
| for (unsigned I = 0; I != NumWords; ++I) | |
| RandomWords.push_back(Ctx.getRandomUInt64()); | |
| return APInt(NumBits, RandomWords); | |
| } | |
| bool isPreferredNaN(const APFloat &Val) { | |
| assert(Val.isNaN() && "Expected NaN."); | |
| const APFloat Preferred = | |
| APFloat::getQNaN(Val.getSemantics(), Val.isNegative()); | |
| return Val.bitwiseIsEqual(Preferred); | |
| } | |
| bool wasmMayProduceExtraNaNPayload(ArrayRef<const APFloat *> Inputs) { | |
| for (const APFloat *Input : Inputs) { | |
| if (!Input->isNaN()) | |
| continue; | |
| if (Input->isSignaling() || !isPreferredNaN(*Input)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| APFloat propagateInputNaN(const APFloat &InputNaN, const fltSemantics &DstSem, | |
| bool QuietingMode, bool FlipSign) { | |
| APFloat Res = InputNaN; | |
| bool LosesInfo; | |
| Res.convert(DstSem, APFloat::rmNearestTiesToEven, &LosesInfo); | |
| if (FlipSign) | |
| Res.changeSign(); | |
| if (QuietingMode && Res.isSignaling()) | |
| Res = Res.makeQuiet(); | |
| return Res; | |
| } | |
| APFloat maybeQuietSNaN(APFloat Val) const { | |
| if (Val.isSignaling() && Ctx.getRandomBool()) | |
| return Val.makeQuiet(); | |
| return Val; | |
| } | |
| APFloat maxnumWithSNaNQuieting(const APFloat &LHS, const APFloat &RHS) { | |
| return maxnum(maybeQuietSNaN(LHS), maybeQuietSNaN(RHS)); | |
| } | |
| APFloat minnumWithSNaNQuieting(const APFloat &LHS, const APFloat &RHS) { | |
| return minnum(maybeQuietSNaN(LHS), maybeQuietSNaN(RHS)); | |
| } | |
| void addPropagatedNaNCandidates(SmallVectorImpl<APFloat> &Candidates, | |
| ArrayRef<const APFloat *> Inputs, | |
| const fltSemantics &DstSem, bool QuietingMode, | |
| bool SignChoice) { | |
| for (const APFloat *Input : Inputs) { | |
| if (!Input->isNaN()) | |
| continue; | |
| addNaNCandidate(Candidates, propagateInputNaN(*Input, DstSem, | |
| QuietingMode, SignChoice)); | |
| } | |
| } | |
| void addTargetSpecificNaNCandidates(SmallVectorImpl<APFloat> &Candidates, | |
| const APFloat &Result, | |
| ArrayRef<const APFloat *> Inputs, | |
| bool SignChoice) { | |
| const Triple &TT = Ctx.getTargetTriple(); | |
| if (TT.isWasm()) { | |
| if (!wasmMayProduceExtraNaNPayload(Inputs)) | |
| return; | |
| APInt Payload = getRandomNaNPayload(Result.getSemantics()); | |
| addNaNCandidate(Candidates, APFloat::getQNaN(Result.getSemantics(), | |
| SignChoice, &Payload)); | |
| return; | |
| } | |
| if (TT.isSPARC32() || TT.isSPARC64()) { | |
| APInt Payload = APInt::getAllOnes(Result.getSemantics().precision - 1); | |
| addNaNCandidate(Candidates, APFloat::getQNaN(Result.getSemantics(), | |
| SignChoice, &Payload)); | |
| } | |
| } | |
| APFloat applyNaNPropagation(const APFloat &Result, | |
| ArrayRef<const APFloat *> Inputs) { | |
| if (!Result.isNaN()) | |
| return Result; | |
| const NaNPropagationBehavior Choice = | |
| Ctx.getEffectiveNaNPropagationBehavior(); | |
| const bool SignChoice = Ctx.getRandomBool(); | |
| const fltSemantics &ResultSem = Result.getSemantics(); | |
| auto PreferredNaN = [&]() { | |
| return APFloat::getQNaN(ResultSem, SignChoice); | |
| }; | |
| SmallVector<APFloat, 4> Candidates; | |
| switch (Choice) { | |
| case NaNPropagationBehavior::PreferredNaN: | |
| return PreferredNaN(); | |
| case NaNPropagationBehavior::QuietingNaN: | |
| addPropagatedNaNCandidates(Candidates, Inputs, ResultSem, | |
| /*QuietingMode=*/true, SignChoice); | |
| return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates); | |
| case NaNPropagationBehavior::UnchangedNaN: | |
| addPropagatedNaNCandidates(Candidates, Inputs, ResultSem, | |
| /*QuietingMode=*/false, SignChoice); | |
| return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates); | |
| case NaNPropagationBehavior::TargetSpecificNaN: | |
| addTargetSpecificNaNCandidates(Candidates, Result, Inputs, SignChoice); | |
| return Candidates.empty() ? PreferredNaN() : pickNaNCandidate(Candidates); | |
| case NaNPropagationBehavior::NonDeterministic: | |
| addNaNCandidate(Candidates, PreferredNaN()); | |
| addPropagatedNaNCandidates(Candidates, Inputs, ResultSem, | |
| /*QuietingMode=*/true, SignChoice); | |
| addPropagatedNaNCandidates(Candidates, Inputs, ResultSem, | |
| /*QuietingMode=*/false, SignChoice); | |
| addTargetSpecificNaNCandidates(Candidates, Result, Inputs, SignChoice); | |
| return pickNaNCandidate(Candidates); | |
| } | |
| llvm_unreachable("Unhandled NaN propagation behavior."); | |
| } | |
| AnyValue computeUnOp(Type *Ty, const AnyValue &Operand, | |
| function_ref<AnyValue(const AnyValue &)> ScalarFn) { | |
| if (Ty->isVectorTy()) { | |
| auto &OperandVec = Operand.asAggregate(); | |
| std::vector<AnyValue> ResVec; | |
| ResVec.reserve(OperandVec.size()); | |
| for (const auto &Scalar : OperandVec) | |
| ResVec.push_back(ScalarFn(Scalar)); | |
| return std::move(ResVec); | |
| } | |
| return ScalarFn(Operand); | |
| } | |
| void visitUnOp(Instruction &I, | |
| function_ref<AnyValue(const AnyValue &)> ScalarFn) { | |
| setResult(I, computeUnOp(I.getType(), getValue(I.getOperand(0)), ScalarFn)); | |
| } | |
| void visitIntUnOp(Instruction &I, | |
| function_ref<AnyValue(const APInt &)> ScalarFn) { | |
| visitUnOp(I, [&](const AnyValue &Operand) -> AnyValue { | |
| if (Operand.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(Operand.asInteger()); | |
| }); | |
| } | |
| void visitBitwiseFPUnOp(Instruction &I, | |
| function_ref<APFloat(const APFloat &)> ScalarFn) { | |
| setResult(I, visitBitwiseFPUnOpWithResult( | |
| I.getType(), cast<FPMathOperator>(I).getFastMathFlags(), | |
| getValue(I.getOperand(0)), ScalarFn)); | |
| } | |
| AnyValue | |
| visitIntUnOpWithResult(Type *RetTy, const AnyValue &Operand, | |
| function_ref<AnyValue(const APInt &)> ScalarFn) { | |
| return computeUnOp(RetTy, Operand, | |
| [&](const AnyValue &OperandInner) -> AnyValue { | |
| if (OperandInner.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(OperandInner.asInteger()); | |
| }); | |
| } | |
| AnyValue visitBitwiseFPUnOpWithResult( | |
| Type *RetTy, const FastMathFlags &FMF, const AnyValue &Operand, | |
| function_ref<APFloat(const APFloat &)> ScalarFn) { | |
| return computeUnOp( | |
| RetTy, Operand, [&](const AnyValue &OperandInner) -> AnyValue { | |
| if (OperandInner.isPoison()) | |
| return AnyValue::poison(); | |
| // We don't flush denormals here since bitwise floating-point | |
| // operations only manipulate on certain bits of the operand. | |
| AnyValue ValidatedOperand = | |
| handleFMFFlags(OperandInner, FMF, /*IsInput=*/true); | |
| if (ValidatedOperand.isPoison()) | |
| return ValidatedOperand; | |
| APFloat Result = ScalarFn(ValidatedOperand.asFloat()); | |
| return handleFMFFlags(Result, FMF, /*IsInput=*/false); | |
| }); | |
| } | |
| AnyValue computeBinOp( | |
| Type *Ty, const AnyValue &LHS, const AnyValue &RHS, | |
| function_ref<AnyValue(const AnyValue &, const AnyValue &)> ScalarFn) { | |
| if (Ty->isVectorTy()) { | |
| auto &LHSVec = LHS.asAggregate(); | |
| auto &RHSVec = RHS.asAggregate(); | |
| std::vector<AnyValue> ResVec; | |
| ResVec.reserve(LHSVec.size()); | |
| for (const auto &[ScalarLHS, ScalarRHS] : zip(LHSVec, RHSVec)) | |
| ResVec.push_back(ScalarFn(ScalarLHS, ScalarRHS)); | |
| return std::move(ResVec); | |
| } | |
| return ScalarFn(LHS, RHS); | |
| } | |
| void visitBinOp( | |
| Instruction &I, | |
| function_ref<AnyValue(const AnyValue &, const AnyValue &)> ScalarFn) { | |
| setResult(I, computeBinOp(I.getType(), getValue(I.getOperand(0)), | |
| getValue(I.getOperand(1)), ScalarFn)); | |
| } | |
| void | |
| visitIntBinOp(Instruction &I, | |
| function_ref<AnyValue(const APInt &, const APInt &)> ScalarFn) { | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue { | |
| if (LHS.isPoison() || RHS.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(LHS.asInteger(), RHS.asInteger()); | |
| }); | |
| } | |
| void visitFPBinOp( | |
| Instruction &I, | |
| function_ref<APFloat(const APFloat &, const APFloat &)> ScalarFn) { | |
| setResult(I, visitFPBinOpWithResult( | |
| I.getType(), cast<FPMathOperator>(I).getFastMathFlags(), | |
| getValue(I.getOperand(0)), getValue(I.getOperand(1)), | |
| ScalarFn)); | |
| } | |
| AnyValue visitIntBinOpWithResult( | |
| Type *RetTy, const AnyValue &LHS, const AnyValue &RHS, | |
| function_ref<AnyValue(const APInt &, const APInt &)> ScalarFn) { | |
| return computeBinOp( | |
| RetTy, LHS, RHS, | |
| [&](const AnyValue &LHSInner, const AnyValue &RHSInner) -> AnyValue { | |
| if (LHSInner.isPoison() || RHSInner.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(LHSInner.asInteger(), RHSInner.asInteger()); | |
| }); | |
| } | |
| AnyValue visitOverflowIntBinOpWithResult( | |
| Type *RetTy, const AnyValue &LHS, const AnyValue &RHS, | |
| function_ref<std::pair<APInt, bool>(const APInt &, const APInt &)> | |
| ScalarFn) { | |
| if (!LHS.isAggregate()) { | |
| if (LHS.isPoison() || RHS.isPoison()) | |
| return std::vector<AnyValue>{AnyValue::poison(), AnyValue::poison()}; | |
| auto [Res, Overflow] = ScalarFn(LHS.asInteger(), RHS.asInteger()); | |
| return std::vector<AnyValue>{AnyValue(Res), AnyValue::boolean(Overflow)}; | |
| } | |
| auto &LHSVec = LHS.asAggregate(); | |
| auto &RHSVec = RHS.asAggregate(); | |
| std::vector<AnyValue> ResVec; | |
| std::vector<AnyValue> OverflowVec; | |
| ResVec.reserve(LHSVec.size()); | |
| OverflowVec.reserve(LHSVec.size()); | |
| for (const auto &[ScalarLHS, ScalarRHS] : zip(LHSVec, RHSVec)) { | |
| if (ScalarLHS.isPoison() || ScalarRHS.isPoison()) { | |
| ResVec.push_back(AnyValue::poison()); | |
| OverflowVec.push_back(AnyValue::poison()); | |
| continue; | |
| } | |
| auto [Res, Overflow] = | |
| ScalarFn(ScalarLHS.asInteger(), ScalarRHS.asInteger()); | |
| ResVec.push_back(AnyValue(Res)); | |
| OverflowVec.push_back(AnyValue::boolean(Overflow)); | |
| } | |
| return std::vector<AnyValue>{AnyValue(std::move(ResVec)), | |
| AnyValue(std::move(OverflowVec))}; | |
| } | |
| AnyValue visitFPBinOpWithResult( | |
| Type *RetTy, const FastMathFlags &FMF, const AnyValue &LHS, | |
| const AnyValue &RHS, | |
| function_ref<APFloat(const APFloat &, const APFloat &)> ScalarFn) { | |
| DenormalMode DenormMode = getCurrentDenormalMode(RetTy); | |
| if (!Ctx.isDefaultFPEnv()) | |
| reportImmediateUB() << "Non-constrained floating-point operation assumes " | |
| "default floating-point environment"; | |
| return computeBinOp( | |
| RetTy, LHS, RHS, | |
| [&](const AnyValue &LHSInner, const AnyValue &RHSInner) -> AnyValue { | |
| if (LHSInner.isPoison() || RHSInner.isPoison()) | |
| return AnyValue::poison(); | |
| AnyValue ValidatedLHS = | |
| handleFMFFlags(LHSInner, FMF, /*IsInput=*/true); | |
| AnyValue ValidatedRHS = | |
| handleFMFFlags(RHSInner, FMF, /*IsInput=*/true); | |
| if (ValidatedLHS.isPoison()) | |
| return ValidatedLHS; | |
| if (ValidatedRHS.isPoison()) | |
| return ValidatedRHS; | |
| // Flush input denormals | |
| APFloat FLHS = handleDenormal(ValidatedLHS.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat FRHS = handleDenormal(ValidatedRHS.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat RawResult = ScalarFn(FLHS, FRHS); | |
| // Flush output denormals and handle fast-math flags. | |
| AnyValue FResult = handleFMFFlags( | |
| handleDenormal(RawResult, DenormMode.Output, /*IsInput=*/false), | |
| FMF, | |
| /*IsInput=*/false); | |
| if (FResult.isPoison()) | |
| return FResult; | |
| APFloat Result = FResult.asFloat(); | |
| return applyNaNPropagation(Result, {&FLHS, &FRHS}); | |
| }); | |
| } | |
| AnyValue | |
| computeTriOp(Type *Ty, const AnyValue &Op1, const AnyValue &Op2, | |
| const AnyValue &Op3, | |
| function_ref<AnyValue(const AnyValue &, const AnyValue &, | |
| const AnyValue &)> | |
| ScalarFn) { | |
| if (Ty->isVectorTy()) { | |
| auto &Op1Vec = Op1.asAggregate(); | |
| auto &Op2Vec = Op2.asAggregate(); | |
| auto &Op3Vec = Op3.asAggregate(); | |
| std::vector<AnyValue> ResVec; | |
| ResVec.reserve(Op1Vec.size()); | |
| for (const auto &[ScalarOp1, ScalarOp2, ScalarOp3] : | |
| zip(Op1Vec, Op2Vec, Op3Vec)) | |
| ResVec.push_back(ScalarFn(ScalarOp1, ScalarOp2, ScalarOp3)); | |
| return std::move(ResVec); | |
| } | |
| return ScalarFn(Op1, Op2, Op3); | |
| } | |
| void visitTriOp(Instruction &I, | |
| function_ref<AnyValue(const AnyValue &, const AnyValue &, | |
| const AnyValue &)> | |
| ScalarFn) { | |
| setResult(I, computeTriOp(I.getType(), getValue(I.getOperand(0)), | |
| getValue(I.getOperand(1)), | |
| getValue(I.getOperand(2)), ScalarFn)); | |
| } | |
| void visitIntTriOp( | |
| Instruction &I, | |
| function_ref<AnyValue(const APInt &, const APInt &, const APInt &)> | |
| ScalarFn) { | |
| visitTriOp(I, | |
| [&](const AnyValue &Op1, const AnyValue &Op2, | |
| const AnyValue &Op3) -> AnyValue { | |
| if (Op1.isPoison() || Op2.isPoison() || Op3.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(Op1.asInteger(), Op2.asInteger(), | |
| Op3.asInteger()); | |
| }); | |
| } | |
| AnyValue visitIntTriOpWithResult( | |
| Type *RetTy, const AnyValue &Op1, const AnyValue &Op2, | |
| const AnyValue &Op3, | |
| function_ref<AnyValue(const APInt &, const APInt &, const APInt &)> | |
| ScalarFn) { | |
| return computeTriOp( | |
| RetTy, Op1, Op2, Op3, | |
| [&](const AnyValue &Op1Inner, const AnyValue &Op2Inner, | |
| const AnyValue &Op3Inner) -> AnyValue { | |
| if (Op1Inner.isPoison() || Op2Inner.isPoison() || Op3Inner.isPoison()) | |
| return AnyValue::poison(); | |
| return ScalarFn(Op1Inner.asInteger(), Op2Inner.asInteger(), | |
| Op3Inner.asInteger()); | |
| }); | |
| } | |
| AnyValue visitFPTriOpWithResult( | |
| Type *RetTy, const FastMathFlags &FMF, const AnyValue &Op1, | |
| const AnyValue &Op2, const AnyValue &Op3, | |
| function_ref<APFloat(const APFloat &, const APFloat &, const APFloat &)> | |
| ScalarFn) { | |
| DenormalMode DenormMode = getCurrentDenormalMode(RetTy); | |
| if (!Ctx.isDefaultFPEnv()) | |
| reportImmediateUB() << "Non-constrained floating-point operation assumes " | |
| "default floating-point environment"; | |
| return computeTriOp( | |
| RetTy, Op1, Op2, Op3, | |
| [&](const AnyValue &Op1Inner, const AnyValue &Op2Inner, | |
| const AnyValue &Op3Inner) -> AnyValue { | |
| if (Op1Inner.isPoison() || Op2Inner.isPoison() || Op3Inner.isPoison()) | |
| return AnyValue::poison(); | |
| AnyValue ValidatedOp1 = | |
| handleFMFFlags(Op1Inner, FMF, /*IsInput=*/true); | |
| AnyValue ValidatedOp2 = | |
| handleFMFFlags(Op2Inner, FMF, /*IsInput=*/true); | |
| AnyValue ValidatedOp3 = | |
| handleFMFFlags(Op3Inner, FMF, /*IsInput=*/true); | |
| if (ValidatedOp1.isPoison()) | |
| return ValidatedOp1; | |
| if (ValidatedOp2.isPoison()) | |
| return ValidatedOp2; | |
| if (ValidatedOp3.isPoison()) | |
| return ValidatedOp3; | |
| // Flush input denormals | |
| APFloat FOp1 = handleDenormal(ValidatedOp1.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat FOp2 = handleDenormal(ValidatedOp2.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat FOp3 = handleDenormal(ValidatedOp3.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat RawResult = ScalarFn(FOp1, FOp2, FOp3); | |
| // Flush output denormals and handle fast-math flags. | |
| AnyValue FResult = handleFMFFlags( | |
| handleDenormal(RawResult, DenormMode.Output, /*IsInput=*/false), | |
| FMF, | |
| /*IsInput=*/false); | |
| if (FResult.isPoison()) | |
| return FResult; | |
| APFloat Result = FResult.asFloat(); | |
| return applyNaNPropagation(Result, {&FOp1, &FOp2, &FOp3}); | |
| }); | |
| } | |
| void jumpTo(Instruction &Terminator, BasicBlock *DestBB) { | |
| if (!Handler.onBBJump(Terminator, *DestBB)) { | |
| setFailed(); | |
| return; | |
| } | |
| BasicBlock *From = CurrentFrame->BB; | |
| CurrentFrame->BB = DestBB; | |
| CurrentFrame->PC = DestBB->begin(); | |
| // Update PHI nodes in batch to avoid the interference between PHI nodes. | |
| // We need to store the incoming values into a temporary buffer. | |
| // Otherwise, the incoming value may be overwritten before it is | |
| // used by other PHI nodes. | |
| SmallVector<std::pair<PHINode *, AnyValue>> IncomingValues; | |
| PHINode *PHI = nullptr; | |
| while ((PHI = dyn_cast<PHINode>(CurrentFrame->PC))) { | |
| AnyValue IncomingVal = getValue(PHI->getIncomingValueForBlock(From)); | |
| // Fast-math flags validation | |
| if (isa<FPMathOperator>(PHI)) { | |
| FastMathFlags FMF = PHI->getFastMathFlags(); | |
| if (FMF.any()) | |
| IncomingVal = | |
| handleFMFFlags(std::move(IncomingVal), FMF, /*IsInput=*/true); | |
| } | |
| handleMetadata(PHI->getType(), IncomingVal, *PHI); | |
| if (hasProgramExited()) | |
| return; | |
| IncomingValues.emplace_back(PHI, IncomingVal); | |
| ++CurrentFrame->PC; | |
| } | |
| for (auto &[K, V] : IncomingValues) | |
| setResult(*K, std::move(V)); | |
| } | |
| /// Helper function to determine whether an inline asm is a no-op, which is | |
| /// used to implement black_box style optimization blockers. | |
| bool isNoopInlineAsm(Value *V, Type *RetTy) { | |
| if (auto *Asm = dyn_cast<InlineAsm>(V)) | |
| return Asm->getAsmString().empty() && RetTy->isVoidTy(); | |
| return false; | |
| } | |
| DenormalMode getCurrentDenormalMode(Type *Ty) { | |
| return CurrentFrame->Func.getDenormalMode( | |
| Ty->getScalarType()->getFltSemantics()); | |
| } | |
| // Helper function to convert BooleanKind to bool. Report an immediate UB if | |
| // a poison is found. | |
| bool getBooleanNonPoison(BooleanKind Boolean) { | |
| if (Boolean == BooleanKind::Poison) | |
| reportImmediateUB() << "Unexpected poison boolean value"; | |
| return Boolean == BooleanKind::True; | |
| } | |
| APInt getIntNonPoison(const AnyValue &V) { | |
| if (V.isPoison()) { | |
| reportImmediateUB() << "Unexpected poison integer value."; | |
| return APInt::getZero(64); | |
| } | |
| return V.asInteger(); | |
| } | |
| AnyValue callMemTransferIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args, | |
| Intrinsic::ID IID) { | |
| const AnyValue &Dest = Args[0]; | |
| const AnyValue &Src = Args[1]; | |
| const AnyValue &Length = Args[2]; | |
| // TODO: Handle isvolatile argument. | |
| if (Length.isPoison()) { | |
| reportImmediateUB() << "Memory transfer intrinsic with poison length."; | |
| return AnyValue(); | |
| } | |
| const APInt &LengthInt = Args[2].asInteger(); | |
| if (LengthInt.getActiveBits() > 64) { | |
| reportImmediateUB() | |
| << "Memory transfer intrinsic length overflows uint64_t."; | |
| return AnyValue(); | |
| } | |
| const uint64_t Len = LengthInt.getZExtValue(); | |
| if (Len == 0) | |
| return AnyValue(); | |
| if (Dest.isPoison()) { | |
| reportImmediateUB() | |
| << "Memory transfer intrinsic with poison destination pointer."; | |
| return AnyValue(); | |
| } | |
| if (Src.isPoison()) { | |
| reportImmediateUB() | |
| << "Memory transfer intrinsic with poison source pointer."; | |
| return AnyValue(); | |
| } | |
| const Pointer &DstPtr = Dest.asPointer(); | |
| const Pointer &SrcPtr = Src.asPointer(); | |
| Align DstAlign = CB.getParamAlign(0).valueOrOne(); | |
| Align SrcAlign = CB.getParamAlign(1).valueOrOne(); | |
| auto [SrcMO, SrcOffset] = | |
| verifyMemAccess(SrcPtr, Len, SrcAlign, /*IsStore=*/false); | |
| if (!SrcMO) | |
| return AnyValue(); | |
| auto [DstMO, DstOffset] = | |
| verifyMemAccess(DstPtr, Len, DstAlign, /*IsStore=*/true); | |
| if (!DstMO) | |
| return AnyValue(); | |
| if (IID == Intrinsic::memcpy || IID == Intrinsic::memcpy_inline) { | |
| if (SrcMO == DstMO && SrcOffset != DstOffset) { | |
| const uint64_t SrcEnd = SrcOffset + Len; | |
| const uint64_t DstEnd = DstOffset + Len; | |
| if (SrcOffset < DstEnd && DstOffset < SrcEnd) { | |
| reportImmediateUB() | |
| << "memcpy with overlapping source and destination."; | |
| return AnyValue(); | |
| } | |
| } | |
| } | |
| MutableArrayRef<Byte> DstBytes = DstMO->getBytes().slice(DstOffset, Len); | |
| ArrayRef<Byte> SrcBytes = SrcMO->getBytes().slice(SrcOffset, Len); | |
| std::memmove(DstBytes.data(), SrcBytes.data(), Len * sizeof(Byte)); | |
| return AnyValue(); | |
| } | |
| AnyValue callMemSetIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args) { | |
| const AnyValue &Dest = Args[0]; | |
| const AnyValue &Val = Args[1]; | |
| const AnyValue &Length = Args[2]; | |
| if (Length.isPoison()) { | |
| reportImmediateUB() << "memset called with poison length."; | |
| return AnyValue(); | |
| } | |
| const APInt &LengthInt = Length.asInteger(); | |
| if (LengthInt.getActiveBits() > 64) { | |
| reportImmediateUB() << "memset called with length overflows uint64_t."; | |
| return AnyValue(); | |
| } | |
| const uint64_t Len = LengthInt.getZExtValue(); | |
| if (Len == 0) | |
| return AnyValue(); | |
| if (Dest.isPoison()) { | |
| reportImmediateUB() << "memset called with poison destination pointer."; | |
| return AnyValue(); | |
| } | |
| const Pointer &DstPtr = Dest.asPointer(); | |
| Align DstAlign = CB.getParamAlign(0).valueOrOne(); | |
| auto [DstMO, DstOffset] = | |
| verifyMemAccess(DstPtr, Len, DstAlign, /*IsStore=*/true); | |
| if (!DstMO) | |
| return AnyValue(); | |
| Byte FillByte = Val.isPoison() | |
| ? Byte::poison() | |
| : Byte::concrete(Val.asInteger().getZExtValue()); | |
| fill(DstMO->getBytes().slice(DstOffset, Len), FillByte); | |
| return AnyValue(); | |
| } | |
| static BooleanKind getMaskLane(const AnyValue &Mask, size_t I) { | |
| return Mask.asAggregate()[I].asBoolean(); | |
| } | |
| AnyValue callExperimentalVectorHistogramIntrinsic(CallBase &CB, | |
| ArrayRef<AnyValue> Args, | |
| Intrinsic::ID IID) { | |
| struct LaneUpdate { | |
| MemoryObject *MO; | |
| uint64_t Offset; | |
| uint64_t Count; | |
| AnyValue Old; | |
| AnyValue New; | |
| }; | |
| const auto &Ptrs = Args[0].asAggregate(); | |
| const AnyValue &Update = Args[1]; | |
| const AnyValue &Mask = Args[2]; | |
| Type *ElemTy = CB.getArgOperand(1)->getType(); | |
| const uint64_t AccessSize = Ctx.getEffectiveTypeStoreSize(ElemTy); | |
| SmallVector<LaneUpdate, 8> Lanes; | |
| Lanes.reserve(Ptrs.size()); | |
| for (size_t I = 0, E = Ptrs.size(); I != E; ++I) { | |
| switch (getMaskLane(Mask, I)) { | |
| case BooleanKind::False: | |
| continue; | |
| case BooleanKind::Poison: | |
| reportImmediateUB() | |
| << "Poison mask lane in experimental vector histogram intrinsic."; | |
| return AnyValue(); | |
| case BooleanKind::True: | |
| break; | |
| } | |
| if (Ptrs[I].isPoison()) { | |
| reportImmediateUB() << "Poison pointer lane in experimental vector " | |
| "histogram intrinsic."; | |
| return AnyValue(); | |
| } | |
| auto [MO, Offset] = | |
| verifyMemAccess(Ptrs[I].asPointer(), AccessSize, Align(1), | |
| /*IsStore=*/true); | |
| if (!MO) | |
| return AnyValue(); | |
| Lanes.push_back({MO, Offset, 0, AnyValue(), AnyValue()}); | |
| } | |
| for (LaneUpdate &Lane : Lanes) { | |
| Lane.Count = count_if(Lanes, [&](const LaneUpdate &Other) { | |
| return Other.MO == Lane.MO && Other.Offset == Lane.Offset; | |
| }); | |
| Lane.Old = Ctx.load(*Lane.MO, Lane.Offset, ElemTy); | |
| } | |
| for (LaneUpdate &Lane : Lanes) { | |
| const AnyValue &Old = Lane.Old; | |
| AnyValue &New = Lane.New; | |
| if (Old.isPoison() || Update.isPoison()) { | |
| New = AnyValue::poison(); | |
| } else { | |
| const APInt &OldInt = Old.asInteger(); | |
| const APInt &UpdateInt = Update.asInteger(); | |
| switch (IID) { | |
| case Intrinsic::experimental_vector_histogram_add: | |
| New = OldInt + UpdateInt * APInt(UpdateInt.getBitWidth(), Lane.Count, | |
| /*isSigned=*/false, | |
| /*implicitTrunc=*/true); | |
| break; | |
| case Intrinsic::experimental_vector_histogram_uadd_sat: { | |
| APInt Acc = OldInt; | |
| for (uint64_t I = 0; I != Lane.Count; ++I) | |
| Acc = Acc.uadd_sat(UpdateInt); | |
| New = Acc; | |
| break; | |
| } | |
| case Intrinsic::experimental_vector_histogram_umax: | |
| New = APIntOps::umax(OldInt, UpdateInt); | |
| break; | |
| case Intrinsic::experimental_vector_histogram_umin: | |
| New = APIntOps::umin(OldInt, UpdateInt); | |
| break; | |
| default: | |
| llvm_unreachable("Unexpected histogram intrinsic ID"); | |
| } | |
| } | |
| } | |
| for (const LaneUpdate &Lane : Lanes) | |
| Ctx.store(*Lane.MO, Lane.Offset, Lane.New, ElemTy); | |
| return AnyValue(); | |
| } | |
| /// Returns the oracle function if \p CB is an llvm.speculative.load in | |
| /// oracle form, nullptr otherwise. | |
| static Function *getSpeculativeLoadOracle(const CallBase &CB) { | |
| return CB.getIntrinsicID() == Intrinsic::speculative_load | |
| ? dyn_cast<Function>(CB.getArgOperand(2)) | |
| : nullptr; | |
| } | |
| AnyValue callSpeculativeLoadIntrinsic(CallBase &CB, const AnyValue &Ptr, | |
| const AnyValue &NumBytes) { | |
| Type *RetTy = CB.getType(); | |
| if (Ptr.isPoison()) { | |
| reportImmediateUB() << "llvm.speculative.load with poison pointer."; | |
| return AnyValue(); | |
| } | |
| if (NumBytes.isPoison()) { | |
| reportImmediateUB() | |
| << "llvm.speculative.load with poison number of accessible bytes."; | |
| return AnyValue(); | |
| } | |
| const uint64_t Size = Ctx.getEffectiveTypeStoreSize(RetTy); | |
| const APInt &NumBytesInt = NumBytes.asInteger(); | |
| if (NumBytesInt.ugt(Size)) { | |
| reportImmediateUB() << "llvm.speculative.load number of accessible bytes " | |
| << NumBytesInt.getZExtValue() | |
| << " exceeds the loaded size " << Size << "."; | |
| return AnyValue(); | |
| } | |
| // Only the accessible bytes are read from memory and must be in bounds of | |
| // the underlying object. All other bytes are poison. | |
| const uint64_t N = NumBytesInt.getZExtValue(); | |
| SmallVector<Byte> Bytes(Size, Byte::poison()); | |
| if (N != 0) { | |
| const bool FromEnd = cast<ConstantInt>(CB.getArgOperand(1))->isOne(); | |
| const uint64_t Start = FromEnd ? Size - N : 0; | |
| const Pointer &PtrVal = Ptr.asPointer(); | |
| auto [MO, Offset] = | |
| verifyMemAccess(PtrVal.getWithNewAddr(PtrVal.address() + Start), N, | |
| Align(1), /*IsStore=*/false); | |
| if (!MO) | |
| return AnyValue(); | |
| copy(MO->getBytes().slice(Offset, N), Bytes.begin() + Start); | |
| } | |
| return Ctx.fromBytes(Bytes, RetTy); | |
| } | |
| /// How masked memory intrinsics compute the address of each lane. | |
| enum class MaskedAddrMode { | |
| /// \p Ptr is the base pointer and lane I accesses element I after it. | |
| Consecutive, | |
| /// \p Ptr holds a separate pointer for each lane. | |
| PerLane, | |
| /// \p Ptr is the base pointer and the offset increases on each active lane. | |
| Compressed, | |
| }; | |
| /// Implements the masked intrinsic family. For loads, \p Data holds the | |
| /// pass-through values and is updated with the loaded lanes; for stores, it | |
| /// holds the values to store. Lane addresses are computed according to \p | |
| /// Mode. | |
| AnyValue callMaskedLoadStoreIntrinsic(CallBase &CB, const AnyValue &Ptr, | |
| const AnyValue &Mask, AnyValue Data, | |
| MaskedAddrMode Mode, bool IsStore) { | |
| StringRef IntrinsicName = Intrinsic::getBaseName(CB.getIntrinsicID()); | |
| Type *ElemTy = (IsStore ? CB.getArgOperand(0)->getType() : CB.getType()) | |
| ->getScalarType(); | |
| // TODO: Support element types that are not a whole number of bytes. | |
| if (Mode != MaskedAddrMode::PerLane && | |
| !DL.typeSizeEqualsStoreSize(ElemTy)) { | |
| reportError() << IntrinsicName << " with element type " << *ElemTy | |
| << " is not supported."; | |
| return AnyValue(); | |
| } | |
| const uint64_t AccessSize = Ctx.getEffectiveTypeStoreSize(ElemTy); | |
| const unsigned IndexBW = | |
| DL.getIndexTypeSizeInBits(CB.getArgOperand(IsStore ? 1 : 0)->getType()); | |
| AnyValue LanePtr; | |
| if (Mode != MaskedAddrMode::PerLane) | |
| LanePtr = Ptr; | |
| GEPNoWrapFlags Flags = GEPNoWrapFlags::none(); | |
| for (auto [I, Lane] : enumerate(Data.asAggregate())) { | |
| if (Mode == MaskedAddrMode::PerLane) | |
| LanePtr = Ptr.asAggregate()[I]; | |
| BooleanKind MaskLane = getMaskLane(Mask, I); | |
| if (MaskLane == BooleanKind::Poison) { | |
| reportImmediateUB() << "Poison mask lane in " << IntrinsicName << '.'; | |
| return AnyValue(); | |
| } | |
| assert(MaskLane == BooleanKind::True || MaskLane == BooleanKind::False); | |
| if (MaskLane == BooleanKind::True) { | |
| if (IsStore) | |
| store(LanePtr, Align(1), Lane, ElemTy); | |
| else | |
| Lane = load(LanePtr, Align(1), ElemTy, /*NoUndef=*/false); | |
| // The first enabled base pointer may not be inbounds, but pointers | |
| // after that must be inbounds. | |
| Flags = GEPNoWrapFlags::inBounds(); | |
| } | |
| if (Mode == MaskedAddrMode::Consecutive || | |
| (Mode == MaskedAddrMode::Compressed && | |
| MaskLane == BooleanKind::True)) { | |
| AnyValue AccumulatedOffset = AnyValue::poison(); | |
| LanePtr = Ctx.computePtrAdd(LanePtr, APInt(IndexBW, AccessSize), Flags, | |
| AccumulatedOffset); | |
| } | |
| } | |
| return IsStore ? AnyValue() : Data; | |
| } | |
| /// Returns \p Ptr, the pointer argument of the VP memory intrinsic \p CB. | |
| /// Without an align attribute it must have the ABI alignment of \p Ty. | |
| AnyValue withVPDefaultAlign(CallBase &CB, AnyValue Ptr, Type *Ty) { | |
| if (!cast<VPIntrinsic>(CB).getPointerAlignment()) | |
| applyAlignAttr(Ptr, DL.getABITypeAlign(Ty)); | |
| return Ptr; | |
| } | |
| /// Implements llvm.vp.{load,store,gather,scatter} as the corresponding masked | |
| /// intrinsic. Lanes at or past \p EVL are disabled even if their \p Mask lane | |
| /// is poison. | |
| AnyValue callVPLoadStoreIntrinsic(CallBase &CB, const AnyValue &Ptr, | |
| const AnyValue &Mask, const AnyValue &EVL, | |
| AnyValue Data, MaskedAddrMode Mode, | |
| bool IsStore) { | |
| StringRef IntrinsicName = Intrinsic::getBaseName(CB.getIntrinsicID()); | |
| if (EVL.isPoison()) { | |
| reportImmediateUB() << IntrinsicName | |
| << " with poison explicit vector length."; | |
| return AnyValue(); | |
| } | |
| std::vector<AnyValue> EnabledMask(Mask.asAggregate()); | |
| const APInt &EVLInt = EVL.asInteger(); | |
| if (EVLInt.ugt(EnabledMask.size())) { | |
| reportImmediateUB() << IntrinsicName << " with explicit vector length " | |
| << EVLInt << " greater than the number of elements " | |
| << EnabledMask.size() << '.'; | |
| return AnyValue(); | |
| } | |
| for (uint64_t I = EVLInt.getZExtValue(); I != EnabledMask.size(); ++I) | |
| EnabledMask[I] = AnyValue::boolean(false); | |
| return callMaskedLoadStoreIntrinsic(CB, Ptr, std::move(EnabledMask), | |
| std::move(Data), Mode, IsStore); | |
| } | |
| /// Implements llvm.experimental.vp.strided.{load,store} as a vp.gather or | |
| /// vp.scatter of the pointers <Ptr, Ptr + Stride, Ptr + 2 * Stride, ...>, | |
| /// where \p Stride is signed and the arithmetic wraps in the pointer index | |
| /// type. Lane 0 is \p Ptr even if \p Stride is poison. The alignment, from | |
| /// an align attribute or else the element type's ABI alignment, only applies | |
| /// to \p Ptr, not to the other lanes' pointers. | |
| AnyValue callVPStridedLoadStoreIntrinsic(CallBase &CB, const AnyValue &Ptr, | |
| const AnyValue &Stride, | |
| const AnyValue &Mask, | |
| const AnyValue &EVL, AnyValue Data, | |
| bool IsStore) { | |
| const unsigned IndexBW = | |
| DL.getIndexTypeSizeInBits(CB.getArgOperand(IsStore ? 1 : 0)->getType()); | |
| const AnyValue StrideIdx = | |
| Context::canonicalizeIndex(Stride, IndexBW, GEPNoWrapFlags::none()); | |
| std::vector<AnyValue> LanePtrs(Mask.asAggregate().size(), Ptr); | |
| for (auto [I, LanePtr] : drop_begin(enumerate(LanePtrs))) { | |
| AnyValue AccumulatedOffset = AnyValue::poison(); | |
| LanePtr = Ctx.computeScaledPtrAdd( | |
| Ptr, StrideIdx, | |
| APInt(IndexBW, I, /*isSigned=*/false, /*implicitTrunc=*/true), | |
| GEPNoWrapFlags::none(), AccumulatedOffset); | |
| } | |
| return callVPLoadStoreIntrinsic(CB, std::move(LanePtrs), Mask, EVL, | |
| std::move(Data), MaskedAddrMode::PerLane, | |
| IsStore); | |
| } | |
| /// Implements llvm.matrix.column.major.store. Column C of \p Data is stored | |
| /// as a <Rows x ElemTy> vector at an inbounds nuw GEP of C * \p Stride | |
| /// elements from \p Ptr. | |
| AnyValue callMatrixColumnMajorStoreIntrinsic(CallBase &CB, | |
| const AnyValue &Ptr, | |
| const AnyValue &Stride, | |
| const AnyValue &Data) { | |
| // TODO: Handle isvolatile argument. | |
| StringRef IntrinsicName = Intrinsic::getBaseName(CB.getIntrinsicID()); | |
| const unsigned Rows = | |
| cast<ConstantInt>(CB.getArgOperand(4))->getZExtValue(); | |
| const unsigned Cols = | |
| cast<ConstantInt>(CB.getArgOperand(5))->getZExtValue(); | |
| if (Stride.isPoison()) { | |
| reportImmediateUB() << IntrinsicName << " with poison stride."; | |
| return AnyValue(); | |
| } | |
| const APInt &StrideInt = Stride.asInteger(); | |
| if (StrideInt.ult(Rows)) { | |
| reportImmediateUB() << IntrinsicName << " with stride " | |
| << StrideInt.getZExtValue() | |
| << " less than the number of rows " << Rows << '.'; | |
| return AnyValue(); | |
| } | |
| Type *ElemTy = CB.getArgOperand(0)->getType()->getScalarType(); | |
| auto *ColTy = FixedVectorType::get(ElemTy, Rows); | |
| const unsigned IndexBW = | |
| DL.getIndexTypeSizeInBits(CB.getArgOperand(1)->getType()); | |
| const APInt StrideIdx = StrideInt.zextOrTrunc(IndexBW); | |
| const APInt ElemSize(IndexBW, Ctx.getEffectiveTypeAllocSize(ElemTy), | |
| /*isSigned=*/false, /*implicitTrunc=*/true); | |
| ArrayRef<AnyValue> Elems = Data.asAggregate(); | |
| for (unsigned C = 0; C != Cols; ++C) { | |
| AnyValue AccumulatedOffset = AnyValue::poison(); | |
| // C * Stride is part of the nuw GEP index: overflow yields poison. | |
| AnyValue ColIdx = mulNoWrap(StrideIdx, | |
| APInt(IndexBW, C, /*IsSigned=*/false, | |
| /*ImplicitTrunc=*/true), | |
| /*HasNSW=*/false, /*HasNUW=*/true); | |
| AnyValue ColPtr = Ctx.computeScaledPtrAdd( | |
| Ptr, ColIdx, ElemSize, GEPNoWrapFlags::all(), AccumulatedOffset); | |
| store(ColPtr, Align(1), Elems.slice(C * Rows, Rows).vec(), ColTy); | |
| } | |
| return AnyValue(); | |
| } | |
| public: | |
| InstExecutor(Context &C, EventHandler &H, Function &F, | |
| ArrayRef<AnyValue> Args, AnyValue &RetVal) | |
| : ExecutorBase(C, H), DL(Ctx.getDataLayout()), | |
| Lib(Ctx, Handler, DL, static_cast<ExecutorBase &>(*this)) { | |
| CallStack.emplace_back(F, /*CallSite=*/nullptr, /*LastFrame=*/nullptr, Args, | |
| RetVal, Ctx.getTLIImpl()); | |
| } | |
| void visitReturnInst(ReturnInst &RI) { | |
| if (auto *RV = RI.getReturnValue()) | |
| CurrentFrame->RetVal = getValue(RV); | |
| else | |
| CurrentFrame->RetVal = AnyValue(); | |
| CurrentFrame->State = FrameState::Exit; | |
| if (!Handler.onInstructionExecuted(RI, None)) | |
| setFailed(); | |
| } | |
| void visitUncondBrInst(UncondBrInst &BI) { jumpTo(BI, BI.getSuccessor()); } | |
| void visitCondBrInst(CondBrInst &BI) { | |
| switch (getValue(BI.getCondition()).asBoolean()) { | |
| case BooleanKind::True: | |
| jumpTo(BI, BI.getSuccessor(0)); | |
| return; | |
| case BooleanKind::False: | |
| jumpTo(BI, BI.getSuccessor(1)); | |
| return; | |
| case BooleanKind::Poison: | |
| reportImmediateUB() << "Branch on poison condition."; | |
| return; | |
| } | |
| } | |
| void visitSwitchInst(SwitchInst &SI) { | |
| auto &Cond = getValue(SI.getCondition()); | |
| if (Cond.isPoison()) { | |
| reportImmediateUB() << "Switch on poison condition."; | |
| return; | |
| } | |
| for (auto &Case : SI.cases()) { | |
| if (Case.getCaseValue()->getValue() == Cond.asInteger()) { | |
| jumpTo(SI, Case.getCaseSuccessor()); | |
| return; | |
| } | |
| } | |
| jumpTo(SI, SI.getDefaultDest()); | |
| } | |
| void visitUnreachableInst(UnreachableInst &) { | |
| reportImmediateUB() << "Unreachable code."; | |
| } | |
| void visitCallBrInst(CallBrInst &CI) { | |
| if (isNoopInlineAsm(CI.getCalledOperand(), CI.getType())) { | |
| jumpTo(CI, CI.getDefaultDest()); | |
| return; | |
| } | |
| Handler.onUnrecognizedInstruction(CI); | |
| setFailed(); | |
| } | |
| void visitIndirectBrInst(IndirectBrInst &IBI) { | |
| auto &Target = getValue(IBI.getAddress()); | |
| if (Target.isPoison()) { | |
| reportImmediateUB() << "Indirect branch on poison."; | |
| return; | |
| } | |
| if (BasicBlock *DestBB = Ctx.getTargetBlock(Target.asPointer())) { | |
| if (any_of(IBI.successors(), | |
| [DestBB](BasicBlock *Succ) { return Succ == DestBB; })) | |
| jumpTo(IBI, DestBB); | |
| else | |
| reportImmediateUB() << "Indirect branch on unlisted target BB."; | |
| return; | |
| } | |
| reportImmediateUB() << "Indirect branch on invalid target BB."; | |
| } | |
| void returnFromCallee() { | |
| if (hasProgramExited()) | |
| return; | |
| auto &CB = cast<CallBase>(*CurrentFrame->PC); | |
| if (CB.doesNotReturn() || (CurrentFrame->ResolvedCallee && | |
| CurrentFrame->ResolvedCallee->doesNotReturn())) | |
| reportImmediateUB() << "Return from a noreturn function/callsite."; | |
| AnyValue &RetVal = CurrentFrame->CalleeRetVal; | |
| if (Function *Oracle = getSpeculativeLoadOracle(CB)) { | |
| // RetVal is the oracle's result; use it to complete the load. | |
| handleAttributes(Oracle->getReturnType(), RetVal, AttributeSet(), | |
| Oracle->getAttributes().getRetAttrs()); | |
| RetVal = | |
| callSpeculativeLoadIntrinsic(CB, CurrentFrame->CalleeArgs[0], RetVal); | |
| if (hasProgramExited()) | |
| return; | |
| } | |
| CurrentFrame->CalleeArgs.clear(); | |
| if (Type *RetTy = CB.getType(); !RetTy->isVoidTy()) { | |
| // Handle attributes on the return value (Attributes from resolved callee | |
| // should be applied if available). | |
| AttributeSet AttrsAtCallSite = CB.getRetAttributes(); | |
| AttributeSet AttrsAtCallee = | |
| CurrentFrame->ResolvedCallee->getAttributes().getRetAttrs(); | |
| if (auto *FPMO = dyn_cast<FPMathOperator>(&CB)) { | |
| if (FastMathFlags FMF = FPMO->getFastMathFlags(); FMF.any()) | |
| RetVal = handleFMFFlags(std::move(RetVal), FMF, /*IsInput=*/false); | |
| } | |
| handleAttributes(RetTy, RetVal, AttrsAtCallSite, AttrsAtCallee); | |
| handleMetadata(RetTy, RetVal, CB); | |
| } | |
| setResult(CB, std::move(RetVal)); | |
| for (auto &ByValArg : CurrentFrame->CalleeByValArgs) | |
| Ctx.free(*ByValArg); | |
| CurrentFrame->CalleeByValArgs.clear(); | |
| if (auto *II = dyn_cast<InvokeInst>(&CB)) | |
| jumpTo(*II, II->getNormalDest()); | |
| else if (CurrentFrame->State == FrameState::Pending) | |
| ++CurrentFrame->PC; | |
| } | |
| AnyValue computeDivRem(unsigned Opcode, const AnyValue &LHS, | |
| const AnyValue &RHS, bool IsExact = false) { | |
| const bool IsSigned = | |
| Opcode == Instruction::SDiv || Opcode == Instruction::SRem; | |
| // Priority: Immediate UB > poison > normal value. | |
| if (RHS.isPoison()) { | |
| reportImmediateUB() << "Division by zero (refine RHS to 0)."; | |
| return AnyValue::poison(); | |
| } | |
| const APInt &RHSVal = RHS.asInteger(); | |
| if (RHSVal.isZero()) { | |
| reportImmediateUB() << "Division by zero."; | |
| return AnyValue::poison(); | |
| } | |
| if (LHS.isPoison()) { | |
| if (IsSigned && RHSVal.isAllOnes()) | |
| reportImmediateUB() | |
| << "Signed division overflow (refine LHS to INT_MIN)."; | |
| return AnyValue::poison(); | |
| } | |
| const APInt &LHSVal = LHS.asInteger(); | |
| if (IsSigned && LHSVal.isMinSignedValue() && RHSVal.isAllOnes()) { | |
| if (Opcode == Instruction::SRem) | |
| reportImmediateUB() << "Signed division overflow. LHS: " << LHSVal | |
| << ", RHS: " << RHSVal; | |
| else | |
| reportImmediateUB() << "Signed division overflow."; | |
| return AnyValue::poison(); | |
| } | |
| APInt Q, R; | |
| if (IsSigned) | |
| APInt::sdivrem(LHSVal, RHSVal, Q, R); | |
| else | |
| APInt::udivrem(LHSVal, RHSVal, Q, R); | |
| if (Opcode == Instruction::SRem || Opcode == Instruction::URem) | |
| return R; | |
| if (IsExact && !R.isZero()) | |
| return AnyValue::poison(); | |
| return Q; | |
| } | |
| AnyValue callVectorReduction(Type *RetTy, FastMathFlags FMF, | |
| Intrinsic::ID IID, ArrayRef<AnyValue> Args) { | |
| switch (IID) { | |
| case Intrinsic::vector_reduce_add: | |
| case Intrinsic::vector_reduce_mul: | |
| case Intrinsic::vector_reduce_and: | |
| case Intrinsic::vector_reduce_or: | |
| case Intrinsic::vector_reduce_xor: | |
| case Intrinsic::vector_reduce_smax: | |
| case Intrinsic::vector_reduce_smin: | |
| case Intrinsic::vector_reduce_umax: | |
| case Intrinsic::vector_reduce_umin: { | |
| std::optional<APInt> Res; | |
| for (const auto &V : Args[0].asAggregate()) { | |
| if (V.isPoison()) { | |
| Res.reset(); | |
| break; | |
| } | |
| const auto &IntV = V.asInteger(); | |
| if (!Res) { | |
| Res = IntV; | |
| continue; | |
| } | |
| switch (IID) { | |
| case Intrinsic::vector_reduce_add: | |
| *Res += IntV; | |
| break; | |
| case Intrinsic::vector_reduce_mul: | |
| *Res *= IntV; | |
| break; | |
| case Intrinsic::vector_reduce_and: | |
| *Res &= IntV; | |
| break; | |
| case Intrinsic::vector_reduce_or: | |
| *Res |= IntV; | |
| break; | |
| case Intrinsic::vector_reduce_xor: | |
| *Res ^= IntV; | |
| break; | |
| case Intrinsic::vector_reduce_smax: | |
| *Res = APIntOps::smax(*Res, IntV); | |
| break; | |
| case Intrinsic::vector_reduce_smin: | |
| *Res = APIntOps::smin(*Res, IntV); | |
| break; | |
| case Intrinsic::vector_reduce_umax: | |
| *Res = APIntOps::umax(*Res, IntV); | |
| break; | |
| case Intrinsic::vector_reduce_umin: | |
| *Res = APIntOps::umin(*Res, IntV); | |
| break; | |
| default: | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| } | |
| } | |
| return Res ? *Res : AnyValue::poison(); | |
| } | |
| case Intrinsic::vector_reduce_fadd: | |
| case Intrinsic::vector_reduce_fmul: | |
| case Intrinsic::vector_reduce_fmaximum: | |
| case Intrinsic::vector_reduce_fminimum: | |
| case Intrinsic::vector_reduce_fmaximumnum: | |
| case Intrinsic::vector_reduce_fminimumnum: { | |
| const auto DenormMode = getCurrentDenormalMode(RetTy); | |
| const bool HasStart = IID == Intrinsic::vector_reduce_fadd || | |
| IID == Intrinsic::vector_reduce_fmul; | |
| const AnyValue &Vector = HasStart ? Args[1] : Args[0]; | |
| std::optional<APFloat> Res; | |
| if (HasStart) { | |
| if (Args[0].isPoison()) | |
| return AnyValue::poison(); | |
| const AnyValue ValidatedStart = | |
| handleFMFFlags(Args[0], FMF, /*IsInput=*/true); | |
| if (ValidatedStart.isPoison()) | |
| return AnyValue::poison(); | |
| Res = handleDenormal(ValidatedStart.asFloat(), DenormMode.Input, | |
| /*IsInput=*/true); | |
| } | |
| for (const auto &V : Vector.asAggregate()) { | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| const AnyValue ValidatedOp = handleFMFFlags(V, FMF, /*IsInput=*/true); | |
| if (ValidatedOp.isPoison()) | |
| return AnyValue::poison(); | |
| APFloat Op = handleDenormal(ValidatedOp.asFloat(), DenormMode.Input, | |
| /*IsInput=*/true); | |
| if (!Res) { | |
| Res = std::move(Op); | |
| continue; | |
| } | |
| switch (IID) { | |
| case Intrinsic::vector_reduce_fadd: | |
| *Res = *Res + Op; | |
| break; | |
| case Intrinsic::vector_reduce_fmul: | |
| *Res = *Res * Op; | |
| break; | |
| case Intrinsic::vector_reduce_fmaximum: | |
| *Res = maximum(*Res, Op); | |
| break; | |
| case Intrinsic::vector_reduce_fminimum: | |
| *Res = minimum(*Res, Op); | |
| break; | |
| case Intrinsic::vector_reduce_fmaximumnum: | |
| *Res = maximumnum(*Res, Op); | |
| break; | |
| case Intrinsic::vector_reduce_fminimumnum: | |
| *Res = minimumnum(*Res, Op); | |
| break; | |
| default: | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| } | |
| } | |
| assert(Res.has_value()); | |
| const AnyValue ValidatedRes = | |
| handleFMFFlags(*Res, FMF, /*IsInput=*/false); | |
| if (ValidatedRes.isPoison()) | |
| return AnyValue::poison(); | |
| const APFloat FRes = | |
| handleDenormal(ValidatedRes.asFloat(), DenormMode.Output, | |
| /*IsInput=*/false); | |
| SmallVector<const APFloat *, 8> InputVec; | |
| InputVec.reserve(Vector.asAggregate().size()); | |
| transform( | |
| Vector.asAggregate(), std::back_inserter(InputVec), | |
| [](const AnyValue &V) -> const APFloat * { return &V.asFloat(); }); | |
| return applyNaNPropagation(FRes, InputVec); | |
| } | |
| case Intrinsic::vector_reduce_fmax: | |
| case Intrinsic::vector_reduce_fmin: { | |
| const auto DenormMode = getCurrentDenormalMode(RetTy); | |
| const auto &Vector = Args[0].asAggregate(); | |
| SmallVector<APFloat, 8> InputFloats; | |
| SmallVector<const APFloat *, 8> InputVec; | |
| InputFloats.reserve(Vector.size()); | |
| InputVec.reserve(Vector.size()); | |
| for (const auto &V : Vector) { | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| const AnyValue ValidatedOp = handleFMFFlags(V, FMF, /*IsInput=*/true); | |
| if (ValidatedOp.isPoison()) | |
| return AnyValue::poison(); | |
| InputFloats.push_back(handleDenormal( | |
| ValidatedOp.asFloat(), DenormMode.Input, /*IsInput=*/true)); | |
| InputVec.push_back(&InputFloats.back()); | |
| } | |
| assert(!InputVec.empty()); | |
| SmallVector<APFloat, 8> Worklist(InputFloats); | |
| const bool HasSNaN = | |
| any_of(InputVec, [](const APFloat *V) { return V->isSignaling(); }); | |
| while (Worklist.size() > 1) { | |
| size_t LHSIdx = 0; | |
| size_t RHSIdx = 1; | |
| if (HasSNaN) { | |
| LHSIdx = Ctx.getRandomUInt64() % Worklist.size(); | |
| RHSIdx = Ctx.getRandomUInt64() % (Worklist.size() - 1); | |
| if (RHSIdx >= LHSIdx) | |
| ++RHSIdx; | |
| } | |
| APFloat Res = | |
| IID == Intrinsic::vector_reduce_fmax | |
| ? maxnumWithSNaNQuieting(Worklist[LHSIdx], Worklist[RHSIdx]) | |
| : minnumWithSNaNQuieting(Worklist[LHSIdx], Worklist[RHSIdx]); | |
| if (LHSIdx < RHSIdx) | |
| std::swap(LHSIdx, RHSIdx); | |
| Worklist.erase(Worklist.begin() + LHSIdx); | |
| Worklist.erase(Worklist.begin() + RHSIdx); | |
| Worklist.push_back(std::move(Res)); | |
| } | |
| AnyValue ValidatedRes = | |
| handleFMFFlags(Worklist.front(), FMF, /*IsInput=*/false); | |
| if (ValidatedRes.isPoison()) | |
| return AnyValue::poison(); | |
| APFloat FRes = handleDenormal(ValidatedRes.asFloat(), DenormMode.Output, | |
| /*IsInput=*/false); | |
| return applyNaNPropagation(FRes, InputVec); | |
| } | |
| default: | |
| llvm_unreachable("Unexpected vector reduction intrinsic ID"); | |
| } | |
| } | |
| AnyValue countTrailingZeroElements(Type *RetTy, ArrayRef<AnyValue> Vec, | |
| bool IsZeroPoison, size_t Length) { | |
| const unsigned RetBW = RetTy->getIntegerBitWidth(); | |
| if (!isUIntN(RetBW, Vec.size())) | |
| return AnyValue::poison(); | |
| for (const AnyValue &V : Vec.take_front(Length)) | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| for (size_t I = 0; I != Length; ++I) | |
| if (!Vec[I].asInteger().isZero()) | |
| return APInt(RetBW, I); | |
| return IsZeroPoison ? AnyValue::poison() : AnyValue(APInt(RetBW, Length)); | |
| } | |
| AnyValue callIntrinsic(CallBase &CB, ArrayRef<AnyValue> Args) { | |
| Intrinsic::ID IID = CB.getIntrinsicID(); | |
| Type *RetTy = CB.getType(); | |
| const FastMathFlags FMF = CB.getFastMathFlagsOrNone(); | |
| switch (IID) { | |
| case Intrinsic::assume: | |
| switch (Args[0].asBoolean()) { | |
| case BooleanKind::True: | |
| for (unsigned Idx = 0; Idx < CB.getNumOperandBundles(); Idx++) { | |
| OperandBundleUse OBU = CB.getOperandBundleAt(Idx); | |
| auto GetBundleArg = [&](uint32_t Offset) -> Value * { | |
| return OBU.Inputs[Offset]; | |
| }; | |
| // Bail out on "cold"() | |
| if (OBU.Inputs.empty()) | |
| continue; | |
| Value *WasOnVal = GetBundleArg(0); | |
| const AnyValue &WasOn = getValue(WasOnVal); | |
| auto Kind = getBundleAttrFromOBU(OBU); | |
| switch (Kind) { | |
| case BundleAttr::Ignore: | |
| continue; | |
| case BundleAttr::NoUndef: | |
| if (violatesNoUndefAttr(WasOn)) | |
| reportImmediateUB() | |
| << "The value " << WasOn << " violates noundef assumption."; | |
| break; | |
| case BundleAttr::Range: { | |
| AnyValue V = WasOn; | |
| const APInt &Lower = cast<ConstantInt>(OBU.Inputs[1])->getValue(); | |
| const APInt &Upper = cast<ConstantInt>(OBU.Inputs[2])->getValue(); | |
| applyRangeAttr(V, ConstantRange(Lower, Upper)); | |
| if (violatesNoUndefAttr(V)) | |
| reportImmediateUB() << "The value " << WasOn << " violates range(" | |
| << Lower << ", " << Upper << ") assumption."; | |
| break; | |
| } | |
| default: | |
| // Other assumptions on pointers or | |
| // unrecognized ones. | |
| break; | |
| } | |
| if (hasProgramExited()) | |
| break; | |
| // Bail out on unrecognized operand bundles. | |
| if (!WasOnVal->getType()->isPointerTy()) | |
| continue; | |
| unsigned AS = WasOnVal->getType()->getPointerAddressSpace(); | |
| if (WasOn.isPoison()) { | |
| reportImmediateUB() << "Assume on poison pointer."; | |
| break; | |
| } | |
| const Pointer &WasOnPtr = WasOn.asPointer(); | |
| switch (Kind) { | |
| case BundleAttr::Align: { | |
| // Alignment assumptions should have 2 or 3 arguments. | |
| APInt Alignment = getIntNonPoison(getValue(GetBundleArg(1))); | |
| APInt CheckedAddr = WasOnPtr.address(); | |
| if (OBU.Inputs.size() == 3) { | |
| APInt Offset = getIntNonPoison(getValue(GetBundleArg(2))); | |
| CheckedAddr -= Offset.sextOrTrunc(CheckedAddr.getBitWidth()); | |
| } | |
| if (!Alignment.isPowerOf2()) { | |
| if (!CheckedAddr.isZero()) | |
| reportImmediateUB() << "Assume on pointer " << WasOn | |
| << " with a nonzero adjusted address and a " | |
| "non-power-of-two alignment " | |
| << Alignment << '.'; | |
| break; | |
| } | |
| if (CheckedAddr.countr_zero() < Alignment.logBase2()) | |
| reportImmediateUB() | |
| << "The pointer " << WasOn << " violates align(" << Alignment | |
| << ") assumption."; | |
| break; | |
| } | |
| case BundleAttr::NonNull: | |
| if (WasOnPtr.isNullPtr(AS, DL)) | |
| reportImmediateUB() | |
| << "The pointer " << WasOn << " violates nonnull assumption."; | |
| break; | |
| case BundleAttr::Dereferenceable: | |
| case BundleAttr::DereferenceableOrNull: { | |
| APInt DereferenceableBytes = | |
| getIntNonPoison(getValue(GetBundleArg(1))); | |
| // Only n > 0 implies that the pointer is dereferenceable. | |
| if (DereferenceableBytes.isZero()) | |
| break; | |
| if (violatesDereferenceableBytesAttr( | |
| WasOn, DereferenceableBytes.getLimitedValue(), | |
| Kind == BundleAttr::DereferenceableOrNull, AS, Ctx)) | |
| reportImmediateUB() << "The pointer " << WasOn << " violates " | |
| << (Kind == BundleAttr::DereferenceableOrNull | |
| ? "dereferenceable_or_null(" | |
| : "dereferenceable(") | |
| << DereferenceableBytes << ") assumption."; | |
| break; | |
| } | |
| default: | |
| // TODO: handle other operand bundles like separate_storage. | |
| break; | |
| } | |
| } | |
| break; | |
| case BooleanKind::False: | |
| case BooleanKind::Poison: | |
| reportImmediateUB() << "Assume on false or poison condition."; | |
| break; | |
| } | |
| return AnyValue(); | |
| case Intrinsic::lifetime_start: | |
| case Intrinsic::lifetime_end: { | |
| auto Ptr = Args[0]; | |
| if (Ptr.isPoison()) | |
| return AnyValue(); | |
| auto *MO = Ctx.checkProvenance(Ptr.asPointer(), | |
| [](const Provenance &) { return true; }); | |
| assert(MO && "Memory object accessed by lifetime intrinsic should be " | |
| "always valid."); | |
| if (IID == Intrinsic::lifetime_start) { | |
| MO->setState(MemoryObjectState::Alive); | |
| fill(MO->getBytes(), Byte::undef()); | |
| } else { | |
| fill(MO->getBytes(), Byte::poison()); | |
| MO->setState(MemoryObjectState::Dead); | |
| } | |
| return AnyValue(); | |
| } | |
| case Intrinsic::ssa_copy: | |
| case Intrinsic::expect: | |
| case Intrinsic::expect_with_probability: | |
| return Args[0]; | |
| case Intrinsic::donothing: | |
| return AnyValue(); | |
| case Intrinsic::vscale: { | |
| const unsigned BitWidth = RetTy->getScalarSizeInBits(); | |
| const APInt VScale(64, Ctx.getVScale()); | |
| if (!VScale.isIntN(BitWidth)) | |
| return AnyValue::poison(); | |
| return VScale.zextOrTrunc(BitWidth); | |
| } | |
| case Intrinsic::abs: { | |
| const bool IsIntMinPoison = getBooleanNonPoison(Args[1].asBoolean()); | |
| return visitIntUnOpWithResult( | |
| RetTy, Args[0], [&](const APInt &Operand) -> AnyValue { | |
| if (IsIntMinPoison && Operand.isMinSignedValue()) | |
| return AnyValue::poison(); | |
| return Operand.abs(); | |
| }); | |
| } | |
| case Intrinsic::smax: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return APIntOps::smax(LHS, RHS); | |
| }); | |
| } | |
| case Intrinsic::smin: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return APIntOps::smin(LHS, RHS); | |
| }); | |
| } | |
| case Intrinsic::umax: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return APIntOps::umax(LHS, RHS); | |
| }); | |
| } | |
| case Intrinsic::umin: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return APIntOps::umin(LHS, RHS); | |
| }); | |
| } | |
| case Intrinsic::scmp: | |
| case Intrinsic::ucmp: { | |
| const unsigned BitWidth = RetTy->getScalarSizeInBits(); | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [&](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| if (LHS == RHS) | |
| return APInt::getZero(BitWidth); | |
| if (IID == Intrinsic::scmp) | |
| return LHS.slt(RHS) ? APInt::getAllOnes(BitWidth) | |
| : APInt(BitWidth, 1); | |
| return LHS.ult(RHS) ? APInt::getAllOnes(BitWidth) | |
| : APInt(BitWidth, 1); | |
| }); | |
| } | |
| case Intrinsic::bitreverse: { | |
| return visitIntUnOpWithResult(RetTy, Args[0], | |
| [](const APInt &Operand) -> AnyValue { | |
| return Operand.reverseBits(); | |
| }); | |
| } | |
| case Intrinsic::bswap: { | |
| return visitIntUnOpWithResult( | |
| RetTy, Args[0], | |
| [](const APInt &Operand) -> AnyValue { return Operand.byteSwap(); }); | |
| } | |
| case Intrinsic::ctpop: { | |
| return visitIntUnOpWithResult( | |
| RetTy, Args[0], [](const APInt &Operand) -> AnyValue { | |
| return APInt(Operand.getBitWidth(), Operand.popcount()); | |
| }); | |
| } | |
| case Intrinsic::ctlz: | |
| case Intrinsic::cttz: { | |
| const bool IsZeroPoison = getBooleanNonPoison(Args[1].asBoolean()); | |
| return visitIntUnOpWithResult( | |
| RetTy, Args[0], [&](const APInt &Operand) -> AnyValue { | |
| if (IsZeroPoison && Operand.isZero()) | |
| return AnyValue::poison(); | |
| if (IID == Intrinsic::ctlz) | |
| return APInt(Operand.getBitWidth(), Operand.countl_zero()); | |
| return APInt(Operand.getBitWidth(), Operand.countr_zero()); | |
| }); | |
| } | |
| case Intrinsic::fshl: | |
| case Intrinsic::fshr: { | |
| return visitIntTriOpWithResult( | |
| RetTy, Args[0], Args[1], Args[2], | |
| [IID](const APInt &Op1, const APInt &Op2, | |
| const APInt &Op3) -> AnyValue { | |
| const unsigned BitWidth = Op1.getBitWidth(); | |
| const uint64_t ShiftAmount = Op3.urem(BitWidth); | |
| const bool IsFShr = IID == Intrinsic::fshr; | |
| if (ShiftAmount == 0) | |
| return IsFShr ? Op2 : Op1; | |
| const uint64_t LShrAmount = | |
| IsFShr ? ShiftAmount : BitWidth - ShiftAmount; | |
| const uint64_t ShlAmount = | |
| !IsFShr ? ShiftAmount : BitWidth - ShiftAmount; | |
| return Op1.shl(ShlAmount) | Op2.lshr(LShrAmount); | |
| }); | |
| } | |
| case Intrinsic::clmul: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return APIntOps::clmul(LHS, RHS); | |
| }); | |
| } | |
| case Intrinsic::sadd_with_overflow: | |
| case Intrinsic::uadd_with_overflow: | |
| case Intrinsic::ssub_with_overflow: | |
| case Intrinsic::usub_with_overflow: | |
| case Intrinsic::smul_with_overflow: | |
| case Intrinsic::umul_with_overflow: { | |
| return visitOverflowIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [IID](const APInt &LHS, const APInt &RHS) -> std::pair<APInt, bool> { | |
| APInt Res; | |
| bool Overflow = false; | |
| switch (IID) { | |
| case Intrinsic::sadd_with_overflow: | |
| Res = LHS.sadd_ov(RHS, Overflow); | |
| break; | |
| case Intrinsic::uadd_with_overflow: | |
| Res = LHS.uadd_ov(RHS, Overflow); | |
| break; | |
| case Intrinsic::ssub_with_overflow: | |
| Res = LHS.ssub_ov(RHS, Overflow); | |
| break; | |
| case Intrinsic::usub_with_overflow: | |
| Res = LHS.usub_ov(RHS, Overflow); | |
| break; | |
| case Intrinsic::smul_with_overflow: | |
| Res = LHS.smul_ov(RHS, Overflow); | |
| break; | |
| case Intrinsic::umul_with_overflow: | |
| Res = LHS.umul_ov(RHS, Overflow); | |
| break; | |
| default: | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| } | |
| return {Res, Overflow}; | |
| }); | |
| } | |
| case Intrinsic::sadd_sat: | |
| case Intrinsic::uadd_sat: | |
| case Intrinsic::ssub_sat: | |
| case Intrinsic::usub_sat: | |
| case Intrinsic::sshl_sat: | |
| case Intrinsic::ushl_sat: { | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [IID](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| switch (IID) { | |
| case Intrinsic::sadd_sat: | |
| return LHS.sadd_sat(RHS); | |
| case Intrinsic::uadd_sat: | |
| return LHS.uadd_sat(RHS); | |
| case Intrinsic::ssub_sat: | |
| return LHS.ssub_sat(RHS); | |
| case Intrinsic::usub_sat: | |
| return LHS.usub_sat(RHS); | |
| case Intrinsic::sshl_sat: { | |
| if (RHS.uge(LHS.getBitWidth())) | |
| return AnyValue::poison(); | |
| return LHS.sshl_sat(RHS); | |
| } | |
| case Intrinsic::ushl_sat: { | |
| if (RHS.uge(LHS.getBitWidth())) | |
| return AnyValue::poison(); | |
| return LHS.ushl_sat(RHS); | |
| } | |
| default: | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| } | |
| }); | |
| } | |
| case Intrinsic::smulh: | |
| case Intrinsic::umulh: | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [IID](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return IID == Intrinsic::smulh ? APIntOps::mulhs(LHS, RHS) | |
| : APIntOps::mulhu(LHS, RHS); | |
| }); | |
| case Intrinsic::pdep: | |
| case Intrinsic::pext: | |
| return visitIntBinOpWithResult( | |
| RetTy, Args[0], Args[1], | |
| [IID](const APInt &Val, const APInt &Mask) -> AnyValue { | |
| return IID == Intrinsic::pdep ? APIntOps::pdep(Val, Mask) | |
| : APIntOps::pext(Val, Mask); | |
| }); | |
| case Intrinsic::vector_reduce_add: | |
| case Intrinsic::vector_reduce_mul: | |
| case Intrinsic::vector_reduce_and: | |
| case Intrinsic::vector_reduce_or: | |
| case Intrinsic::vector_reduce_xor: | |
| case Intrinsic::vector_reduce_smax: | |
| case Intrinsic::vector_reduce_smin: | |
| case Intrinsic::vector_reduce_umax: | |
| case Intrinsic::vector_reduce_umin: | |
| return callVectorReduction(RetTy, FMF, IID, Args); | |
| case Intrinsic::vector_insert: { | |
| assert(!Args[2].isPoison() && | |
| "Verifier should reject poison vector_insert immarg."); | |
| const auto &Vec = Args[0].asAggregate(); | |
| const auto &SubVec = Args[1].asAggregate(); | |
| const auto &Idx = Args[2].asInteger(); | |
| auto EC = | |
| cast<VectorType>(CB.getArgOperand(1)->getType())->getElementCount(); | |
| const uint64_t RawOffset = Idx.getZExtValue(); | |
| const uint32_t MinSize = EC.getKnownMinValue(); | |
| assert(RawOffset % MinSize == 0 && | |
| "Verifier should reject misaligned vector_insert index."); | |
| const uint64_t Chunk = RawOffset / MinSize; | |
| const uint64_t EVL = Ctx.getEVL(EC); | |
| if (Chunk > std::numeric_limits<uint64_t>::max() / EVL) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| const uint64_t Offset = Chunk * EVL; | |
| if (Offset > Vec.size() || SubVec.size() > Vec.size() - Offset) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Vec.size()); | |
| for (size_t I = 0; I != Vec.size(); ++I) { | |
| if (I >= Offset && I < Offset + SubVec.size()) | |
| Res.push_back(SubVec[I - Offset]); | |
| else | |
| Res.push_back(Vec[I]); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::vector_extract: { | |
| assert(!Args[1].isPoison() && | |
| "Verifier should reject poison vector_extract immarg."); | |
| const auto &Vec = Args[0].asAggregate(); | |
| const auto &Idx = Args[1].asInteger(); | |
| auto EC = cast<VectorType>(RetTy)->getElementCount(); | |
| const uint64_t RawOffset = Idx.getZExtValue(); | |
| const uint32_t MinSize = EC.getKnownMinValue(); | |
| assert(RawOffset % MinSize == 0 && | |
| "Verifier should reject misaligned vector_extract index."); | |
| const uint64_t Chunk = RawOffset / MinSize; | |
| const uint64_t EVL = Ctx.getEVL(EC); | |
| if (Chunk > std::numeric_limits<uint64_t>::max() / EVL) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| const uint64_t Offset = Chunk * EVL; | |
| if (Offset > Vec.size() || EVL > Vec.size() - Offset) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| return std::vector<AnyValue>(Vec.begin() + Offset, | |
| Vec.begin() + Offset + EVL); | |
| } | |
| case Intrinsic::vector_reverse: { | |
| auto Vec = Args[0].asAggregate(); | |
| std::reverse(Vec.begin(), Vec.end()); | |
| return std::move(Vec); | |
| } | |
| case Intrinsic::vector_deinterleave2: | |
| case Intrinsic::vector_deinterleave3: | |
| case Intrinsic::vector_deinterleave4: | |
| case Intrinsic::vector_deinterleave5: | |
| case Intrinsic::vector_deinterleave6: | |
| case Intrinsic::vector_deinterleave7: | |
| case Intrinsic::vector_deinterleave8: { | |
| const unsigned Factor = getDeinterleaveIntrinsicFactor(IID); | |
| if (Factor == 0) | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| const auto &Vec = Args[0].asAggregate(); | |
| std::vector<std::vector<AnyValue>> Res(Factor); | |
| for (auto &SubVec : Res) | |
| SubVec.reserve(Vec.size() / Factor); | |
| for (size_t I = 0, E = Vec.size(); I != E; ++I) | |
| Res[I % Factor].push_back(Vec[I]); | |
| std::vector<AnyValue> AggRes; | |
| AggRes.reserve(Factor); | |
| for (auto &SubVec : Res) | |
| AggRes.emplace_back(std::move(SubVec)); | |
| return AnyValue(std::move(AggRes)); | |
| } | |
| case Intrinsic::vector_interleave2: | |
| case Intrinsic::vector_interleave3: | |
| case Intrinsic::vector_interleave4: | |
| case Intrinsic::vector_interleave5: | |
| case Intrinsic::vector_interleave6: | |
| case Intrinsic::vector_interleave7: | |
| case Intrinsic::vector_interleave8: { | |
| const unsigned Factor = getInterleaveIntrinsicFactor(IID); | |
| if (Factor == 0) | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| const auto &Vec = Args[0].asAggregate(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Vec.size() * Factor); | |
| for (size_t I = 0, E = Vec.size(); I != E; ++I) { | |
| for (unsigned J = 0; J != Factor; ++J) | |
| Res.push_back(Args[J].asAggregate()[I]); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::vector_splice_left: { | |
| if (Args[2].isPoison()) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| const auto &LHS = Args[0].asAggregate(); | |
| const auto &RHS = Args[1].asAggregate(); | |
| const auto &Off = Args[2].asInteger(); | |
| const size_t Len = LHS.size(); | |
| if (Off.ugt(Len)) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| uint64_t Offset = Off.getZExtValue(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Len); | |
| for (size_t I = 0; I != Len; ++I) { | |
| size_t Pos = I + Offset; | |
| Res.push_back(Pos < Len ? LHS[Pos] : RHS[Pos - Len]); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::vector_splice_right: { | |
| if (Args[2].isPoison()) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| const auto &LHS = Args[0].asAggregate(); | |
| const auto &RHS = Args[1].asAggregate(); | |
| const auto &Off = Args[2].asInteger(); | |
| const size_t Len = LHS.size(); | |
| if (Off.ugt(Len)) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| uint64_t Offset = Len - Off.getZExtValue(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Len); | |
| for (size_t I = 0; I != Len; ++I) { | |
| size_t Pos = I + Offset; | |
| Res.push_back(Pos < Len ? LHS[Pos] : RHS[Pos - Len]); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::stepvector: { | |
| std::vector<AnyValue> Res; | |
| const uint32_t Len = | |
| Ctx.getEVL(cast<VectorType>(RetTy)->getElementCount()); | |
| const unsigned BitWidth = RetTy->getScalarSizeInBits(); | |
| Res.reserve(Len); | |
| for (uint64_t I = 0; I != Len; ++I) { | |
| Res.push_back( | |
| APInt(BitWidth, I, /*IsSigned=*/false, /*ImplicitTrunc=*/true)); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::get_active_lane_mask: { | |
| if (Args[0].isPoison() || Args[1].isPoison()) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| // Lane I is active iff Base + I < N without wrapping, i.e. iff Base < N | |
| // and I < N - Base, where N - Base cannot wrap. | |
| const APInt &Base = Args[0].asInteger(); | |
| const APInt &N = Args[1].asInteger(); | |
| const uint32_t Len = | |
| Ctx.getEVL(cast<VectorType>(RetTy)->getElementCount()); | |
| const uint32_t NumActive = | |
| Base.uge(N) ? 0 : (N - Base).getLimitedValue(Len); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Len); | |
| Res.insert(Res.end(), NumActive, AnyValue::boolean(true)); | |
| Res.insert(Res.end(), Len - NumActive, AnyValue::boolean(false)); | |
| return std::move(Res); | |
| } | |
| case Intrinsic::mask_beforefirst: { | |
| const auto &Mask = Args[0].asAggregate(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Mask.size()); | |
| bool SeenActive = false; | |
| for (const AnyValue &Lane : Mask) { | |
| switch (Lane.asBoolean()) { | |
| case BooleanKind::True: | |
| SeenActive = true; | |
| break; | |
| case BooleanKind::False: | |
| break; | |
| case BooleanKind::Poison: | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| } | |
| Res.push_back(AnyValue::boolean(!SeenActive)); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::vector_reduce_fadd: | |
| case Intrinsic::vector_reduce_fmul: | |
| case Intrinsic::vector_reduce_fmaximum: | |
| case Intrinsic::vector_reduce_fminimum: | |
| case Intrinsic::vector_reduce_fmaximumnum: | |
| case Intrinsic::vector_reduce_fminimumnum: | |
| case Intrinsic::vector_reduce_fmax: | |
| case Intrinsic::vector_reduce_fmin: | |
| return callVectorReduction(RetTy, FMF, IID, Args); | |
| case Intrinsic::fabs: { | |
| return visitBitwiseFPUnOpWithResult( | |
| RetTy, FMF, Args[0], | |
| [](const APFloat &Operand) -> APFloat { return abs(Operand); }); | |
| } | |
| case Intrinsic::fma: { | |
| return visitFPTriOpWithResult( | |
| RetTy, FMF, Args[0], Args[1], Args[2], | |
| [](const APFloat &Op1, const APFloat &Op2, | |
| const APFloat &Op3) -> APFloat { | |
| auto Res = Op1; | |
| Res.fusedMultiplyAdd(Op2, Op3, RoundingMode::NearestTiesToEven); | |
| return Res; | |
| }); | |
| } | |
| case Intrinsic::fmuladd: { | |
| return visitFPTriOpWithResult( | |
| RetTy, FMF, Args[0], Args[1], Args[2], | |
| [&](const APFloat &Op1, const APFloat &Op2, | |
| const APFloat &Op3) -> APFloat { | |
| if (Ctx.fuseMultiplyAdd()) { | |
| auto Res = Op1; | |
| Res.fusedMultiplyAdd(Op2, Op3, RoundingMode::NearestTiesToEven); | |
| return Res; | |
| } | |
| return Op1 * Op2 + Op3; | |
| }); | |
| } | |
| case Intrinsic::is_fpclass: { | |
| const FPClassTest Mask = | |
| static_cast<FPClassTest>(Args[1].asInteger().getZExtValue()); | |
| return computeUnOp(RetTy, Args[0], [&](const AnyValue &Op) -> AnyValue { | |
| if (Op.isPoison()) | |
| return AnyValue::poison(); | |
| return AnyValue::boolean( | |
| static_cast<bool>(Op.asFloat().classify() & Mask)); | |
| }); | |
| } | |
| case Intrinsic::copysign: { | |
| return computeBinOp( | |
| RetTy, Args[0], Args[1], | |
| [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue { | |
| if (LHS.isPoison() || RHS.isPoison()) | |
| return AnyValue::poison(); | |
| const AnyValue ValidatedLHS = | |
| handleFMFFlags(LHS, FMF, /*IsInput=*/true); | |
| const AnyValue ValidatedRHS = | |
| handleFMFFlags(RHS, FMF, /*IsInput=*/true); | |
| if (ValidatedLHS.isPoison() || ValidatedRHS.isPoison()) | |
| return AnyValue::poison(); | |
| return handleFMFFlags(APFloat::copySign(ValidatedLHS.asFloat(), | |
| ValidatedRHS.asFloat()), | |
| FMF, /*IsInput=*/false); | |
| }); | |
| } | |
| case Intrinsic::maxnum: | |
| case Intrinsic::minnum: | |
| case Intrinsic::maximum: | |
| case Intrinsic::minimum: | |
| case Intrinsic::maximumnum: | |
| case Intrinsic::minimumnum: { | |
| return visitFPBinOpWithResult( | |
| RetTy, FMF, Args[0], Args[1], | |
| [&](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| switch (IID) { | |
| case Intrinsic::maximum: | |
| return maximum(LHS, RHS); | |
| case Intrinsic::minimum: | |
| return minimum(LHS, RHS); | |
| case Intrinsic::maximumnum: | |
| return maximumnum(LHS, RHS); | |
| case Intrinsic::minimumnum: | |
| return minimumnum(LHS, RHS); | |
| case Intrinsic::maxnum: | |
| return maxnumWithSNaNQuieting(LHS, RHS); | |
| case Intrinsic::minnum: | |
| return minnumWithSNaNQuieting(LHS, RHS); | |
| default: | |
| llvm_unreachable("Unexpected intrinsic ID"); | |
| } | |
| }); | |
| } | |
| case Intrinsic::fptosi_sat: | |
| case Intrinsic::fptoui_sat: { | |
| const auto BitWidth = RetTy->getScalarSizeInBits(); | |
| return computeUnOp(RetTy, Args[0], [&](const AnyValue &Op) -> AnyValue { | |
| if (Op.isPoison()) | |
| return AnyValue::poison(); | |
| const APFloat &Operand = Op.asFloat(); | |
| APSInt V(BitWidth, IID == Intrinsic::fptoui_sat); | |
| [[maybe_unused]] bool IsExact; | |
| Operand.convertToInteger(V, APFloat::rmTowardZero, &IsExact); | |
| return V; | |
| }); | |
| } | |
| case Intrinsic::memcpy: | |
| case Intrinsic::memcpy_inline: | |
| case Intrinsic::memmove: | |
| return callMemTransferIntrinsic(CB, Args, IID); | |
| case Intrinsic::memset: | |
| case Intrinsic::memset_inline: | |
| return callMemSetIntrinsic(CB, Args); | |
| case Intrinsic::speculative_load: | |
| assert(!getSpeculativeLoadOracle(CB) && | |
| "oracle form must be handled earlier"); | |
| return callSpeculativeLoadIntrinsic(CB, Args[0], Args[2]); | |
| case Intrinsic::masked_load: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[0], Args[1], Args[2], | |
| MaskedAddrMode::Consecutive, | |
| /*IsStore=*/false); | |
| case Intrinsic::masked_store: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[1], Args[2], Args[0], | |
| MaskedAddrMode::Consecutive, | |
| /*IsStore=*/true); | |
| case Intrinsic::masked_gather: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[0], Args[1], Args[2], | |
| MaskedAddrMode::PerLane, | |
| /*IsStore=*/false); | |
| case Intrinsic::masked_scatter: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[1], Args[2], Args[0], | |
| MaskedAddrMode::PerLane, | |
| /*IsStore=*/true); | |
| case Intrinsic::masked_expandload: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[0], Args[1], Args[2], | |
| MaskedAddrMode::Compressed, | |
| /*IsStore=*/false); | |
| case Intrinsic::masked_compressstore: | |
| return callMaskedLoadStoreIntrinsic(CB, Args[1], Args[2], Args[0], | |
| MaskedAddrMode::Compressed, | |
| /*IsStore=*/true); | |
| case Intrinsic::vp_load: | |
| return callVPLoadStoreIntrinsic( | |
| CB, withVPDefaultAlign(CB, Args[0], RetTy), Args[1], Args[2], | |
| AnyValue::getPoisonValue(Ctx, RetTy), MaskedAddrMode::Consecutive, | |
| /*IsStore=*/false); | |
| case Intrinsic::vp_store: | |
| return callVPLoadStoreIntrinsic( | |
| CB, withVPDefaultAlign(CB, Args[1], CB.getArgOperand(0)->getType()), | |
| Args[2], Args[3], Args[0], MaskedAddrMode::Consecutive, | |
| /*IsStore=*/true); | |
| case Intrinsic::vp_gather: | |
| return callVPLoadStoreIntrinsic( | |
| CB, withVPDefaultAlign(CB, Args[0], RetTy->getScalarType()), Args[1], | |
| Args[2], AnyValue::getPoisonValue(Ctx, RetTy), | |
| MaskedAddrMode::PerLane, | |
| /*IsStore=*/false); | |
| case Intrinsic::vp_scatter: | |
| return callVPLoadStoreIntrinsic( | |
| CB, | |
| withVPDefaultAlign(CB, Args[1], | |
| CB.getArgOperand(0)->getType()->getScalarType()), | |
| Args[2], Args[3], Args[0], MaskedAddrMode::PerLane, | |
| /*IsStore=*/true); | |
| case Intrinsic::experimental_vp_strided_load: | |
| return callVPStridedLoadStoreIntrinsic( | |
| CB, withVPDefaultAlign(CB, Args[0], RetTy->getScalarType()), Args[1], | |
| Args[2], Args[3], AnyValue::getPoisonValue(Ctx, RetTy), | |
| /*IsStore=*/false); | |
| case Intrinsic::experimental_vp_strided_store: | |
| return callVPStridedLoadStoreIntrinsic( | |
| CB, | |
| withVPDefaultAlign(CB, Args[1], | |
| CB.getArgOperand(0)->getType()->getScalarType()), | |
| Args[2], Args[3], Args[4], Args[0], | |
| /*IsStore=*/true); | |
| case Intrinsic::vp_merge: { | |
| std::vector<AnyValue> Cond(Args[0].asAggregate()); | |
| const AnyValue &EVL = Args[3]; | |
| if (EVL.isPoison() || EVL.asInteger().ugt(Cond.size())) | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| for (uint64_t I = EVL.asInteger().getZExtValue(); I != Cond.size(); ++I) | |
| Cond[I] = AnyValue::boolean(false); | |
| return computeSelect(std::move(Cond), Args[1], Args[2], RetTy, FMF); | |
| } | |
| case Intrinsic::matrix_transpose: { | |
| const auto &Vec = Args[0].asAggregate(); | |
| const unsigned Rows = | |
| cast<ConstantInt>(CB.getArgOperand(1))->getZExtValue(); | |
| const unsigned Cols = | |
| cast<ConstantInt>(CB.getArgOperand(2))->getZExtValue(); | |
| // Column I of the result is row I of the column-major input. | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Vec.size()); | |
| for (unsigned I = 0; I != Rows; ++I) | |
| for (unsigned J = 0; J != Cols; ++J) | |
| Res.push_back(Vec[J * Rows + I]); | |
| return std::move(Res); | |
| } | |
| case Intrinsic::matrix_column_major_store: | |
| return callMatrixColumnMajorStoreIntrinsic(CB, Args[1], Args[2], Args[0]); | |
| case Intrinsic::experimental_noalias_scope_decl: | |
| // FIXME: Not implemented yet. Currently it acts as a noop. | |
| return AnyValue(); | |
| case Intrinsic::experimental_cttz_elts: { | |
| const auto &Vec = Args[0].asAggregate(); | |
| return countTrailingZeroElements(RetTy, Vec, Args[1].asInteger().isOne(), | |
| Vec.size()); | |
| } | |
| case Intrinsic::experimental_get_vector_length: { | |
| auto *VFC = cast<ConstantInt>(CB.getArgOperand(1)); | |
| auto *ScalableC = cast<ConstantInt>(CB.getArgOperand(2)); | |
| if (Args[0].isPoison()) | |
| return AnyValue::poison(); | |
| const APInt &Cnt = Args[0].asInteger(); | |
| uint64_t MaxLanes = VFC->getZExtValue(); | |
| if (ScalableC->isOne()) | |
| MaxLanes *= Ctx.getVScale(); | |
| uint64_t Res = 0; | |
| if (!Cnt.isZero()) { | |
| if (Cnt.getActiveBits() <= 64 && Cnt.getZExtValue() <= MaxLanes) { | |
| Res = Cnt.getZExtValue(); | |
| } else { | |
| APInt Max(Cnt.getBitWidth(), MaxLanes); | |
| APInt NumIters = | |
| APIntOps::RoundingUDiv(Cnt, Max, APInt::Rounding::UP); | |
| uint64_t Lower = | |
| APIntOps::RoundingUDiv(Cnt, NumIters, APInt::Rounding::UP) | |
| .getZExtValue(); | |
| uint64_t Range = MaxLanes - Lower + 1; | |
| Res = Lower + Ctx.getRandomUInt64() % Range; | |
| } | |
| } | |
| if (isIntN(32, Res)) | |
| return APInt(32, Res); | |
| return AnyValue::poison(); | |
| } | |
| case Intrinsic::experimental_vector_extract_last_active: { | |
| const auto &Data = Args[0].asAggregate(); | |
| const AnyValue &Mask = Args[1]; | |
| for (size_t I = Data.size(); I != 0; --I) { | |
| switch (getMaskLane(Mask, I - 1)) { | |
| case BooleanKind::True: | |
| return Data[I - 1]; | |
| case BooleanKind::False: | |
| break; | |
| case BooleanKind::Poison: | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| } | |
| } | |
| return Args[2]; | |
| } | |
| case Intrinsic::experimental_vector_compress: { | |
| const auto &Val = Args[0].asAggregate(); | |
| const AnyValue &Mask = Args[1]; | |
| const auto &Passthru = Args[2].asAggregate(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Val.size()); | |
| for (size_t I = 0, E = Val.size(); I != E; ++I) { | |
| switch (getMaskLane(Mask, I)) { | |
| case BooleanKind::True: | |
| Res.push_back(Val[I]); | |
| break; | |
| case BooleanKind::False: | |
| break; | |
| case BooleanKind::Poison: | |
| return AnyValue::getPoisonValue(Ctx, RetTy); | |
| } | |
| } | |
| for (size_t I = Res.size(), E = Val.size(); I != E; ++I) | |
| Res.push_back(Passthru[I]); | |
| return std::move(Res); | |
| } | |
| case Intrinsic::experimental_vector_match: { | |
| const auto &Search = Args[0].asAggregate(); | |
| const auto &Needles = Args[1].asAggregate(); | |
| const auto &Mask = Args[2].asAggregate(); | |
| std::vector<AnyValue> Res; | |
| Res.reserve(Search.size()); | |
| for (size_t I = 0, E = Search.size(); I != E; ++I) { | |
| switch (Mask[I].asBoolean()) { | |
| case BooleanKind::False: | |
| Res.push_back(AnyValue::boolean(false)); | |
| continue; | |
| case BooleanKind::Poison: | |
| Res.push_back(AnyValue::poison()); | |
| continue; | |
| case BooleanKind::True: | |
| break; | |
| } | |
| if (Search[I].isPoison()) { | |
| Res.push_back(AnyValue::poison()); | |
| continue; | |
| } | |
| bool Found = false; | |
| bool SawPoison = false; | |
| for (const AnyValue &Needle : Needles) { | |
| if (Needle.isPoison()) { | |
| SawPoison = true; | |
| break; | |
| } | |
| if (Search[I].asInteger() == Needle.asInteger()) | |
| Found = true; | |
| } | |
| if (SawPoison) | |
| Res.push_back(AnyValue::poison()); | |
| else | |
| Res.push_back(AnyValue::boolean(Found)); | |
| } | |
| return std::move(Res); | |
| } | |
| case Intrinsic::experimental_vector_histogram_add: | |
| case Intrinsic::experimental_vector_histogram_uadd_sat: | |
| case Intrinsic::experimental_vector_histogram_umax: | |
| case Intrinsic::experimental_vector_histogram_umin: | |
| return callExperimentalVectorHistogramIntrinsic(CB, Args, IID); | |
| default: | |
| Handler.onUnrecognizedInstruction(CB); | |
| setFailed(); | |
| return AnyValue(); | |
| } | |
| } | |
| AnyValue callLibFunc(CallBase &CB, Function *ResolvedCallee, | |
| ArrayRef<AnyValue> CalleeArgs) { | |
| LibFunc LF = CurrentFrame->TLI.getLibFunc(*ResolvedCallee); | |
| // Respect nobuiltin attributes on call site. | |
| if (CB.isNoBuiltin() || LF == NotLibFunc) { | |
| Handler.onUnrecognizedInstruction(CB); | |
| setFailed(); | |
| return AnyValue(); | |
| } | |
| if (auto LibCallRes = Lib.executeLibcall(LF, CB.getName(), | |
| CB.getFunctionType(), CalleeArgs)) | |
| return *LibCallRes; | |
| if (ExitInfo) | |
| return AnyValue(); | |
| Handler.onUnrecognizedInstruction(CB); | |
| setFailed(); | |
| return AnyValue(); | |
| } | |
| /// Handle both poison-generating and UB-implying attributes for parameters | |
| /// and return values. | |
| void handleAttributes(Type *Ty, AnyValue &V, AttributeSet AttrsAtCallSite, | |
| AttributeSet AttrsAtCallee) { | |
| if (Ty->isIntOrIntVectorTy()) { | |
| if (auto CRAttr = AttrsAtCallSite.getAttribute(Attribute::Range); | |
| CRAttr.isValid()) | |
| applyRangeAttr(V, CRAttr.getRange()); | |
| if (auto CRAttr = AttrsAtCallee.getAttribute(Attribute::Range); | |
| CRAttr.isValid()) | |
| applyRangeAttr(V, CRAttr.getRange()); | |
| } | |
| if (AttributeFuncs::isNoFPClassCompatibleType(Ty)) { | |
| if (auto CRAttr = AttrsAtCallSite.getAttribute(Attribute::NoFPClass); | |
| CRAttr.isValid()) | |
| applyNoFPClassAttr(V, CRAttr.getNoFPClass()); | |
| if (auto CRAttr = AttrsAtCallee.getAttribute(Attribute::NoFPClass); | |
| CRAttr.isValid()) | |
| applyNoFPClassAttr(V, CRAttr.getNoFPClass()); | |
| } | |
| if (Ty->isPointerTy()) { | |
| if (AttrsAtCallSite.hasAttribute(Attribute::NonNull) || | |
| AttrsAtCallee.hasAttribute(Attribute::NonNull)) | |
| applyNonNullAttr(V, Ty->getPointerAddressSpace(), DL); | |
| } | |
| if (Ty->isPtrOrPtrVectorTy()) { | |
| if (MaybeAlign Align = AttrsAtCallSite.getAlignment()) | |
| applyAlignAttr(V, *Align); | |
| if (MaybeAlign Align = AttrsAtCallee.getAlignment()) | |
| applyAlignAttr(V, *Align); | |
| } | |
| if ((AttrsAtCallSite.hasAttribute(Attribute::NoUndef) || | |
| AttrsAtCallee.hasAttribute(Attribute::NoUndef)) && | |
| violatesNoUndefAttr(V)) { | |
| reportImmediateUB() << "The value " << V | |
| << " violates noundef attribute."; | |
| return; | |
| } | |
| if (Ty->isPointerTy()) { | |
| unsigned AS = Ty->getPointerAddressSpace(); | |
| if (uint64_t DereferenceableBytes = | |
| std::max(AttrsAtCallSite.getDereferenceableBytes(), | |
| AttrsAtCallee.getDereferenceableBytes())) { | |
| if (violatesDereferenceableBytesAttr(V, DereferenceableBytes, | |
| /*OrNull=*/false, AS, Ctx)) | |
| reportImmediateUB() | |
| << "The value " << V << " violates dereferenceable(" | |
| << DereferenceableBytes << ") attribute."; | |
| } | |
| if (uint64_t DereferenceableOrNullBytes = | |
| std::max(AttrsAtCallSite.getDereferenceableOrNullBytes(), | |
| AttrsAtCallee.getDereferenceableOrNullBytes())) { | |
| if (violatesDereferenceableBytesAttr(V, DereferenceableOrNullBytes, | |
| /*OrNull=*/true, AS, Ctx)) | |
| reportImmediateUB() << "The value " << V | |
| << " violates " | |
| "dereferenceable_or_null(" | |
| << DereferenceableOrNullBytes << ") attribute."; | |
| } | |
| } | |
| } | |
| /// Handle both poison-generating and UB-implying metadata on instructions. | |
| void handleMetadata(Type *Ty, AnyValue &V, Instruction &I) { | |
| auto ExtractFirstIntOperand = [](const MDNode *Node) { | |
| return mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue(); | |
| }; | |
| if (Ty->isIntOrIntVectorTy()) { | |
| if (MDNode *Ranges = I.getMetadata(LLVMContext::MD_range)) { | |
| SmallVector<ConstantRange> RangeList; | |
| for (uint32_t I = 0; I < Ranges->getNumOperands(); I += 2) { | |
| RangeList.emplace_back( | |
| mdconst::extract<ConstantInt>(Ranges->getOperand(I))->getValue(), | |
| mdconst::extract<ConstantInt>(Ranges->getOperand(I + 1)) | |
| ->getValue()); | |
| } | |
| forEachScalarValue(V, [&](AnyValue &Scalar) { | |
| if (!Scalar.isInteger()) | |
| return; | |
| for (auto &CR : RangeList) | |
| if (CR.contains(Scalar.asInteger())) | |
| return; | |
| Scalar = AnyValue::poison(); | |
| }); | |
| } | |
| } | |
| if (AttributeFuncs::isNoFPClassCompatibleType(Ty)) { | |
| if (const MDNode *NoFPClass = I.getMetadata(LLVMContext::MD_nofpclass)) { | |
| applyNoFPClassAttr( | |
| V, static_cast<FPClassTest>(ExtractFirstIntOperand(NoFPClass))); | |
| } | |
| } | |
| if (Ty->isPointerTy()) { | |
| if (I.hasMetadata(LLVMContext::MD_nonnull)) | |
| applyNonNullAttr(V, Ty->getPointerAddressSpace(), DL); | |
| // Unlike align attributes, !align is only defined for pointer types. | |
| if (const MDNode *Alignment = I.getMetadata(LLVMContext::MD_align)) | |
| applyAlignAttr(V, Align(ExtractFirstIntOperand(Alignment))); | |
| } | |
| if (I.hasMetadata(LLVMContext::MD_noundef) && violatesNoUndefAttr(V)) { | |
| reportImmediateUB() << "The value " << V | |
| << " violates !noundef metadata."; | |
| return; | |
| } | |
| if (Ty->isPointerTy()) { | |
| unsigned AS = Ty->getPointerAddressSpace(); | |
| if (const MDNode *DereferenceableBytes = | |
| I.getMetadata(LLVMContext::MD_dereferenceable)) { | |
| uint64_t Bytes = ExtractFirstIntOperand(DereferenceableBytes); | |
| if (violatesDereferenceableBytesAttr(V, Bytes, | |
| /*OrNull=*/false, AS, Ctx)) | |
| reportImmediateUB() | |
| << "The value " << V << " violates !dereferenceable !{i64 " | |
| << Bytes << "} metadata."; | |
| } else if (const MDNode *DereferenceableOrNullBytes = | |
| I.getMetadata(LLVMContext::MD_dereferenceable_or_null)) { | |
| uint64_t Bytes = ExtractFirstIntOperand(DereferenceableOrNullBytes); | |
| if (violatesDereferenceableBytesAttr(V, Bytes, | |
| /*OrNull=*/true, AS, Ctx)) | |
| reportImmediateUB() | |
| << "The value " << V << " violates !dereferenceable_or_null!{i64 " | |
| << Bytes << "} metadata."; | |
| } | |
| } | |
| } | |
| void enterCall(CallBase &CB) { | |
| Function *Callee = CB.getCalledFunction(); | |
| // TODO: handle initializes | |
| auto &CalleeArgs = CurrentFrame->CalleeArgs; | |
| assert(CalleeArgs.empty() && | |
| "Forgot to call returnFromCallee before entering a new call."); | |
| for (Value *Arg : CB.args()) | |
| CalleeArgs.push_back(getValue(Arg)); | |
| if (!Callee) { | |
| Value *CalledOperand = CB.getCalledOperand(); | |
| if (isNoopInlineAsm(CalledOperand, CB.getType())) { | |
| CurrentFrame->ResolvedCallee = nullptr; | |
| CurrentFrame->CalleeRetVal = AnyValue(); | |
| returnFromCallee(); | |
| return; | |
| } | |
| if (isa<InlineAsm>(CalledOperand)) { | |
| Handler.onUnrecognizedInstruction(CB); | |
| setFailed(); | |
| return; | |
| } | |
| auto &CalleeVal = getValue(CalledOperand); | |
| if (CalleeVal.isPoison()) { | |
| reportImmediateUB() << "Indirect call through poison function pointer."; | |
| return; | |
| } | |
| Callee = Ctx.getTargetFunction(CalleeVal.asPointer()); | |
| if (!Callee) { | |
| reportImmediateUB() | |
| << "Indirect call through invalid function pointer."; | |
| return; | |
| } | |
| if (Callee->getFunctionType() != CB.getFunctionType()) { | |
| reportImmediateUB() << "Indirect call through a function pointer with " | |
| "mismatched signature. Expected: " | |
| << *CB.getFunctionType() | |
| << ", Actual: " << *Callee->getFunctionType(); | |
| return; | |
| } | |
| } | |
| assert(Callee && "Expected a resolved callee function."); | |
| assert( | |
| Callee->getFunctionType() == CB.getFunctionType() && | |
| "Expected the callee function type to match the call site signature."); | |
| // Handle parameter attributes (Attributes from resolved callee should be | |
| // applied if available). | |
| for (auto [I, Arg] : enumerate(CB.args())) { | |
| Type *ArgTy = Arg->getType(); | |
| AnyValue &ArgVal = CalleeArgs[I]; | |
| // CallBase::paramHasAttr also checks parameter attributes at known | |
| // callee. We do it explicitly to avoid duplication. | |
| AttributeSet AttrsAtCallSite = CB.getParamAttributes(I); | |
| AttributeSet AttrsAtCallee = Callee->getAttributes().getParamAttrs(I); | |
| if (ArgTy->isPointerTy()) { | |
| auto *ByValTy = AttrsAtCallSite.getByValType(); | |
| auto *ByValTyFromCallee = AttrsAtCallee.getByValType(); | |
| if (ByValTy != ByValTyFromCallee) { | |
| reportImmediateUB() | |
| << "Mismatched byval attribute between callee and callsite."; | |
| return; | |
| } | |
| if (ByValTy) { | |
| if (ArgVal.isPoison()) { | |
| reportImmediateUB() << "Invalid poison byval pointer argument."; | |
| return; | |
| } | |
| uint64_t Size = Ctx.getEffectiveTypeAllocSize(ByValTy); | |
| MaybeAlign AllocAlign = AttrsAtCallSite.getAlignment(); | |
| // Ignore the alignment at the callsite when it is set on the callee. | |
| if (MaybeAlign CalleeAlign = AttrsAtCallee.getAlignment()) | |
| AllocAlign = CalleeAlign; | |
| if (!AllocAlign.has_value()) { | |
| // If the alignment is not specified, we use the default ABI | |
| // alignment. This is the default behavior of | |
| // TargetLoweringBase::getByValTypeAlignment. | |
| AllocAlign = DL.getABITypeAlign(ByValTy); | |
| } | |
| assert(I < Callee->arg_size() && | |
| "Byval pointers cannot be passed via variadic arguments."); | |
| if (Ctx.exceedsMaxObjectSize(Size, ArgTy->getPointerAddressSpace())) { | |
| reportImmediateUB() | |
| << "Byval argument with size that exceeds the maximum object " | |
| "size. Size: " | |
| << Size; | |
| return; | |
| } | |
| auto Obj = Ctx.allocate( | |
| Size, AllocAlign->value(), Callee->getArg(I)->getName(), | |
| ArgTy->getPointerAddressSpace(), MemInitKind::Uninitialized, | |
| MemAllocKind::Stack); | |
| if (!Obj) { | |
| reportError() | |
| << "Insufficient stack space for byval pointer argument."; | |
| return; | |
| } | |
| if (auto [MO, Offset] = verifyMemAccess( | |
| ArgVal.asPointer(), Size, | |
| std::max(AllocAlign.value(), | |
| AttrsAtCallSite.getAlignment().valueOrOne()), | |
| /*IsStore=*/false); | |
| MO) | |
| copy(MO->getBytes().slice(Offset, Size), Obj->getBytes().begin()); | |
| else | |
| return; | |
| CurrentFrame->CalleeByValArgs.push_back(Obj); | |
| ArgVal = Ctx.deriveFromMemoryObject(std::move(Obj)); | |
| } | |
| } | |
| handleAttributes(ArgTy, ArgVal, AttrsAtCallSite, AttrsAtCallee); | |
| } | |
| CurrentFrame->ResolvedCallee = Callee; | |
| ArrayRef<AnyValue> Args = CalleeArgs; | |
| if (Function *Oracle = getSpeculativeLoadOracle(CB)) { | |
| Args = Args.drop_front(3); | |
| for (auto [Arg, ArgVal] : | |
| zip_equal(Oracle->args(), MutableArrayRef(CalleeArgs).drop_front(3))) | |
| handleAttributes(Arg.getType(), ArgVal, AttributeSet(), | |
| Arg.getAttributes()); | |
| Callee = Oracle; | |
| } | |
| if (Callee->isIntrinsic()) { | |
| CurrentFrame->CalleeRetVal = callIntrinsic(CB, CalleeArgs); | |
| returnFromCallee(); | |
| return; | |
| } else if (Callee->isDeclaration()) { | |
| CurrentFrame->CalleeRetVal = callLibFunc(CB, Callee, CalleeArgs); | |
| returnFromCallee(); | |
| return; | |
| } else { | |
| uint32_t MaxStackDepth = Ctx.getMaxStackDepth(); | |
| if (MaxStackDepth && CallStack.size() >= MaxStackDepth) { | |
| reportError() << "Maximum stack depth exceeded."; | |
| return; | |
| } | |
| assert(!Callee->empty() && "Expected a defined function."); | |
| // Suspend the current frame and push the callee frame onto the stack. | |
| AnyValue &RetVal = CurrentFrame->CalleeRetVal; | |
| CurrentFrame->State = FrameState::Pending; | |
| CallStack.emplace_back(*Callee, &CB, CurrentFrame, Args, RetVal, | |
| Ctx.getTLIImpl()); | |
| } | |
| } | |
| void visitCallInst(CallInst &CI) { enterCall(CI); } | |
| void visitInvokeInst(InvokeInst &II) { | |
| // TODO: handle exceptions | |
| enterCall(II); | |
| } | |
| void visitAdd(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) { | |
| return addNoWrap(LHS, RHS, I.hasNoSignedWrap(), I.hasNoUnsignedWrap()); | |
| }); | |
| } | |
| void visitSub(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) { | |
| return subNoWrap(LHS, RHS, I.hasNoSignedWrap(), I.hasNoUnsignedWrap()); | |
| }); | |
| } | |
| void visitMul(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) { | |
| return mulNoWrap(LHS, RHS, I.hasNoSignedWrap(), I.hasNoUnsignedWrap()); | |
| }); | |
| } | |
| void visitSDiv(BinaryOperator &I) { | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) { | |
| return computeDivRem(I.getOpcode(), LHS, RHS, I.isExact()); | |
| }); | |
| } | |
| void visitSRem(BinaryOperator &I) { | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) { | |
| return computeDivRem(I.getOpcode(), LHS, RHS); | |
| }); | |
| } | |
| void visitUDiv(BinaryOperator &I) { | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) { | |
| return computeDivRem(I.getOpcode(), LHS, RHS, I.isExact()); | |
| }); | |
| } | |
| void visitURem(BinaryOperator &I) { | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) { | |
| return computeDivRem(I.getOpcode(), LHS, RHS); | |
| }); | |
| } | |
| void visitFAdd(BinaryOperator &I) { | |
| visitFPBinOp(I, [](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| APFloat Res = LHS; | |
| Res.add(RHS, APFloat::rmNearestTiesToEven); | |
| return Res; | |
| }); | |
| } | |
| void visitFSub(BinaryOperator &I) { | |
| visitFPBinOp(I, [](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| APFloat Res = LHS; | |
| Res.subtract(RHS, APFloat::rmNearestTiesToEven); | |
| return Res; | |
| }); | |
| } | |
| void visitFMul(BinaryOperator &I) { | |
| visitFPBinOp(I, [](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| APFloat Res = LHS; | |
| Res.multiply(RHS, APFloat::rmNearestTiesToEven); | |
| return Res; | |
| }); | |
| } | |
| void visitFDiv(BinaryOperator &I) { | |
| visitFPBinOp(I, [](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| APFloat Res = LHS; | |
| Res.divide(RHS, APFloat::rmNearestTiesToEven); | |
| return Res; | |
| }); | |
| } | |
| void visitFRem(BinaryOperator &I) { | |
| visitFPBinOp(I, [](const APFloat &LHS, const APFloat &RHS) -> APFloat { | |
| APFloat Res = LHS; | |
| Res.mod(RHS); | |
| return Res; | |
| }); | |
| } | |
| void visitFNeg(UnaryOperator &I) { | |
| visitBitwiseFPUnOp( | |
| I, [](const APFloat &Operand) -> APFloat { return -Operand; }); | |
| } | |
| void visitTruncInst(TruncInst &Trunc) { | |
| visitIntUnOp(Trunc, [&](const APInt &Operand) -> AnyValue { | |
| unsigned DestBW = Trunc.getType()->getScalarSizeInBits(); | |
| if (Trunc.hasNoSignedWrap() && Operand.getSignificantBits() > DestBW) | |
| return AnyValue::poison(); | |
| if (Trunc.hasNoUnsignedWrap() && Operand.getActiveBits() > DestBW) | |
| return AnyValue::poison(); | |
| return Operand.trunc(DestBW); | |
| }); | |
| } | |
| void visitZExtInst(ZExtInst &ZExt) { | |
| visitIntUnOp(ZExt, [&](const APInt &Operand) -> AnyValue { | |
| uint32_t DestBW = ZExt.getDestTy()->getScalarSizeInBits(); | |
| if (ZExt.hasNonNeg() && Operand.isNegative()) | |
| return AnyValue::poison(); | |
| return Operand.zext(DestBW); | |
| }); | |
| } | |
| void visitSExtInst(SExtInst &SExt) { | |
| visitIntUnOp(SExt, [&](const APInt &Operand) -> AnyValue { | |
| uint32_t DestBW = SExt.getDestTy()->getScalarSizeInBits(); | |
| return Operand.sext(DestBW); | |
| }); | |
| } | |
| void visitFPExtInst(FPExtInst &FPExt) { visitFPConvInst(FPExt); } | |
| void visitFPTruncInst(FPTruncInst &FPTrunc) { visitFPConvInst(FPTrunc); } | |
| void visitFPConvInst(Instruction &I) { | |
| if (!Ctx.isDefaultFPEnv()) | |
| reportImmediateUB() << "Non-constrained floating-point operation assumes " | |
| "default floating-point environment"; | |
| const fltSemantics &DstSem = | |
| I.getType()->getScalarType()->getFltSemantics(); | |
| visitUnOp(I, [&](const AnyValue &Operand) -> AnyValue { | |
| if (Operand.isPoison()) | |
| return AnyValue::poison(); | |
| FastMathFlags FMF = cast<FPMathOperator>(I).getFastMathFlags(); | |
| DenormalMode DenormMode = | |
| getCurrentDenormalMode(I.getOperand(0)->getType()); | |
| auto ValidatedOperand = handleFMFFlags(Operand, FMF, /*IsInput=*/true); | |
| if (ValidatedOperand.isPoison()) | |
| return ValidatedOperand; | |
| APFloat FOperand = handleDenormal(ValidatedOperand.asFloat(), | |
| DenormMode.Input, /*IsInput=*/true); | |
| APFloat SourceNaN = FOperand; | |
| bool LosesInfo; | |
| FOperand.convert(DstSem, Ctx.getCurrentRoundingMode(), &LosesInfo); | |
| if (auto ValidateRes = handleFMFFlags(FOperand, FMF, /*IsInput=*/false); | |
| ValidateRes.isPoison()) | |
| return ValidateRes; | |
| FOperand = handleDenormal(std::move(FOperand), DenormMode.Output, true); | |
| return AnyValue(applyNaNPropagation(FOperand, {&SourceNaN})); | |
| }); | |
| } | |
| void visitFPToSIInst(FPToSIInst &FPToSI) { | |
| visitFPToIntInst(FPToSI, /*IsUnsigned=*/false); | |
| } | |
| void visitFPToUIInst(FPToUIInst &FPToUI) { | |
| visitFPToIntInst(FPToUI, /*IsUnsigned=*/true); | |
| } | |
| void visitFPToIntInst(Instruction &I, bool IsUnsigned) { | |
| // Note: We DO NOT use CurrentRoundingMode here. | |
| // Language specs require truncation towards zero for FP-to-Int conversions. | |
| visitUnOp(I, [&](const AnyValue &Operand) -> AnyValue { | |
| if (Operand.isPoison()) | |
| return AnyValue::poison(); | |
| APSInt Res(I.getType()->getScalarSizeInBits(), /*isUnsigned=*/IsUnsigned); | |
| bool IsExact; | |
| APFloat::opStatus Status = Operand.asFloat().convertToInteger( | |
| Res, APFloat::rmTowardZero, &IsExact); | |
| if (Status == APFloat::opInvalidOp) | |
| return AnyValue::poison(); | |
| return AnyValue(Res); | |
| }); | |
| } | |
| void visitSIToFPInst(SIToFPInst &SIToFP) { | |
| visitIntToFPInst(SIToFP, /*IsSigned=*/true); | |
| } | |
| void visitUIToFPInst(UIToFPInst &UIToFP) { | |
| visitIntToFPInst(UIToFP, /*IsSigned=*/false); | |
| } | |
| void visitIntToFPInst(Instruction &I, bool IsSigned) { | |
| const fltSemantics &DstSem = | |
| I.getType()->getScalarType()->getFltSemantics(); | |
| FastMathFlags FMF = cast<FPMathOperator>(I).getFastMathFlags(); | |
| visitUnOp(I, [&](const AnyValue &Operand) -> AnyValue { | |
| if (Operand.isPoison()) | |
| return AnyValue::poison(); | |
| APInt IOperand = Operand.asInteger(); | |
| if (isa<UIToFPInst>(I) && I.hasNonNeg() && IOperand.isNegative()) | |
| return AnyValue::poison(); | |
| APFloat Res(DstSem); | |
| Res.convertFromAPInt(Operand.asInteger(), /*IsSigned=*/IsSigned, | |
| Ctx.getCurrentRoundingMode()); | |
| // We need IsInput=true here because the nsz flag applies to the output. | |
| return handleFMFFlags(Res, FMF, /*IsInput=*/true); | |
| }); | |
| } | |
| void visitAnd(BinaryOperator &I) { | |
| visitIntBinOp(I, [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return LHS & RHS; | |
| }); | |
| } | |
| void visitXor(BinaryOperator &I) { | |
| visitIntBinOp(I, [](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| return LHS ^ RHS; | |
| }); | |
| } | |
| void visitOr(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| if (cast<PossiblyDisjointInst>(I).isDisjoint() && LHS.intersects(RHS)) | |
| return AnyValue::poison(); | |
| return LHS | RHS; | |
| }); | |
| } | |
| void visitShl(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| if (RHS.uge(LHS.getBitWidth())) | |
| return AnyValue::poison(); | |
| if (I.hasNoSignedWrap() && RHS.uge(LHS.getNumSignBits())) | |
| return AnyValue::poison(); | |
| if (I.hasNoUnsignedWrap() && RHS.ugt(LHS.countl_zero())) | |
| return AnyValue::poison(); | |
| return LHS.shl(RHS); | |
| }); | |
| } | |
| void visitLShr(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| if (RHS.uge(LHS.getBitWidth()) || | |
| (cast<PossiblyExactOperator>(I).isExact() && | |
| RHS.ugt(LHS.countr_zero()))) | |
| return AnyValue::poison(); | |
| return LHS.lshr(RHS); | |
| }); | |
| } | |
| void visitAShr(BinaryOperator &I) { | |
| visitIntBinOp(I, [&](const APInt &LHS, const APInt &RHS) -> AnyValue { | |
| if (RHS.uge(LHS.getBitWidth()) || | |
| (cast<PossiblyExactOperator>(I).isExact() && | |
| RHS.ugt(LHS.countr_zero()))) | |
| return AnyValue::poison(); | |
| return LHS.ashr(RHS); | |
| }); | |
| } | |
| void visitICmpInst(ICmpInst &I) { | |
| Type *OpTy = I.getOperand(0)->getType(); | |
| unsigned AddrWidth = | |
| OpTy->isPtrOrPtrVectorTy() ? DL.getAddressSizeInBits(OpTy) : 0; | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue { | |
| if (LHS.isPoison() || RHS.isPoison()) | |
| return AnyValue::poison(); | |
| const APInt &LHSVal = LHS.isPointer() | |
| ? LHS.asPointer().address().trunc(AddrWidth) | |
| : LHS.asInteger(); | |
| const APInt &RHSVal = RHS.isPointer() | |
| ? RHS.asPointer().address().trunc(AddrWidth) | |
| : RHS.asInteger(); | |
| if (I.hasSameSign() && LHSVal.isNonNegative() != RHSVal.isNonNegative()) | |
| return AnyValue::poison(); | |
| return AnyValue::boolean( | |
| ICmpInst::compare(LHSVal, RHSVal, I.getPredicate())); | |
| }); | |
| } | |
| void visitFCmpInst(FCmpInst &I) { | |
| DenormalMode DenormMode = | |
| getCurrentDenormalMode(I.getOperand(0)->getType()); | |
| FastMathFlags FMF = I.getFastMathFlags(); | |
| visitBinOp(I, [&](const AnyValue &LHS, const AnyValue &RHS) -> AnyValue { | |
| if (LHS.isPoison() || RHS.isPoison()) | |
| return AnyValue::poison(); | |
| if (auto ValidateRes = handleFMFFlags(LHS, FMF, /*IsInput=*/true); | |
| ValidateRes.isPoison()) | |
| return ValidateRes; | |
| if (auto ValidateRes = handleFMFFlags(RHS, FMF, /*IsInput=*/true); | |
| ValidateRes.isPoison()) | |
| return ValidateRes; | |
| APFloat FLHS = | |
| handleDenormal(LHS.asFloat(), DenormMode.Input, /*IsInput=*/true); | |
| APFloat FRHS = | |
| handleDenormal(RHS.asFloat(), DenormMode.Input, /*IsInput=*/true); | |
| return AnyValue::boolean(FCmpInst::compare(FLHS, FRHS, I.getPredicate())); | |
| }); | |
| } | |
| /// Computes `select CondVal, TVVal, FVVal` of type \p Ty with fast-math flags | |
| /// \p FMF. \p CondVal is either an i1 or a vector of i1. | |
| AnyValue computeSelect(const AnyValue &CondVal, const AnyValue &TVVal, | |
| const AnyValue &FVVal, Type *Ty, FastMathFlags FMF) { | |
| AnyValue Res; | |
| if (!CondVal.isAggregate()) { | |
| switch (CondVal.asBoolean()) { | |
| case BooleanKind::True: | |
| Res = TVVal; | |
| break; | |
| case BooleanKind::False: | |
| Res = FVVal; | |
| break; | |
| case BooleanKind::Poison: | |
| Res = AnyValue::getPoisonValue(Ctx, Ty); | |
| break; | |
| } | |
| } else { | |
| auto &Cond = CondVal.asAggregate(); | |
| auto &TV = TVVal.asAggregate(); | |
| auto &FV = FVVal.asAggregate(); | |
| std::vector<AnyValue> ResVec; | |
| size_t Len = Cond.size(); | |
| ResVec.reserve(Len); | |
| for (uint32_t I = 0; I != Len; ++I) { | |
| switch (Cond[I].asBoolean()) { | |
| case BooleanKind::True: | |
| ResVec.push_back(TV[I]); | |
| break; | |
| case BooleanKind::False: | |
| ResVec.push_back(FV[I]); | |
| break; | |
| case BooleanKind::Poison: | |
| ResVec.push_back(AnyValue::getPoisonValue(Ctx, Ty->getScalarType())); | |
| break; | |
| } | |
| } | |
| Res = AnyValue(std::move(ResVec)); | |
| } | |
| // Handle fast-math flags | |
| if (FMF.any()) | |
| Res = handleFMFFlags(std::move(Res), FMF, /*IsInput=*/true); | |
| return Res; | |
| } | |
| void visitSelect(SelectInst &SI) { | |
| setResult(SI, computeSelect(getValue(SI.getCondition()), | |
| getValue(SI.getTrueValue()), | |
| getValue(SI.getFalseValue()), SI.getType(), | |
| SI.getFastMathFlagsOrNone())); | |
| } | |
| void visitAllocaInst(AllocaInst &AI) { | |
| uint64_t AllocSize = Ctx.getEffectiveTypeSize(AI.getAllocationBaseSize(DL)); | |
| if (AI.isArrayAllocation()) { | |
| auto &Size = getValue(AI.getArraySize()); | |
| if (Size.isPoison()) { | |
| reportImmediateUB() << "Alloca with poison array size."; | |
| return; | |
| } | |
| if (Size.asInteger().getActiveBits() > 64) { | |
| reportImmediateUB() | |
| << "Alloca with large array size that overflows uint64_t. Size: " | |
| << Size.asInteger(); | |
| return; | |
| } | |
| bool Overflowed = false; | |
| AllocSize = SaturatingMultiply(AllocSize, Size.asInteger().getZExtValue(), | |
| &Overflowed); | |
| if (Overflowed) { | |
| reportImmediateUB() | |
| << "Alloca with allocation size that overflows uint64_t. Size: " | |
| << Size.asInteger(); | |
| return; | |
| } | |
| } | |
| if (Ctx.exceedsMaxObjectSize(AllocSize, AI.getAddressSpace())) { | |
| reportImmediateUB() | |
| << "Alloca with allocation size that exceeds the maximum object " | |
| "size. Size: " | |
| << AllocSize; | |
| return; | |
| } | |
| // If it is used by llvm.lifetime.start, it should be initially dead. | |
| bool IsInitiallyDead = any_of(AI.users(), [](User *U) { | |
| return match(U, m_Intrinsic<Intrinsic::lifetime_start>()); | |
| }); | |
| auto Obj = Ctx.allocate(AllocSize, AI.getPointerAlignment(DL).value(), | |
| AI.getName(), AI.getAddressSpace(), | |
| IsInitiallyDead ? MemInitKind::Poisoned | |
| : MemInitKind::Uninitialized, | |
| MemAllocKind::Stack); | |
| if (!Obj) { | |
| reportError() << "Insufficient stack space."; | |
| return; | |
| } | |
| CurrentFrame->Allocas.push_back(Obj); | |
| setResult(AI, Ctx.deriveFromMemoryObject(Obj)); | |
| } | |
| void visitGetElementPtrInst(GetElementPtrInst &GEP) { | |
| setResult(GEP, Ctx.computeGEP(cast<GEPOperator>(GEP), | |
| [this](Value *V) -> const AnyValue & { | |
| return getValue(V); | |
| })); | |
| } | |
| void visitPtrToInt(PtrToIntInst &I) { | |
| unsigned BitWidth = I.getType()->getScalarSizeInBits(); | |
| return visitUnOp(I, [this, BitWidth](const AnyValue &V) -> AnyValue { | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| Ctx.exposeProvenance(V.asPointer().provenance()); | |
| return V.asPointer().address().zextOrTrunc(BitWidth); | |
| }); | |
| } | |
| void visitIntToPtr(IntToPtrInst &I) { | |
| return visitUnOp(I, [&](const AnyValue &V) -> AnyValue { | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| auto Prov = Ctx.getWildcardProvenance(); | |
| // TODO: check metadata | |
| return Pointer(std::move(Prov), | |
| V.asInteger().zextOrTrunc(DL.getPointerSizeInBits( | |
| I.getType()->getPointerAddressSpace()))); | |
| }); | |
| } | |
| void visitPtrToAddr(PtrToAddrInst &I) { | |
| unsigned BitWidth = I.getType()->getScalarSizeInBits(); | |
| return visitUnOp(I, [&](const AnyValue &V) -> AnyValue { | |
| if (V.isPoison()) | |
| return AnyValue::poison(); | |
| return V.asPointer().address().trunc(BitWidth); | |
| }); | |
| } | |
| void visitLoadInst(LoadInst &LI) { | |
| auto RetVal = load(getValue(LI.getPointerOperand()), LI.getAlign(), | |
| LI.getType(), LI.hasMetadata(LLVMContext::MD_noundef)); | |
| // TODO: track volatile loads | |
| handleMetadata(LI.getType(), RetVal, LI); | |
| setResult(LI, std::move(RetVal)); | |
| } | |
| void visitStoreInst(StoreInst &SI) { | |
| auto &Ptr = getValue(SI.getPointerOperand()); | |
| auto &Val = getValue(SI.getValueOperand()); | |
| // TODO: track volatile stores | |
| // TODO: handle metadata | |
| store(Ptr, SI.getAlign(), Val, SI.getValueOperand()->getType()); | |
| if (!hasProgramExited() && !Handler.onInstructionExecuted(SI, AnyValue())) | |
| setFailed(); | |
| } | |
| void visitInstruction(Instruction &I) { | |
| Handler.onUnrecognizedInstruction(I); | |
| setFailed(); | |
| } | |
| void visitExtractValueInst(ExtractValueInst &EVI) { | |
| auto &Res = getValue(EVI.getAggregateOperand()); | |
| const AnyValue *Pos = &Res; | |
| for (unsigned Idx : EVI.indices()) | |
| Pos = &Pos->asAggregate()[Idx]; | |
| setResult(EVI, *Pos); | |
| } | |
| void visitInsertValueInst(InsertValueInst &IVI) { | |
| AnyValue Res = getValue(IVI.getAggregateOperand()); | |
| AnyValue *Pos = &Res; | |
| for (unsigned Idx : IVI.indices()) | |
| Pos = &Pos->asAggregate()[Idx]; | |
| *Pos = getValue(IVI.getInsertedValueOperand()); | |
| setResult(IVI, std::move(Res)); | |
| } | |
| void visitInsertElementInst(InsertElementInst &IEI) { | |
| auto Res = getValue(IEI.getOperand(0)); | |
| auto &ResVec = Res.asAggregate(); | |
| auto &Idx = getValue(IEI.getOperand(2)); | |
| if (Idx.isPoison() || Idx.asInteger().uge(ResVec.size())) { | |
| setResult(IEI, AnyValue::getPoisonValue(Ctx, IEI.getType())); | |
| return; | |
| } | |
| ResVec[Idx.asInteger().getZExtValue()] = getValue(IEI.getOperand(1)); | |
| setResult(IEI, std::move(Res)); | |
| } | |
| void visitExtractElementInst(ExtractElementInst &EEI) { | |
| auto &SrcVec = getValue(EEI.getOperand(0)).asAggregate(); | |
| auto &Idx = getValue(EEI.getOperand(1)); | |
| if (Idx.isPoison() || Idx.asInteger().uge(SrcVec.size())) { | |
| setResult(EEI, AnyValue::getPoisonValue(Ctx, EEI.getType())); | |
| return; | |
| } | |
| setResult(EEI, SrcVec[Idx.asInteger().getZExtValue()]); | |
| } | |
| void visitShuffleVectorInst(ShuffleVectorInst &SVI) { | |
| auto &LHSVec = getValue(SVI.getOperand(0)).asAggregate(); | |
| auto &RHSVec = getValue(SVI.getOperand(1)).asAggregate(); | |
| uint32_t Size = cast<VectorType>(SVI.getOperand(0)->getType()) | |
| ->getElementCount() | |
| .getKnownMinValue(); | |
| std::vector<AnyValue> Res; | |
| uint32_t DstLen = Ctx.getEVL(SVI.getType()->getElementCount()); | |
| Res.reserve(DstLen); | |
| uint32_t Stride = SVI.getShuffleMask().size(); | |
| // For scalable vectors, we need to repeat the shuffle mask until we fill | |
| // the destination vector. | |
| for (uint32_t Off = 0; Off != DstLen; Off += Stride) { | |
| for (int Idx : SVI.getShuffleMask()) { | |
| if (Idx == PoisonMaskElem) | |
| Res.push_back( | |
| AnyValue::getPoisonValue(Ctx, SVI.getType()->getScalarType())); | |
| else if (Idx < static_cast<int>(Size)) | |
| Res.push_back(LHSVec[Idx]); | |
| else | |
| Res.push_back(RHSVec[Idx - Size]); | |
| } | |
| } | |
| setResult(SVI, std::move(Res)); | |
| } | |
| void visitBitCastInst(BitCastInst &BCI) { | |
| // The conversion is done as if the value had been stored to memory and read | |
| // back as the target type. | |
| SmallVector<Byte> Bytes; | |
| Bytes.resize(Ctx.getEffectiveTypeStoreSize(BCI.getType()), | |
| Byte::concrete(0)); | |
| Ctx.toBytes(getValue(BCI.getOperand(0)), BCI.getOperand(0)->getType(), | |
| Bytes); | |
| setResult(BCI, Ctx.fromBytes(Bytes, BCI.getType())); | |
| } | |
| void visitBitInsertInst(BitInsertInst &BII) { | |
| Type *BaseTy = BII.getType()->getScalarType(); | |
| Type *ValTy = BII.getOperand(1)->getType()->getScalarType(); | |
| uint64_t NumBits = DL.getTypeSizeInBits(ValTy).getFixedValue(); | |
| visitTriOp(BII, | |
| [&](const AnyValue &Base, const AnyValue &Val, | |
| const AnyValue &Offset) -> AnyValue { | |
| if (Offset.isPoison()) | |
| return AnyValue::getPoisonValue(Ctx, BaseTy); | |
| ByteValue Res = Base.asByte(); | |
| uint64_t OffsetInBits = Offset.asInteger().getZExtValue(); | |
| if (OffsetInBits + NumBits > Res.getBitWidth()) | |
| return AnyValue::getPoisonValue(Ctx, BaseTy); | |
| Ctx.insertBits(Res, OffsetInBits, Val, ValTy); | |
| return Res; | |
| }); | |
| } | |
| void visitBitExtractInst(BitExtractInst &BEI) { | |
| Type *Ty = BEI.getType()->getScalarType(); | |
| uint64_t NumBits = DL.getTypeSizeInBits(Ty).getFixedValue(); | |
| visitBinOp(BEI, | |
| [&](const AnyValue &Src, const AnyValue &Offset) -> AnyValue { | |
| if (Offset.isPoison()) | |
| return AnyValue::getPoisonValue(Ctx, Ty); | |
| const ByteValue &Bytes = Src.asByte(); | |
| uint64_t OffsetInBits = Offset.asInteger().getZExtValue(); | |
| if (OffsetInBits + NumBits > Bytes.getBitWidth()) | |
| return AnyValue::getPoisonValue(Ctx, Ty); | |
| return Ctx.extractBits(Bytes, OffsetInBits, Ty); | |
| }); | |
| } | |
| void visitFreezeInst(FreezeInst &FI) { | |
| AnyValue Val = getValue(FI.getOperand(0)); | |
| Ctx.freeze(Val, FI.getType()); | |
| setResult(FI, std::move(Val)); | |
| } | |
| /// This function implements the main interpreter loop. | |
| /// It handles function calls in a non-recursive manner to avoid stack | |
| /// overflows. | |
| ProgramExitInfo runMainLoop() { | |
| uint32_t MaxSteps = Ctx.getMaxSteps(); | |
| uint32_t Steps = 0; | |
| while (!hasProgramExited() && !CallStack.empty()) { | |
| Frame &Top = CallStack.back(); | |
| CurrentFrame = &Top; | |
| if (Top.State == FrameState::Entry) { | |
| Handler.onFunctionEntry(Top.Func, Top.Args, Top.CallSite); | |
| } else { | |
| assert(Top.State == FrameState::Pending && | |
| "Expected to return from a callee."); | |
| returnFromCallee(); | |
| } | |
| Top.State = FrameState::Running; | |
| // Interpreter loop inside a function | |
| while (!hasProgramExited()) { | |
| assert(Top.State == FrameState::Running && | |
| "Expected to be in running state."); | |
| if (MaxSteps != 0 && Steps >= MaxSteps) { | |
| reportError() << "Exceeded maximum number of execution steps."; | |
| break; | |
| } | |
| ++Steps; | |
| Instruction &I = *Top.PC; | |
| visit(&I); | |
| Ctx.resetNoncacheableConstantBuffer(); | |
| if (hasProgramExited()) | |
| break; | |
| // A function call or return has occurred. | |
| // We need to exit the inner loop and switch to a different frame. | |
| if (Top.State != FrameState::Running) | |
| break; | |
| // Otherwise, move to the next instruction if it is not a terminator. | |
| // For terminators, the PC is updated in the visit* method. | |
| if (!I.isTerminator()) | |
| ++Top.PC; | |
| } | |
| if (hasProgramExited()) | |
| break; | |
| if (Top.State == FrameState::Exit) { | |
| assert((Top.Func.getReturnType()->isVoidTy() || !Top.RetVal.isNone()) && | |
| "Expected return value to be set on function exit."); | |
| Handler.onFunctionExit(Top.Func, Top.RetVal); | |
| // Free stack objects allocated in this frame. | |
| for (auto &Obj : Top.Allocas) | |
| Ctx.free(*Obj); | |
| CallStack.pop_back(); | |
| } else { | |
| assert(Top.State == FrameState::Pending && | |
| "Expected to enter a callee."); | |
| } | |
| } | |
| if (!hasProgramExited()) | |
| requestProgramExit(ProgramExitInfo::ProgramExitKind::Returned); | |
| return *getExitInfo(); | |
| } | |
| }; | |
| ProgramExitInfo Context::runFunction(Function &F, ArrayRef<AnyValue> Args, | |
| AnyValue &RetVal, EventHandler &Handler) { | |
| InstExecutor Executor(*this, Handler, F, Args, RetVal); | |
| return Executor.runMainLoop(); | |
| } | |
| } // namespace llvm::ubi |