| //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is distributed under the University of Illinois Open Source |
| // License. See LICENSE.TXT for details. |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This implements routines for translating from LLVM IR into SelectionDAG IR. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "SelectionDAGBuilder.h" |
| #include "llvm/ADT/APFloat.h" |
| #include "llvm/ADT/APInt.h" |
| #include "llvm/ADT/ArrayRef.h" |
| #include "llvm/ADT/BitVector.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/ADT/None.h" |
| #include "llvm/ADT/Optional.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/SmallPtrSet.h" |
| #include "llvm/ADT/SmallSet.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/ADT/StringRef.h" |
| #include "llvm/ADT/Triple.h" |
| #include "llvm/ADT/Twine.h" |
| #include "llvm/Analysis/AliasAnalysis.h" |
| #include "llvm/Analysis/BranchProbabilityInfo.h" |
| #include "llvm/Analysis/ConstantFolding.h" |
| #include "llvm/Analysis/EHPersonalities.h" |
| #include "llvm/Analysis/Loads.h" |
| #include "llvm/Analysis/MemoryLocation.h" |
| #include "llvm/Analysis/TargetLibraryInfo.h" |
| #include "llvm/Analysis/ValueTracking.h" |
| #include "llvm/Analysis/VectorUtils.h" |
| #include "llvm/CodeGen/Analysis.h" |
| #include "llvm/CodeGen/FunctionLoweringInfo.h" |
| #include "llvm/CodeGen/GCMetadata.h" |
| #include "llvm/CodeGen/ISDOpcodes.h" |
| #include "llvm/CodeGen/MachineBasicBlock.h" |
| #include "llvm/CodeGen/MachineFrameInfo.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstr.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineJumpTableInfo.h" |
| #include "llvm/CodeGen/MachineMemOperand.h" |
| #include "llvm/CodeGen/MachineModuleInfo.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/MachineValueType.h" |
| #include "llvm/CodeGen/RuntimeLibcalls.h" |
| #include "llvm/CodeGen/SelectionDAG.h" |
| #include "llvm/CodeGen/SelectionDAGNodes.h" |
| #include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
| #include "llvm/CodeGen/StackMaps.h" |
| #include "llvm/CodeGen/TargetFrameLowering.h" |
| #include "llvm/CodeGen/TargetInstrInfo.h" |
| #include "llvm/CodeGen/ValueTypes.h" |
| #include "llvm/CodeGen/WinEHFuncInfo.h" |
| #include "llvm/IR/Argument.h" |
| #include "llvm/IR/Attributes.h" |
| #include "llvm/IR/BasicBlock.h" |
| #include "llvm/IR/CFG.h" |
| #include "llvm/IR/CallSite.h" |
| #include "llvm/IR/CallingConv.h" |
| #include "llvm/IR/Constant.h" |
| #include "llvm/IR/ConstantRange.h" |
| #include "llvm/IR/Constants.h" |
| #include "llvm/IR/DataLayout.h" |
| #include "llvm/IR/DebugInfoMetadata.h" |
| #include "llvm/IR/DebugLoc.h" |
| #include "llvm/IR/DerivedTypes.h" |
| #include "llvm/IR/Function.h" |
| #include "llvm/IR/GetElementPtrTypeIterator.h" |
| #include "llvm/IR/InlineAsm.h" |
| #include "llvm/IR/InstrTypes.h" |
| #include "llvm/IR/Instruction.h" |
| #include "llvm/IR/Instructions.h" |
| #include "llvm/IR/IntrinsicInst.h" |
| #include "llvm/IR/Intrinsics.h" |
| #include "llvm/IR/LLVMContext.h" |
| #include "llvm/IR/Metadata.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/IR/Operator.h" |
| #include "llvm/IR/Statepoint.h" |
| #include "llvm/IR/Type.h" |
| #include "llvm/IR/User.h" |
| #include "llvm/IR/Value.h" |
| #include "llvm/MC/MCContext.h" |
| #include "llvm/MC/MCSymbol.h" |
| #include "llvm/Support/AtomicOrdering.h" |
| #include "llvm/Support/BranchProbability.h" |
| #include "llvm/Support/Casting.h" |
| #include "llvm/Support/CodeGen.h" |
| #include "llvm/Support/CommandLine.h" |
| #include "llvm/Support/Compiler.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/MathExtras.h" |
| #include "llvm/Support/raw_ostream.h" |
| #include "llvm/Target/TargetIntrinsicInfo.h" |
| #include "llvm/Target/TargetLowering.h" |
| #include "llvm/Target/TargetMachine.h" |
| #include "llvm/Target/TargetOpcodes.h" |
| #include "llvm/Target/TargetOptions.h" |
| #include "llvm/Target/TargetRegisterInfo.h" |
| #include "llvm/Target/TargetSubtargetInfo.h" |
| #include <algorithm> |
| #include <cassert> |
| #include <cstddef> |
| #include <cstdint> |
| #include <cstring> |
| #include <iterator> |
| #include <limits> |
| #include <numeric> |
| #include <tuple> |
| #include <utility> |
| #include <vector> |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "isel" |
| |
| /// LimitFloatPrecision - Generate low-precision inline sequences for |
| /// some float libcalls (6, 8 or 12 bits). |
| static unsigned LimitFloatPrecision; |
| |
| static cl::opt<unsigned, true> |
| LimitFPPrecision("limit-float-precision", |
| cl::desc("Generate low-precision inline sequences " |
| "for some float libcalls"), |
| cl::location(LimitFloatPrecision), |
| cl::init(0)); |
| |
| // Limit the width of DAG chains. This is important in general to prevent |
| // DAG-based analysis from blowing up. For example, alias analysis and |
| // load clustering may not complete in reasonable time. It is difficult to |
| // recognize and avoid this situation within each individual analysis, and |
| // future analyses are likely to have the same behavior. Limiting DAG width is |
| // the safe approach and will be especially important with global DAGs. |
| // |
| // MaxParallelChains default is arbitrarily high to avoid affecting |
| // optimization, but could be lowered to improve compile time. Any ld-ld-st-st |
| // sequence over this should have been converted to llvm.memcpy by the |
| // frontend. It is easy to induce this behavior with .ll code such as: |
| // %buffer = alloca [4096 x i8] |
| // %data = load [4096 x i8]* %argPtr |
| // store [4096 x i8] %data, [4096 x i8]* %buffer |
| static const unsigned MaxParallelChains = 64; |
| |
| // True if the Value passed requires ABI mangling as it is a parameter to a |
| // function or a return value from a function which is not an intrinsic. |
| static bool isABIRegCopy(const Value *V) { |
| const bool IsRetInst = V && isa<ReturnInst>(V); |
| const bool IsCallInst = V && isa<CallInst>(V); |
| const bool IsInLineAsm = |
| IsCallInst && static_cast<const CallInst *>(V)->isInlineAsm(); |
| const bool IsIndirectFunctionCall = |
| IsCallInst && !IsInLineAsm && |
| !static_cast<const CallInst *>(V)->getCalledFunction(); |
| // It is possible that the call instruction is an inline asm statement or an |
| // indirect function call in which case the return value of |
| // getCalledFunction() would be nullptr. |
| const bool IsInstrinsicCall = |
| IsCallInst && !IsInLineAsm && !IsIndirectFunctionCall && |
| static_cast<const CallInst *>(V)->getCalledFunction()->getIntrinsicID() != |
| Intrinsic::not_intrinsic; |
| |
| return IsRetInst || (IsCallInst && (!IsInLineAsm && !IsInstrinsicCall)); |
| } |
| |
| static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
| const SDValue *Parts, unsigned NumParts, |
| MVT PartVT, EVT ValueVT, const Value *V, |
| bool IsABIRegCopy); |
| |
| /// getCopyFromParts - Create a value that contains the specified legal parts |
| /// combined into the value they represent. If the parts combine to a type |
| /// larger than ValueVT then AssertOp can be used to specify whether the extra |
| /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT |
| /// (ISD::AssertSext). |
| static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, |
| const SDValue *Parts, unsigned NumParts, |
| MVT PartVT, EVT ValueVT, const Value *V, |
| Optional<ISD::NodeType> AssertOp = None, |
| bool IsABIRegCopy = false) { |
| if (ValueVT.isVector()) |
| return getCopyFromPartsVector(DAG, DL, Parts, NumParts, |
| PartVT, ValueVT, V, IsABIRegCopy); |
| |
| assert(NumParts > 0 && "No parts to assemble!"); |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| SDValue Val = Parts[0]; |
| |
| if (NumParts > 1) { |
| // Assemble the value from multiple parts. |
| if (ValueVT.isInteger()) { |
| unsigned PartBits = PartVT.getSizeInBits(); |
| unsigned ValueBits = ValueVT.getSizeInBits(); |
| |
| // Assemble the power of 2 part. |
| unsigned RoundParts = NumParts & (NumParts - 1) ? |
| 1 << Log2_32(NumParts) : NumParts; |
| unsigned RoundBits = PartBits * RoundParts; |
| EVT RoundVT = RoundBits == ValueBits ? |
| ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); |
| SDValue Lo, Hi; |
| |
| EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); |
| |
| if (RoundParts > 2) { |
| Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, |
| PartVT, HalfVT, V); |
| Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, |
| RoundParts / 2, PartVT, HalfVT, V); |
| } else { |
| Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); |
| Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); |
| } |
| |
| if (DAG.getDataLayout().isBigEndian()) |
| std::swap(Lo, Hi); |
| |
| Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); |
| |
| if (RoundParts < NumParts) { |
| // Assemble the trailing non-power-of-2 part. |
| unsigned OddParts = NumParts - RoundParts; |
| EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); |
| Hi = getCopyFromParts(DAG, DL, |
| Parts + RoundParts, OddParts, PartVT, OddVT, V); |
| |
| // Combine the round and odd parts. |
| Lo = Val; |
| if (DAG.getDataLayout().isBigEndian()) |
| std::swap(Lo, Hi); |
| EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
| Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); |
| Hi = |
| DAG.getNode(ISD::SHL, DL, TotalVT, Hi, |
| DAG.getConstant(Lo.getValueSizeInBits(), DL, |
| TLI.getPointerTy(DAG.getDataLayout()))); |
| Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); |
| Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); |
| } |
| } else if (PartVT.isFloatingPoint()) { |
| // FP split into multiple FP parts (for ppcf128) |
| assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && |
| "Unexpected split"); |
| SDValue Lo, Hi; |
| Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); |
| Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); |
| if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) |
| std::swap(Lo, Hi); |
| Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); |
| } else { |
| // FP split into integer parts (soft fp) |
| assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && |
| !PartVT.isVector() && "Unexpected split"); |
| EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
| Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V); |
| } |
| } |
| |
| // There is now one part, held in Val. Correct it to match ValueVT. |
| // PartEVT is the type of the register class that holds the value. |
| // ValueVT is the type of the inline asm operation. |
| EVT PartEVT = Val.getValueType(); |
| |
| if (PartEVT == ValueVT) |
| return Val; |
| |
| if (PartEVT.isInteger() && ValueVT.isFloatingPoint() && |
| ValueVT.bitsLT(PartEVT)) { |
| // For an FP value in an integer part, we need to truncate to the right |
| // width first. |
| PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
| Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); |
| } |
| |
| // Handle types that have the same size. |
| if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) |
| return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
| |
| // Handle types with different sizes. |
| if (PartEVT.isInteger() && ValueVT.isInteger()) { |
| if (ValueVT.bitsLT(PartEVT)) { |
| // For a truncate, see if we have any information to |
| // indicate whether the truncated bits will always be |
| // zero or sign-extension. |
| if (AssertOp.hasValue()) |
| Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, |
| DAG.getValueType(ValueVT)); |
| return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
| } |
| return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); |
| } |
| |
| if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { |
| // FP_ROUND's are always exact here. |
| if (ValueVT.bitsLT(Val.getValueType())) |
| return DAG.getNode( |
| ISD::FP_ROUND, DL, ValueVT, Val, |
| DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()))); |
| |
| return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); |
| } |
| |
| llvm_unreachable("Unknown mismatch!"); |
| } |
| |
| static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, |
| const Twine &ErrMsg) { |
| const Instruction *I = dyn_cast_or_null<Instruction>(V); |
| if (!V) |
| return Ctx.emitError(ErrMsg); |
| |
| const char *AsmError = ", possible invalid constraint for vector type"; |
| if (const CallInst *CI = dyn_cast<CallInst>(I)) |
| if (isa<InlineAsm>(CI->getCalledValue())) |
| return Ctx.emitError(I, ErrMsg + AsmError); |
| |
| return Ctx.emitError(I, ErrMsg); |
| } |
| |
| /// getCopyFromPartsVector - Create a value that contains the specified legal |
| /// parts combined into the value they represent. If the parts combine to a |
| /// type larger than ValueVT then AssertOp can be used to specify whether the |
| /// extra bits are known to be zero (ISD::AssertZext) or sign extended from |
| /// ValueVT (ISD::AssertSext). |
| static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
| const SDValue *Parts, unsigned NumParts, |
| MVT PartVT, EVT ValueVT, const Value *V, |
| bool IsABIRegCopy) { |
| assert(ValueVT.isVector() && "Not a vector value"); |
| assert(NumParts > 0 && "No parts to assemble!"); |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| SDValue Val = Parts[0]; |
| |
| // Handle a multi-element vector. |
| if (NumParts > 1) { |
| EVT IntermediateVT; |
| MVT RegisterVT; |
| unsigned NumIntermediates; |
| unsigned NumRegs; |
| |
| if (IsABIRegCopy) { |
| NumRegs = TLI.getVectorTypeBreakdownForCallingConv( |
| *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates, |
| RegisterVT); |
| } else { |
| NumRegs = |
| TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, |
| NumIntermediates, RegisterVT); |
| } |
| |
| assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); |
| NumParts = NumRegs; // Silence a compiler warning. |
| assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); |
| assert(RegisterVT.getSizeInBits() == |
| Parts[0].getSimpleValueType().getSizeInBits() && |
| "Part type sizes don't match!"); |
| |
| // Assemble the parts into intermediate operands. |
| SmallVector<SDValue, 8> Ops(NumIntermediates); |
| if (NumIntermediates == NumParts) { |
| // If the register was not expanded, truncate or copy the value, |
| // as appropriate. |
| for (unsigned i = 0; i != NumParts; ++i) |
| Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, |
| PartVT, IntermediateVT, V); |
| } else if (NumParts > 0) { |
| // If the intermediate type was expanded, build the intermediate |
| // operands from the parts. |
| assert(NumParts % NumIntermediates == 0 && |
| "Must expand into a divisible number of parts!"); |
| unsigned Factor = NumParts / NumIntermediates; |
| for (unsigned i = 0; i != NumIntermediates; ++i) |
| Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, |
| PartVT, IntermediateVT, V); |
| } |
| |
| // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the |
| // intermediate operands. |
| EVT BuiltVectorTy = |
| EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(), |
| (IntermediateVT.isVector() |
| ? IntermediateVT.getVectorNumElements() * NumParts |
| : NumIntermediates)); |
| Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS |
| : ISD::BUILD_VECTOR, |
| DL, BuiltVectorTy, Ops); |
| } |
| |
| // There is now one part, held in Val. Correct it to match ValueVT. |
| EVT PartEVT = Val.getValueType(); |
| |
| if (PartEVT == ValueVT) |
| return Val; |
| |
| if (PartEVT.isVector()) { |
| // If the element type of the source/dest vectors are the same, but the |
| // parts vector has more elements than the value vector, then we have a |
| // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the |
| // elements we want. |
| if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) { |
| assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && |
| "Cannot narrow, it would be a lossy transformation"); |
| return DAG.getNode( |
| ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, |
| DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
| } |
| |
| // Vector/Vector bitcast. |
| if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) |
| return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
| |
| assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && |
| "Cannot handle this kind of promotion"); |
| // Promoted vector extract |
| return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); |
| |
| } |
| |
| // Trivial bitcast if the types are the same size and the destination |
| // vector type is legal. |
| if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && |
| TLI.isTypeLegal(ValueVT)) |
| return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
| |
| if (ValueVT.getVectorNumElements() != 1) { |
| // Certain ABIs require that vectors are passed as integers. For vectors |
| // are the same size, this is an obvious bitcast. |
| if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { |
| return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
| } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) { |
| // Bitcast Val back the original type and extract the corresponding |
| // vector we want. |
| unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits(); |
| EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(), |
| ValueVT.getVectorElementType(), Elts); |
| Val = DAG.getBitcast(WiderVecType, Val); |
| return DAG.getNode( |
| ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, |
| DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
| } |
| |
| diagnosePossiblyInvalidConstraint( |
| *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); |
| return DAG.getUNDEF(ValueVT); |
| } |
| |
| // Handle cases such as i8 -> <1 x i1> |
| EVT ValueSVT = ValueVT.getVectorElementType(); |
| if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) |
| Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) |
| : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); |
| |
| return DAG.getBuildVector(ValueVT, DL, Val); |
| } |
| |
| static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, |
| SDValue Val, SDValue *Parts, unsigned NumParts, |
| MVT PartVT, const Value *V, bool IsABIRegCopy); |
| |
| /// getCopyToParts - Create a series of nodes that contain the specified value |
| /// split into legal parts. If the parts contain more bits than Val, then, for |
| /// integers, ExtendKind can be used to specify how to generate the extra bits. |
| static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, |
| SDValue *Parts, unsigned NumParts, MVT PartVT, |
| const Value *V, |
| ISD::NodeType ExtendKind = ISD::ANY_EXTEND, |
| bool IsABIRegCopy = false) { |
| EVT ValueVT = Val.getValueType(); |
| |
| // Handle the vector case separately. |
| if (ValueVT.isVector()) |
| return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, |
| IsABIRegCopy); |
| |
| unsigned PartBits = PartVT.getSizeInBits(); |
| unsigned OrigNumParts = NumParts; |
| assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && |
| "Copying to an illegal type!"); |
| |
| if (NumParts == 0) |
| return; |
| |
| assert(!ValueVT.isVector() && "Vector case handled elsewhere"); |
| EVT PartEVT = PartVT; |
| if (PartEVT == ValueVT) { |
| assert(NumParts == 1 && "No-op copy with multiple parts!"); |
| Parts[0] = Val; |
| return; |
| } |
| |
| if (NumParts * PartBits > ValueVT.getSizeInBits()) { |
| // If the parts cover more bits than the value has, promote the value. |
| if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { |
| assert(NumParts == 1 && "Do not know what to promote to!"); |
| Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); |
| } else { |
| if (ValueVT.isFloatingPoint()) { |
| // FP values need to be bitcast, then extended if they are being put |
| // into a larger container. |
| ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
| Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); |
| } |
| assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && |
| ValueVT.isInteger() && |
| "Unknown mismatch!"); |
| ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
| Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); |
| if (PartVT == MVT::x86mmx) |
| Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
| } |
| } else if (PartBits == ValueVT.getSizeInBits()) { |
| // Different types of the same size. |
| assert(NumParts == 1 && PartEVT != ValueVT); |
| Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
| } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { |
| // If the parts cover less bits than value has, truncate the value. |
| assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && |
| ValueVT.isInteger() && |
| "Unknown mismatch!"); |
| ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
| Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
| if (PartVT == MVT::x86mmx) |
| Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
| } |
| |
| // The value may have changed - recompute ValueVT. |
| ValueVT = Val.getValueType(); |
| assert(NumParts * PartBits == ValueVT.getSizeInBits() && |
| "Failed to tile the value with PartVT!"); |
| |
| if (NumParts == 1) { |
| if (PartEVT != ValueVT) { |
| diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, |
| "scalar-to-vector conversion failed"); |
| Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
| } |
| |
| Parts[0] = Val; |
| return; |
| } |
| |
| // Expand the value into multiple parts. |
| if (NumParts & (NumParts - 1)) { |
| // The number of parts is not a power of 2. Split off and copy the tail. |
| assert(PartVT.isInteger() && ValueVT.isInteger() && |
| "Do not know what to expand to!"); |
| unsigned RoundParts = 1 << Log2_32(NumParts); |
| unsigned RoundBits = RoundParts * PartBits; |
| unsigned OddParts = NumParts - RoundParts; |
| SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, |
| DAG.getIntPtrConstant(RoundBits, DL)); |
| getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V); |
| |
| if (DAG.getDataLayout().isBigEndian()) |
| // The odd parts were reversed by getCopyToParts - unreverse them. |
| std::reverse(Parts + RoundParts, Parts + NumParts); |
| |
| NumParts = RoundParts; |
| ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); |
| Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); |
| } |
| |
| // The number of parts is a power of 2. Repeatedly bisect the value using |
| // EXTRACT_ELEMENT. |
| Parts[0] = DAG.getNode(ISD::BITCAST, DL, |
| EVT::getIntegerVT(*DAG.getContext(), |
| ValueVT.getSizeInBits()), |
| Val); |
| |
| for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { |
| for (unsigned i = 0; i < NumParts; i += StepSize) { |
| unsigned ThisBits = StepSize * PartBits / 2; |
| EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); |
| SDValue &Part0 = Parts[i]; |
| SDValue &Part1 = Parts[i+StepSize/2]; |
| |
| Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, |
| ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); |
| Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, |
| ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); |
| |
| if (ThisBits == PartBits && ThisVT != PartVT) { |
| Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); |
| Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); |
| } |
| } |
| } |
| |
| if (DAG.getDataLayout().isBigEndian()) |
| std::reverse(Parts, Parts + OrigNumParts); |
| } |
| |
| |
| /// getCopyToPartsVector - Create a series of nodes that contain the specified |
| /// value split into legal parts. |
| static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, |
| SDValue Val, SDValue *Parts, unsigned NumParts, |
| MVT PartVT, const Value *V, |
| bool IsABIRegCopy) { |
| EVT ValueVT = Val.getValueType(); |
| assert(ValueVT.isVector() && "Not a vector"); |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| |
| if (NumParts == 1) { |
| EVT PartEVT = PartVT; |
| if (PartEVT == ValueVT) { |
| // Nothing to do. |
| } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { |
| // Bitconvert vector->vector case. |
| Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); |
| } else if (PartVT.isVector() && |
| PartEVT.getVectorElementType() == ValueVT.getVectorElementType() && |
| PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) { |
| EVT ElementVT = PartVT.getVectorElementType(); |
| // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in |
| // undef elements. |
| SmallVector<SDValue, 16> Ops; |
| for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i) |
| Ops.push_back(DAG.getNode( |
| ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val, |
| DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())))); |
| |
| for (unsigned i = ValueVT.getVectorNumElements(), |
| e = PartVT.getVectorNumElements(); i != e; ++i) |
| Ops.push_back(DAG.getUNDEF(ElementVT)); |
| |
| Val = DAG.getBuildVector(PartVT, DL, Ops); |
| |
| // FIXME: Use CONCAT for 2x -> 4x. |
| |
| //SDValue UndefElts = DAG.getUNDEF(VectorTy); |
| //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts); |
| } else if (PartVT.isVector() && |
| PartEVT.getVectorElementType().bitsGE( |
| ValueVT.getVectorElementType()) && |
| PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) { |
| |
| // Promoted vector extract |
| Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); |
| } else { |
| if (ValueVT.getVectorNumElements() == 1) { |
| Val = DAG.getNode( |
| ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, |
| DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
| } else { |
| assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() && |
| "lossy conversion of vector to scalar type"); |
| EVT IntermediateType = |
| EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); |
| Val = DAG.getBitcast(IntermediateType, Val); |
| Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); |
| } |
| } |
| |
| assert(Val.getValueType() == PartVT && "Unexpected vector part value type"); |
| Parts[0] = Val; |
| return; |
| } |
| |
| // Handle a multi-element vector. |
| EVT IntermediateVT; |
| MVT RegisterVT; |
| unsigned NumIntermediates; |
| unsigned NumRegs; |
| if (IsABIRegCopy) { |
| NumRegs = TLI.getVectorTypeBreakdownForCallingConv( |
| *DAG.getContext(), ValueVT, IntermediateVT, NumIntermediates, |
| RegisterVT); |
| } else { |
| NumRegs = |
| TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, |
| NumIntermediates, RegisterVT); |
| } |
| unsigned NumElements = ValueVT.getVectorNumElements(); |
| |
| assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); |
| NumParts = NumRegs; // Silence a compiler warning. |
| assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); |
| |
| // Convert the vector to the appropiate type if necessary. |
| unsigned DestVectorNoElts = |
| NumIntermediates * |
| (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1); |
| EVT BuiltVectorTy = EVT::getVectorVT( |
| *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts); |
| if (Val.getValueType() != BuiltVectorTy) |
| Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); |
| |
| // Split the vector into intermediate operands. |
| SmallVector<SDValue, 8> Ops(NumIntermediates); |
| for (unsigned i = 0; i != NumIntermediates; ++i) { |
| if (IntermediateVT.isVector()) |
| Ops[i] = |
| DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, |
| DAG.getConstant(i * (NumElements / NumIntermediates), DL, |
| TLI.getVectorIdxTy(DAG.getDataLayout()))); |
| else |
| Ops[i] = DAG.getNode( |
| ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, |
| DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
| } |
| |
| // Split the intermediate operands into legal parts. |
| if (NumParts == NumIntermediates) { |
| // If the register was not expanded, promote or copy the value, |
| // as appropriate. |
| for (unsigned i = 0; i != NumParts; ++i) |
| getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V); |
| } else if (NumParts > 0) { |
| // If the intermediate type was expanded, split each the value into |
| // legal parts. |
| assert(NumIntermediates != 0 && "division by zero"); |
| assert(NumParts % NumIntermediates == 0 && |
| "Must expand into a divisible number of parts!"); |
| unsigned Factor = NumParts / NumIntermediates; |
| for (unsigned i = 0; i != NumIntermediates; ++i) |
| getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V); |
| } |
| } |
| |
| RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt, |
| EVT valuevt, bool IsABIMangledValue) |
| : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs), |
| RegCount(1, regs.size()), IsABIMangled(IsABIMangledValue) {} |
| |
| RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI, |
| const DataLayout &DL, unsigned Reg, Type *Ty, |
| bool IsABIMangledValue) { |
| ComputeValueVTs(TLI, DL, Ty, ValueVTs); |
| |
| IsABIMangled = IsABIMangledValue; |
| |
| for (EVT ValueVT : ValueVTs) { |
| unsigned NumRegs = IsABIMangledValue |
| ? TLI.getNumRegistersForCallingConv(Context, ValueVT) |
| : TLI.getNumRegisters(Context, ValueVT); |
| MVT RegisterVT = IsABIMangledValue |
| ? TLI.getRegisterTypeForCallingConv(Context, ValueVT) |
| : TLI.getRegisterType(Context, ValueVT); |
| for (unsigned i = 0; i != NumRegs; ++i) |
| Regs.push_back(Reg + i); |
| RegVTs.push_back(RegisterVT); |
| RegCount.push_back(NumRegs); |
| Reg += NumRegs; |
| } |
| } |
| |
| SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, |
| FunctionLoweringInfo &FuncInfo, |
| const SDLoc &dl, SDValue &Chain, |
| SDValue *Flag, const Value *V) const { |
| // A Value with type {} or [0 x %t] needs no registers. |
| if (ValueVTs.empty()) |
| return SDValue(); |
| |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| |
| // Assemble the legal parts into the final values. |
| SmallVector<SDValue, 4> Values(ValueVTs.size()); |
| SmallVector<SDValue, 8> Parts; |
| for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { |
| // Copy the legal parts from the registers. |
| EVT ValueVT = ValueVTs[Value]; |
| unsigned NumRegs = RegCount[Value]; |
| MVT RegisterVT = IsABIMangled |
| ? TLI.getRegisterTypeForCallingConv(RegVTs[Value]) |
| : RegVTs[Value]; |
| |
| Parts.resize(NumRegs); |
| for (unsigned i = 0; i != NumRegs; ++i) { |
| SDValue P; |
| if (!Flag) { |
| P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); |
| } else { |
| P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); |
| *Flag = P.getValue(2); |
| } |
| |
| Chain = P.getValue(1); |
| Parts[i] = P; |
| |
| // If the source register was virtual and if we know something about it, |
| // add an assert node. |
| if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) || |
| !RegisterVT.isInteger() || RegisterVT.isVector()) |
| continue; |
| |
| const FunctionLoweringInfo::LiveOutInfo *LOI = |
| FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); |
| if (!LOI) |
| continue; |
| |
| unsigned RegSize = RegisterVT.getSizeInBits(); |
| unsigned NumSignBits = LOI->NumSignBits; |
| unsigned NumZeroBits = LOI->Known.countMinLeadingZeros(); |
| |
| if (NumZeroBits == RegSize) { |
| // The current value is a zero. |
| // Explicitly express that as it would be easier for |
| // optimizations to kick in. |
| Parts[i] = DAG.getConstant(0, dl, RegisterVT); |
| continue; |
| } |
| |
| // FIXME: We capture more information than the dag can represent. For |
| // now, just use the tightest assertzext/assertsext possible. |
| bool isSExt = true; |
| EVT FromVT(MVT::Other); |
| if (NumSignBits == RegSize) { |
| isSExt = true; // ASSERT SEXT 1 |
| FromVT = MVT::i1; |
| } else if (NumZeroBits >= RegSize - 1) { |
| isSExt = false; // ASSERT ZEXT 1 |
| FromVT = MVT::i1; |
| } else if (NumSignBits > RegSize - 8) { |
| isSExt = true; // ASSERT SEXT 8 |
| FromVT = MVT::i8; |
| } else if (NumZeroBits >= RegSize - 8) { |
| isSExt = false; // ASSERT ZEXT 8 |
| FromVT = MVT::i8; |
| } else if (NumSignBits > RegSize - 16) { |
| isSExt = true; // ASSERT SEXT 16 |
| FromVT = MVT::i16; |
| } else if (NumZeroBits >= RegSize - 16) { |
| isSExt = false; // ASSERT ZEXT 16 |
| FromVT = MVT::i16; |
| } else if (NumSignBits > RegSize - 32) { |
| isSExt = true; // ASSERT SEXT 32 |
| FromVT = MVT::i32; |
| } else if (NumZeroBits >= RegSize - 32) { |
| isSExt = false; // ASSERT ZEXT 32 |
| FromVT = MVT::i32; |
| } else { |
| continue; |
| } |
| // Add an assertion node. |
| assert(FromVT != MVT::Other); |
| Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, |
| RegisterVT, P, DAG.getValueType(FromVT)); |
| } |
| |
| Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), |
| NumRegs, RegisterVT, ValueVT, V); |
| Part += NumRegs; |
| Parts.clear(); |
| } |
| |
| return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); |
| } |
| |
| void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, |
| const SDLoc &dl, SDValue &Chain, SDValue *Flag, |
| const Value *V, |
| ISD::NodeType PreferredExtendType) const { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| ISD::NodeType ExtendKind = PreferredExtendType; |
| |
| // Get the list of the values's legal parts. |
| unsigned NumRegs = Regs.size(); |
| SmallVector<SDValue, 8> Parts(NumRegs); |
| for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { |
| unsigned NumParts = RegCount[Value]; |
| |
| MVT RegisterVT = IsABIMangled |
| ? TLI.getRegisterTypeForCallingConv(RegVTs[Value]) |
| : RegVTs[Value]; |
| |
| if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT)) |
| ExtendKind = ISD::ZERO_EXTEND; |
| |
| getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), |
| &Parts[Part], NumParts, RegisterVT, V, ExtendKind); |
| Part += NumParts; |
| } |
| |
| // Copy the parts into the registers. |
| SmallVector<SDValue, 8> Chains(NumRegs); |
| for (unsigned i = 0; i != NumRegs; ++i) { |
| SDValue Part; |
| if (!Flag) { |
| Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); |
| } else { |
| Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); |
| *Flag = Part.getValue(1); |
| } |
| |
| Chains[i] = Part.getValue(0); |
| } |
| |
| if (NumRegs == 1 || Flag) |
| // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is |
| // flagged to it. That is the CopyToReg nodes and the user are considered |
| // a single scheduling unit. If we create a TokenFactor and return it as |
| // chain, then the TokenFactor is both a predecessor (operand) of the |
| // user as well as a successor (the TF operands are flagged to the user). |
| // c1, f1 = CopyToReg |
| // c2, f2 = CopyToReg |
| // c3 = TokenFactor c1, c2 |
| // ... |
| // = op c3, ..., f2 |
| Chain = Chains[NumRegs-1]; |
| else |
| Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); |
| } |
| |
| void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, |
| unsigned MatchingIdx, const SDLoc &dl, |
| SelectionDAG &DAG, |
| std::vector<SDValue> &Ops) const { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| |
| unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); |
| if (HasMatching) |
| Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); |
| else if (!Regs.empty() && |
| TargetRegisterInfo::isVirtualRegister(Regs.front())) { |
| // Put the register class of the virtual registers in the flag word. That |
| // way, later passes can recompute register class constraints for inline |
| // assembly as well as normal instructions. |
| // Don't do this for tied operands that can use the regclass information |
| // from the def. |
| const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); |
| const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); |
| Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); |
| } |
| |
| SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); |
| Ops.push_back(Res); |
| |
| if (Code == InlineAsm::Kind_Clobber) { |
| // Clobbers should always have a 1:1 mapping with registers, and may |
| // reference registers that have illegal (e.g. vector) types. Hence, we |
| // shouldn't try to apply any sort of splitting logic to them. |
| assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && |
| "No 1:1 mapping from clobbers to regs?"); |
| unsigned SP = TLI.getStackPointerRegisterToSaveRestore(); |
| (void)SP; |
| for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) { |
| Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); |
| assert( |
| (Regs[I] != SP || |
| DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && |
| "If we clobbered the stack pointer, MFI should know about it."); |
| } |
| return; |
| } |
| |
| for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { |
| unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]); |
| MVT RegisterVT = RegVTs[Value]; |
| for (unsigned i = 0; i != NumRegs; ++i) { |
| assert(Reg < Regs.size() && "Mismatch in # registers expected"); |
| unsigned TheReg = Regs[Reg++]; |
| Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); |
| } |
| } |
| } |
| |
| void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, |
| const TargetLibraryInfo *li) { |
| AA = aa; |
| GFI = gfi; |
| LibInfo = li; |
| DL = &DAG.getDataLayout(); |
| Context = DAG.getContext(); |
| LPadToCallSiteMap.clear(); |
| } |
| |
| void SelectionDAGBuilder::clear() { |
| NodeMap.clear(); |
| UnusedArgNodeMap.clear(); |
| PendingLoads.clear(); |
| PendingExports.clear(); |
| CurInst = nullptr; |
| HasTailCall = false; |
| SDNodeOrder = LowestSDNodeOrder; |
| StatepointLowering.clear(); |
| } |
| |
| void SelectionDAGBuilder::clearDanglingDebugInfo() { |
| DanglingDebugInfoMap.clear(); |
| } |
| |
| SDValue SelectionDAGBuilder::getRoot() { |
| if (PendingLoads.empty()) |
| return DAG.getRoot(); |
| |
| if (PendingLoads.size() == 1) { |
| SDValue Root = PendingLoads[0]; |
| DAG.setRoot(Root); |
| PendingLoads.clear(); |
| return Root; |
| } |
| |
| // Otherwise, we have to make a token factor node. |
| SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, |
| PendingLoads); |
| PendingLoads.clear(); |
| DAG.setRoot(Root); |
| return Root; |
| } |
| |
| SDValue SelectionDAGBuilder::getControlRoot() { |
| SDValue Root = DAG.getRoot(); |
| |
| if (PendingExports.empty()) |
| return Root; |
| |
| // Turn all of the CopyToReg chains into one factored node. |
| if (Root.getOpcode() != ISD::EntryToken) { |
| unsigned i = 0, e = PendingExports.size(); |
| for (; i != e; ++i) { |
| assert(PendingExports[i].getNode()->getNumOperands() > 1); |
| if (PendingExports[i].getNode()->getOperand(0) == Root) |
| break; // Don't add the root if we already indirectly depend on it. |
| } |
| |
| if (i == e) |
| PendingExports.push_back(Root); |
| } |
| |
| Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, |
| PendingExports); |
| PendingExports.clear(); |
| DAG.setRoot(Root); |
| return Root; |
| } |
| |
| void SelectionDAGBuilder::visit(const Instruction &I) { |
| // Set up outgoing PHI node register values before emitting the terminator. |
| if (isa<TerminatorInst>(&I)) { |
| HandlePHINodesInSuccessorBlocks(I.getParent()); |
| } |
| |
| // Increase the SDNodeOrder if dealing with a non-debug instruction. |
| if (!isa<DbgInfoIntrinsic>(I)) |
| ++SDNodeOrder; |
| |
| CurInst = &I; |
| |
| visit(I.getOpcode(), I); |
| |
| if (!isa<TerminatorInst>(&I) && !HasTailCall && |
| !isStatepoint(&I)) // statepoints handle their exports internally |
| CopyToExportRegsIfNeeded(&I); |
| |
| CurInst = nullptr; |
| } |
| |
| void SelectionDAGBuilder::visitPHI(const PHINode &) { |
| llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); |
| } |
| |
| void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { |
| // Note: this doesn't use InstVisitor, because it has to work with |
| // ConstantExpr's in addition to instructions. |
| switch (Opcode) { |
| default: llvm_unreachable("Unknown instruction type encountered!"); |
| // Build the switch statement using the Instruction.def file. |
| #define HANDLE_INST(NUM, OPCODE, CLASS) \ |
| case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; |
| #include "llvm/IR/Instruction.def" |
| } |
| } |
| |
| // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, |
| // generate the debug data structures now that we've seen its definition. |
| void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, |
| SDValue Val) { |
| DanglingDebugInfo &DDI = DanglingDebugInfoMap[V]; |
| if (DDI.getDI()) { |
| const DbgValueInst *DI = DDI.getDI(); |
| DebugLoc dl = DDI.getdl(); |
| unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); |
| DILocalVariable *Variable = DI->getVariable(); |
| DIExpression *Expr = DI->getExpression(); |
| assert(Variable->isValidLocationForIntrinsic(dl) && |
| "Expected inlined-at fields to agree"); |
| SDDbgValue *SDV; |
| if (Val.getNode()) { |
| if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) { |
| SDV = getDbgValue(Val, Variable, Expr, dl, DbgSDNodeOrder); |
| DAG.AddDbgValue(SDV, Val.getNode(), false); |
| } |
| } else |
| DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
| DanglingDebugInfoMap[V] = DanglingDebugInfo(); |
| } |
| } |
| |
| /// getCopyFromRegs - If there was virtual register allocated for the value V |
| /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. |
| SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { |
| DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V); |
| SDValue Result; |
| |
| if (It != FuncInfo.ValueMap.end()) { |
| unsigned InReg = It->second; |
| |
| RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), |
| DAG.getDataLayout(), InReg, Ty, isABIRegCopy(V)); |
| SDValue Chain = DAG.getEntryNode(); |
| Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, |
| V); |
| resolveDanglingDebugInfo(V, Result); |
| } |
| |
| return Result; |
| } |
| |
| /// getValue - Return an SDValue for the given Value. |
| SDValue SelectionDAGBuilder::getValue(const Value *V) { |
| // If we already have an SDValue for this value, use it. It's important |
| // to do this first, so that we don't create a CopyFromReg if we already |
| // have a regular SDValue. |
| SDValue &N = NodeMap[V]; |
| if (N.getNode()) return N; |
| |
| // If there's a virtual register allocated and initialized for this |
| // value, use it. |
| if (SDValue copyFromReg = getCopyFromRegs(V, V->getType())) |
| return copyFromReg; |
| |
| // Otherwise create a new SDValue and remember it. |
| SDValue Val = getValueImpl(V); |
| NodeMap[V] = Val; |
| resolveDanglingDebugInfo(V, Val); |
| return Val; |
| } |
| |
| // Return true if SDValue exists for the given Value |
| bool SelectionDAGBuilder::findValue(const Value *V) const { |
| return (NodeMap.find(V) != NodeMap.end()) || |
| (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end()); |
| } |
| |
| /// getNonRegisterValue - Return an SDValue for the given Value, but |
| /// don't look in FuncInfo.ValueMap for a virtual register. |
| SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { |
| // If we already have an SDValue for this value, use it. |
| SDValue &N = NodeMap[V]; |
| if (N.getNode()) { |
| if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) { |
| // Remove the debug location from the node as the node is about to be used |
| // in a location which may differ from the original debug location. This |
| // is relevant to Constant and ConstantFP nodes because they can appear |
| // as constant expressions inside PHI nodes. |
| N->setDebugLoc(DebugLoc()); |
| } |
| return N; |
| } |
| |
| // Otherwise create a new SDValue and remember it. |
| SDValue Val = getValueImpl(V); |
| NodeMap[V] = Val; |
| resolveDanglingDebugInfo(V, Val); |
| return Val; |
| } |
| |
| /// getValueImpl - Helper function for getValue and getNonRegisterValue. |
| /// Create an SDValue for the given value. |
| SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| |
| if (const Constant *C = dyn_cast<Constant>(V)) { |
| EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); |
| |
| if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) |
| return DAG.getConstant(*CI, getCurSDLoc(), VT); |
| |
| if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) |
| return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); |
| |
| if (isa<ConstantPointerNull>(C)) { |
| unsigned AS = V->getType()->getPointerAddressSpace(); |
| return DAG.getConstant(0, getCurSDLoc(), |
| TLI.getPointerTy(DAG.getDataLayout(), AS)); |
| } |
| |
| if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) |
| return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); |
| |
| if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) |
| return DAG.getUNDEF(VT); |
| |
| if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { |
| visit(CE->getOpcode(), *CE); |
| SDValue N1 = NodeMap[V]; |
| assert(N1.getNode() && "visit didn't populate the NodeMap!"); |
| return N1; |
| } |
| |
| if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { |
| SmallVector<SDValue, 4> Constants; |
| for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); |
| OI != OE; ++OI) { |
| SDNode *Val = getValue(*OI).getNode(); |
| // If the operand is an empty aggregate, there are no values. |
| if (!Val) continue; |
| // Add each leaf value from the operand to the Constants list |
| // to form a flattened list of all the values. |
| for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) |
| Constants.push_back(SDValue(Val, i)); |
| } |
| |
| return DAG.getMergeValues(Constants, getCurSDLoc()); |
| } |
| |
| if (const ConstantDataSequential *CDS = |
| dyn_cast<ConstantDataSequential>(C)) { |
| SmallVector<SDValue, 4> Ops; |
| for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { |
| SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); |
| // Add each leaf value from the operand to the Constants list |
| // to form a flattened list of all the values. |
| for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) |
| Ops.push_back(SDValue(Val, i)); |
| } |
| |
| if (isa<ArrayType>(CDS->getType())) |
| return DAG.getMergeValues(Ops, getCurSDLoc()); |
| return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); |
| } |
| |
| if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { |
| assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && |
| "Unknown struct or array constant!"); |
| |
| SmallVector<EVT, 4> ValueVTs; |
| ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); |
| unsigned NumElts = ValueVTs.size(); |
| if (NumElts == 0) |
| return SDValue(); // empty struct |
| SmallVector<SDValue, 4> Constants(NumElts); |
| for (unsigned i = 0; i != NumElts; ++i) { |
| EVT EltVT = ValueVTs[i]; |
| if (isa<UndefValue>(C)) |
| Constants[i] = DAG.getUNDEF(EltVT); |
| else if (EltVT.isFloatingPoint()) |
| Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); |
| else |
| Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); |
| } |
| |
| return DAG.getMergeValues(Constants, getCurSDLoc()); |
| } |
| |
| if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) |
| return DAG.getBlockAddress(BA, VT); |
| |
| VectorType *VecTy = cast<VectorType>(V->getType()); |
| unsigned NumElements = VecTy->getNumElements(); |
| |
| // Now that we know the number and type of the elements, get that number of |
| // elements into the Ops array based on what kind of constant it is. |
| SmallVector<SDValue, 16> Ops; |
| if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { |
| for (unsigned i = 0; i != NumElements; ++i) |
| Ops.push_back(getValue(CV->getOperand(i))); |
| } else { |
| assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!"); |
| EVT EltVT = |
| TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); |
| |
| SDValue Op; |
| if (EltVT.isFloatingPoint()) |
| Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); |
| else |
| Op = DAG.getConstant(0, getCurSDLoc(), EltVT); |
| Ops.assign(NumElements, Op); |
| } |
| |
| // Create a BUILD_VECTOR node. |
| return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); |
| } |
| |
| // If this is a static alloca, generate it as the frameindex instead of |
| // computation. |
| if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { |
| DenseMap<const AllocaInst*, int>::iterator SI = |
| FuncInfo.StaticAllocaMap.find(AI); |
| if (SI != FuncInfo.StaticAllocaMap.end()) |
| return DAG.getFrameIndex(SI->second, |
| TLI.getFrameIndexTy(DAG.getDataLayout())); |
| } |
| |
| // If this is an instruction which fast-isel has deferred, select it now. |
| if (const Instruction *Inst = dyn_cast<Instruction>(V)) { |
| unsigned InReg = FuncInfo.InitializeRegForValue(Inst); |
| |
| RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, |
| Inst->getType(), isABIRegCopy(V)); |
| SDValue Chain = DAG.getEntryNode(); |
| return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); |
| } |
| |
| llvm_unreachable("Can't get register for value!"); |
| } |
| |
| void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) { |
| auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
| bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX; |
| bool IsCoreCLR = Pers == EHPersonality::CoreCLR; |
| MachineBasicBlock *CatchPadMBB = FuncInfo.MBB; |
| // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. |
| if (IsMSVCCXX || IsCoreCLR) |
| CatchPadMBB->setIsEHFuncletEntry(); |
| |
| DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot())); |
| } |
| |
| void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) { |
| // Update machine-CFG edge. |
| MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()]; |
| FuncInfo.MBB->addSuccessor(TargetMBB); |
| |
| auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
| bool IsSEH = isAsynchronousEHPersonality(Pers); |
| if (IsSEH) { |
| // If this is not a fall-through branch or optimizations are switched off, |
| // emit the branch. |
| if (TargetMBB != NextBlock(FuncInfo.MBB) || |
| TM.getOptLevel() == CodeGenOpt::None) |
| DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, |
| getControlRoot(), DAG.getBasicBlock(TargetMBB))); |
| return; |
| } |
| |
| // Figure out the funclet membership for the catchret's successor. |
| // This will be used by the FuncletLayout pass to determine how to order the |
| // BB's. |
| // A 'catchret' returns to the outer scope's color. |
| Value *ParentPad = I.getCatchSwitchParentPad(); |
| const BasicBlock *SuccessorColor; |
| if (isa<ConstantTokenNone>(ParentPad)) |
| SuccessorColor = &FuncInfo.Fn->getEntryBlock(); |
| else |
| SuccessorColor = cast<Instruction>(ParentPad)->getParent(); |
| assert(SuccessorColor && "No parent funclet for catchret!"); |
| MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor]; |
| assert(SuccessorColorMBB && "No MBB for SuccessorColor!"); |
| |
| // Create the terminator node. |
| SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, |
| getControlRoot(), DAG.getBasicBlock(TargetMBB), |
| DAG.getBasicBlock(SuccessorColorMBB)); |
| DAG.setRoot(Ret); |
| } |
| |
| void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { |
| // Don't emit any special code for the cleanuppad instruction. It just marks |
| // the start of a funclet. |
| FuncInfo.MBB->setIsEHFuncletEntry(); |
| FuncInfo.MBB->setIsCleanupFuncletEntry(); |
| } |
| |
| /// When an invoke or a cleanupret unwinds to the next EH pad, there are |
| /// many places it could ultimately go. In the IR, we have a single unwind |
| /// destination, but in the machine CFG, we enumerate all the possible blocks. |
| /// This function skips over imaginary basic blocks that hold catchswitch |
| /// instructions, and finds all the "real" machine |
| /// basic block destinations. As those destinations may not be successors of |
| /// EHPadBB, here we also calculate the edge probability to those destinations. |
| /// The passed-in Prob is the edge probability to EHPadBB. |
| static void findUnwindDestinations( |
| FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, |
| BranchProbability Prob, |
| SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> |
| &UnwindDests) { |
| EHPersonality Personality = |
| classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); |
| bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; |
| bool IsCoreCLR = Personality == EHPersonality::CoreCLR; |
| |
| while (EHPadBB) { |
| const Instruction *Pad = EHPadBB->getFirstNonPHI(); |
| BasicBlock *NewEHPadBB = nullptr; |
| if (isa<LandingPadInst>(Pad)) { |
| // Stop on landingpads. They are not funclets. |
| UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); |
| break; |
| } else if (isa<CleanupPadInst>(Pad)) { |
| // Stop on cleanup pads. Cleanups are always funclet entries for all known |
| // personalities. |
| UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); |
| UnwindDests.back().first->setIsEHFuncletEntry(); |
| break; |
| } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { |
| // Add the catchpad handlers to the possible destinations. |
| for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { |
| UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); |
| // For MSVC++ and the CLR, catchblocks are funclets and need prologues. |
| if (IsMSVCCXX || IsCoreCLR) |
| UnwindDests.back().first->setIsEHFuncletEntry(); |
| } |
| NewEHPadBB = CatchSwitch->getUnwindDest(); |
| } else { |
| continue; |
| } |
| |
| BranchProbabilityInfo *BPI = FuncInfo.BPI; |
| if (BPI && NewEHPadBB) |
| Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); |
| EHPadBB = NewEHPadBB; |
| } |
| } |
| |
| void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { |
| // Update successor info. |
| SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; |
| auto UnwindDest = I.getUnwindDest(); |
| BranchProbabilityInfo *BPI = FuncInfo.BPI; |
| BranchProbability UnwindDestProb = |
| (BPI && UnwindDest) |
| ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest) |
| : BranchProbability::getZero(); |
| findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests); |
| for (auto &UnwindDest : UnwindDests) { |
| UnwindDest.first->setIsEHPad(); |
| addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); |
| } |
| FuncInfo.MBB->normalizeSuccProbs(); |
| |
| // Create the terminator node. |
| SDValue Ret = |
| DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot()); |
| DAG.setRoot(Ret); |
| } |
| |
| void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) { |
| report_fatal_error("visitCatchSwitch not yet implemented!"); |
| } |
| |
| void SelectionDAGBuilder::visitRet(const ReturnInst &I) { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| auto &DL = DAG.getDataLayout(); |
| SDValue Chain = getControlRoot(); |
| SmallVector<ISD::OutputArg, 8> Outs; |
| SmallVector<SDValue, 8> OutVals; |
| |
| // Calls to @llvm.experimental.deoptimize don't generate a return value, so |
| // lower |
| // |
| // %val = call <ty> @llvm.experimental.deoptimize() |
| // ret <ty> %val |
| // |
| // differently. |
| if (I.getParent()->getTerminatingDeoptimizeCall()) { |
| LowerDeoptimizingReturn(); |
| return; |
| } |
| |
| if (!FuncInfo.CanLowerReturn) { |
| unsigned DemoteReg = FuncInfo.DemoteRegister; |
| const Function *F = I.getParent()->getParent(); |
| |
| // Emit a store of the return value through the virtual register. |
| // Leave Outs empty so that LowerReturn won't try to load return |
| // registers the usual way. |
| SmallVector<EVT, 1> PtrValueVTs; |
| ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()), |
| PtrValueVTs); |
| |
| SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), |
| DemoteReg, PtrValueVTs[0]); |
| SDValue RetOp = getValue(I.getOperand(0)); |
| |
| SmallVector<EVT, 4> ValueVTs; |
| SmallVector<uint64_t, 4> Offsets; |
| ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets); |
| unsigned NumValues = ValueVTs.size(); |
| |
| // An aggregate return value cannot wrap around the address space, so |
| // offsets to its parts don't wrap either. |
| SDNodeFlags Flags; |
| Flags.setNoUnsignedWrap(true); |
| |
| SmallVector<SDValue, 4> Chains(NumValues); |
| for (unsigned i = 0; i != NumValues; ++i) { |
| SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(), |
| RetPtr.getValueType(), RetPtr, |
| DAG.getIntPtrConstant(Offsets[i], |
| getCurSDLoc()), |
| Flags); |
| Chains[i] = DAG.getStore(Chain, getCurSDLoc(), |
| SDValue(RetOp.getNode(), RetOp.getResNo() + i), |
| // FIXME: better loc info would be nice. |
| Add, MachinePointerInfo()); |
| } |
| |
| Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), |
| MVT::Other, Chains); |
| } else if (I.getNumOperands() != 0) { |
| SmallVector<EVT, 4> ValueVTs; |
| ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs); |
| unsigned NumValues = ValueVTs.size(); |
| if (NumValues) { |
| SDValue RetOp = getValue(I.getOperand(0)); |
| |
| const Function *F = I.getParent()->getParent(); |
| |
| ISD::NodeType ExtendKind = ISD::ANY_EXTEND; |
| if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex, |
| Attribute::SExt)) |
| ExtendKind = ISD::SIGN_EXTEND; |
| else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex, |
| Attribute::ZExt)) |
| ExtendKind = ISD::ZERO_EXTEND; |
| |
| LLVMContext &Context = F->getContext(); |
| bool RetInReg = F->getAttributes().hasAttribute( |
| AttributeList::ReturnIndex, Attribute::InReg); |
| |
| for (unsigned j = 0; j != NumValues; ++j) { |
| EVT VT = ValueVTs[j]; |
| |
| if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) |
| VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind); |
| |
| unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, VT); |
| MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, VT); |
| SmallVector<SDValue, 4> Parts(NumParts); |
| getCopyToParts(DAG, getCurSDLoc(), |
| SDValue(RetOp.getNode(), RetOp.getResNo() + j), |
| &Parts[0], NumParts, PartVT, &I, ExtendKind, true); |
| |
| // 'inreg' on function refers to return value |
| ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
| if (RetInReg) |
| Flags.setInReg(); |
| |
| // Propagate extension type if any |
| if (ExtendKind == ISD::SIGN_EXTEND) |
| Flags.setSExt(); |
| else if (ExtendKind == ISD::ZERO_EXTEND) |
| Flags.setZExt(); |
| |
| for (unsigned i = 0; i < NumParts; ++i) { |
| Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(), |
| VT, /*isfixed=*/true, 0, 0)); |
| OutVals.push_back(Parts[i]); |
| } |
| } |
| } |
| } |
| |
| // Push in swifterror virtual register as the last element of Outs. This makes |
| // sure swifterror virtual register will be returned in the swifterror |
| // physical register. |
| const Function *F = I.getParent()->getParent(); |
| if (TLI.supportSwiftError() && |
| F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { |
| assert(FuncInfo.SwiftErrorArg && "Need a swift error argument"); |
| ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
| Flags.setSwiftError(); |
| Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/, |
| EVT(TLI.getPointerTy(DL)) /*argvt*/, |
| true /*isfixed*/, 1 /*origidx*/, |
| 0 /*partOffs*/)); |
| // Create SDNode for the swifterror virtual register. |
| OutVals.push_back( |
| DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt( |
| &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first, |
| EVT(TLI.getPointerTy(DL)))); |
| } |
| |
| bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg(); |
| CallingConv::ID CallConv = |
| DAG.getMachineFunction().getFunction()->getCallingConv(); |
| Chain = DAG.getTargetLoweringInfo().LowerReturn( |
| Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); |
| |
| // Verify that the target's LowerReturn behaved as expected. |
| assert(Chain.getNode() && Chain.getValueType() == MVT::Other && |
| "LowerReturn didn't return a valid chain!"); |
| |
| // Update the DAG with the new chain value resulting from return lowering. |
| DAG.setRoot(Chain); |
| } |
| |
| /// CopyToExportRegsIfNeeded - If the given value has virtual registers |
| /// created for it, emit nodes to copy the value into the virtual |
| /// registers. |
| void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { |
| // Skip empty types |
| if (V->getType()->isEmptyTy()) |
| return; |
| |
| DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V); |
| if (VMI != FuncInfo.ValueMap.end()) { |
| assert(!V->use_empty() && "Unused value assigned virtual registers!"); |
| CopyValueToVirtualRegister(V, VMI->second); |
| } |
| } |
| |
| /// ExportFromCurrentBlock - If this condition isn't known to be exported from |
| /// the current basic block, add it to ValueMap now so that we'll get a |
| /// CopyTo/FromReg. |
| void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { |
| // No need to export constants. |
| if (!isa<Instruction>(V) && !isa<Argument>(V)) return; |
| |
| // Already exported? |
| if (FuncInfo.isExportedInst(V)) return; |
| |
| unsigned Reg = FuncInfo.InitializeRegForValue(V); |
| CopyValueToVirtualRegister(V, Reg); |
| } |
| |
| bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, |
| const BasicBlock *FromBB) { |
| // The operands of the setcc have to be in this block. We don't know |
| // how to export them from some other block. |
| if (const Instruction *VI = dyn_cast<Instruction>(V)) { |
| // Can export from current BB. |
| if (VI->getParent() == FromBB) |
| return true; |
| |
| // Is already exported, noop. |
| return FuncInfo.isExportedInst(V); |
| } |
| |
| // If this is an argument, we can export it if the BB is the entry block or |
| // if it is already exported. |
| if (isa<Argument>(V)) { |
| if (FromBB == &FromBB->getParent()->getEntryBlock()) |
| return true; |
| |
| // Otherwise, can only export this if it is already exported. |
| return FuncInfo.isExportedInst(V); |
| } |
| |
| // Otherwise, constants can always be exported. |
| return true; |
| } |
| |
| /// Return branch probability calculated by BranchProbabilityInfo for IR blocks. |
| BranchProbability |
| SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src, |
| const MachineBasicBlock *Dst) const { |
| BranchProbabilityInfo *BPI = FuncInfo.BPI; |
| const BasicBlock *SrcBB = Src->getBasicBlock(); |
| const BasicBlock *DstBB = Dst->getBasicBlock(); |
| if (!BPI) { |
| // If BPI is not available, set the default probability as 1 / N, where N is |
| // the number of successors. |
| auto SuccSize = std::max<uint32_t>( |
| std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1); |
| return BranchProbability(1, SuccSize); |
| } |
| return BPI->getEdgeProbability(SrcBB, DstBB); |
| } |
| |
| void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src, |
| MachineBasicBlock *Dst, |
| BranchProbability Prob) { |
| if (!FuncInfo.BPI) |
| Src->addSuccessorWithoutProb(Dst); |
| else { |
| if (Prob.isUnknown()) |
| Prob = getEdgeProbability(Src, Dst); |
| Src->addSuccessor(Dst, Prob); |
| } |
| } |
| |
| static bool InBlock(const Value *V, const BasicBlock *BB) { |
| if (const Instruction *I = dyn_cast<Instruction>(V)) |
| return I->getParent() == BB; |
| return true; |
| } |
| |
| /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. |
| /// This function emits a branch and is used at the leaves of an OR or an |
| /// AND operator tree. |
| void |
| SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, |
| MachineBasicBlock *TBB, |
| MachineBasicBlock *FBB, |
| MachineBasicBlock *CurBB, |
| MachineBasicBlock *SwitchBB, |
| BranchProbability TProb, |
| BranchProbability FProb, |
| bool InvertCond) { |
| const BasicBlock *BB = CurBB->getBasicBlock(); |
| |
| // If the leaf of the tree is a comparison, merge the condition into |
| // the caseblock. |
| if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { |
| // The operands of the cmp have to be in this block. We don't know |
| // how to export them from some other block. If this is the first block |
| // of the sequence, no exporting is needed. |
| if (CurBB == SwitchBB || |
| (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && |
| isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { |
| ISD::CondCode Condition; |
| if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { |
| ICmpInst::Predicate Pred = |
| InvertCond ? IC->getInversePredicate() : IC->getPredicate(); |
| Condition = getICmpCondCode(Pred); |
| } else { |
| const FCmpInst *FC = cast<FCmpInst>(Cond); |
| FCmpInst::Predicate Pred = |
| InvertCond ? FC->getInversePredicate() : FC->getPredicate(); |
| Condition = getFCmpCondCode(Pred); |
| if (TM.Options.NoNaNsFPMath) |
| Condition = getFCmpCodeWithoutNaN(Condition); |
| } |
| |
| CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, |
| TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); |
| SwitchCases.push_back(CB); |
| return; |
| } |
| } |
| |
| // Create a CaseBlock record representing this branch. |
| ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; |
| CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), |
| nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); |
| SwitchCases.push_back(CB); |
| } |
| |
| /// FindMergedConditions - If Cond is an expression like |
| void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, |
| MachineBasicBlock *TBB, |
| MachineBasicBlock *FBB, |
| MachineBasicBlock *CurBB, |
| MachineBasicBlock *SwitchBB, |
| Instruction::BinaryOps Opc, |
| BranchProbability TProb, |
| BranchProbability FProb, |
| bool InvertCond) { |
| // Skip over not part of the tree and remember to invert op and operands at |
| // next level. |
| if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) { |
| const Value *CondOp = BinaryOperator::getNotArgument(Cond); |
| if (InBlock(CondOp, CurBB->getBasicBlock())) { |
| FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb, |
| !InvertCond); |
| return; |
| } |
| } |
| |
| const Instruction *BOp = dyn_cast<Instruction>(Cond); |
| // Compute the effective opcode for Cond, taking into account whether it needs |
| // to be inverted, e.g. |
| // and (not (or A, B)), C |
| // gets lowered as |
| // and (and (not A, not B), C) |
| unsigned BOpc = 0; |
| if (BOp) { |
| BOpc = BOp->getOpcode(); |
| if (InvertCond) { |
| if (BOpc == Instruction::And) |
| BOpc = Instruction::Or; |
| else if (BOpc == Instruction::Or) |
| BOpc = Instruction::And; |
| } |
| } |
| |
| // If this node is not part of the or/and tree, emit it as a branch. |
| if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) || |
| BOpc != Opc || !BOp->hasOneUse() || |
| BOp->getParent() != CurBB->getBasicBlock() || |
| !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) || |
| !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) { |
| EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, |
| TProb, FProb, InvertCond); |
| return; |
| } |
| |
| // Create TmpBB after CurBB. |
| MachineFunction::iterator BBI(CurBB); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); |
| CurBB->getParent()->insert(++BBI, TmpBB); |
| |
| if (Opc == Instruction::Or) { |
| // Codegen X | Y as: |
| // BB1: |
| // jmp_if_X TBB |
| // jmp TmpBB |
| // TmpBB: |
| // jmp_if_Y TBB |
| // jmp FBB |
| // |
| |
| // We have flexibility in setting Prob for BB1 and Prob for TmpBB. |
| // The requirement is that |
| // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) |
| // = TrueProb for original BB. |
| // Assuming the original probabilities are A and B, one choice is to set |
| // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to |
| // A/(1+B) and 2B/(1+B). This choice assumes that |
| // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. |
| // Another choice is to assume TrueProb for BB1 equals to TrueProb for |
| // TmpBB, but the math is more complicated. |
| |
| auto NewTrueProb = TProb / 2; |
| auto NewFalseProb = TProb / 2 + FProb; |
| // Emit the LHS condition. |
| FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc, |
| NewTrueProb, NewFalseProb, InvertCond); |
| |
| // Normalize A/2 and B to get A/(1+B) and 2B/(1+B). |
| SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb}; |
| BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); |
| // Emit the RHS condition into TmpBB. |
| FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, |
| Probs[0], Probs[1], InvertCond); |
| } else { |
| assert(Opc == Instruction::And && "Unknown merge op!"); |
| // Codegen X & Y as: |
| // BB1: |
| // jmp_if_X TmpBB |
| // jmp FBB |
| // TmpBB: |
| // jmp_if_Y TBB |
| // jmp FBB |
| // |
| // This requires creation of TmpBB after CurBB. |
| |
| // We have flexibility in setting Prob for BB1 and Prob for TmpBB. |
| // The requirement is that |
| // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) |
| // = FalseProb for original BB. |
| // Assuming the original probabilities are A and B, one choice is to set |
| // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to |
| // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == |
| // TrueProb for BB1 * FalseProb for TmpBB. |
| |
| auto NewTrueProb = TProb + FProb / 2; |
| auto NewFalseProb = FProb / 2; |
| // Emit the LHS condition. |
| FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc, |
| NewTrueProb, NewFalseProb, InvertCond); |
| |
| // Normalize A and B/2 to get 2A/(1+A) and B/(1+A). |
| SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2}; |
| BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); |
| // Emit the RHS condition into TmpBB. |
| FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc, |
| Probs[0], Probs[1], InvertCond); |
| } |
| } |
| |
| /// If the set of cases should be emitted as a series of branches, return true. |
| /// If we should emit this as a bunch of and/or'd together conditions, return |
| /// false. |
| bool |
| SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { |
| if (Cases.size() != 2) return true; |
| |
| // If this is two comparisons of the same values or'd or and'd together, they |
| // will get folded into a single comparison, so don't emit two blocks. |
| if ((Cases[0].CmpLHS == Cases[1].CmpLHS && |
| Cases[0].CmpRHS == Cases[1].CmpRHS) || |
| (Cases[0].CmpRHS == Cases[1].CmpLHS && |
| Cases[0].CmpLHS == Cases[1].CmpRHS)) { |
| return false; |
| } |
| |
| // Handle: (X != null) | (Y != null) --> (X|Y) != 0 |
| // Handle: (X == null) & (Y == null) --> (X|Y) == 0 |
| if (Cases[0].CmpRHS == Cases[1].CmpRHS && |
| Cases[0].CC == Cases[1].CC && |
| isa<Constant>(Cases[0].CmpRHS) && |
| cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { |
| if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) |
| return false; |
| if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| void SelectionDAGBuilder::visitBr(const BranchInst &I) { |
| MachineBasicBlock *BrMBB = FuncInfo.MBB; |
| |
| // Update machine-CFG edges. |
| MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; |
| |
| if (I.isUnconditional()) { |
| // Update machine-CFG edges. |
| BrMBB->addSuccessor(Succ0MBB); |
| |
| // If this is not a fall-through branch or optimizations are switched off, |
| // emit the branch. |
| if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) |
| DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), |
| MVT::Other, getControlRoot(), |
| DAG.getBasicBlock(Succ0MBB))); |
| |
| return; |
| } |
| |
| // If this condition is one of the special cases we handle, do special stuff |
| // now. |
| const Value *CondVal = I.getCondition(); |
| MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; |
| |
| // If this is a series of conditions that are or'd or and'd together, emit |
| // this as a sequence of branches instead of setcc's with and/or operations. |
| // As long as jumps are not expensive, this should improve performance. |
| // For example, instead of something like: |
| // cmp A, B |
| // C = seteq |
| // cmp D, E |
| // F = setle |
| // or C, F |
| // jnz foo |
| // Emit: |
| // cmp A, B |
| // je foo |
| // cmp D, E |
| // jle foo |
| if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) { |
| Instruction::BinaryOps Opcode = BOp->getOpcode(); |
| if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() && |
| !I.getMetadata(LLVMContext::MD_unpredictable) && |
| (Opcode == Instruction::And || Opcode == Instruction::Or)) { |
| FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, |
| Opcode, |
| getEdgeProbability(BrMBB, Succ0MBB), |
| getEdgeProbability(BrMBB, Succ1MBB), |
| /*InvertCond=*/false); |
| // If the compares in later blocks need to use values not currently |
| // exported from this block, export them now. This block should always |
| // be the first entry. |
| assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); |
| |
| // Allow some cases to be rejected. |
| if (ShouldEmitAsBranches(SwitchCases)) { |
| for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) { |
| ExportFromCurrentBlock(SwitchCases[i].CmpLHS); |
| ExportFromCurrentBlock(SwitchCases[i].CmpRHS); |
| } |
| |
| // Emit the branch for this block. |
| visitSwitchCase(SwitchCases[0], BrMBB); |
| SwitchCases.erase(SwitchCases.begin()); |
| return; |
| } |
| |
| // Okay, we decided not to do this, remove any inserted MBB's and clear |
| // SwitchCases. |
| for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) |
| FuncInfo.MF->erase(SwitchCases[i].ThisBB); |
| |
| SwitchCases.clear(); |
| } |
| } |
| |
| // Create a CaseBlock record representing this branch. |
| CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), |
| nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc()); |
| |
| // Use visitSwitchCase to actually insert the fast branch sequence for this |
| // cond branch. |
| visitSwitchCase(CB, BrMBB); |
| } |
| |
| /// visitSwitchCase - Emits the necessary code to represent a single node in |
| /// the binary search tree resulting from lowering a switch instruction. |
| void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, |
| MachineBasicBlock *SwitchBB) { |
| SDValue Cond; |
| SDValue CondLHS = getValue(CB.CmpLHS); |
| SDLoc dl = CB.DL; |
| |
| // Build the setcc now. |
| if (!CB.CmpMHS) { |
| // Fold "(X == true)" to X and "(X == false)" to !X to |
| // handle common cases produced by branch lowering. |
| if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && |
| CB.CC == ISD::SETEQ) |
| Cond = CondLHS; |
| else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && |
| CB.CC == ISD::SETEQ) { |
| SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); |
| Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); |
| } else |
| Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); |
| } else { |
| assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); |
| |
| const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); |
| const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); |
| |
| SDValue CmpOp = getValue(CB.CmpMHS); |
| EVT VT = CmpOp.getValueType(); |
| |
| if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { |
| Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), |
| ISD::SETLE); |
| } else { |
| SDValue SUB = DAG.getNode(ISD::SUB, dl, |
| VT, CmpOp, DAG.getConstant(Low, dl, VT)); |
| Cond = DAG.getSetCC(dl, MVT::i1, SUB, |
| DAG.getConstant(High-Low, dl, VT), ISD::SETULE); |
| } |
| } |
| |
| // Update successor info |
| addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); |
| // TrueBB and FalseBB are always different unless the incoming IR is |
| // degenerate. This only happens when running llc on weird IR. |
| if (CB.TrueBB != CB.FalseBB) |
| addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb); |
| SwitchBB->normalizeSuccProbs(); |
| |
| // If the lhs block is the next block, invert the condition so that we can |
| // fall through to the lhs instead of the rhs block. |
| if (CB.TrueBB == NextBlock(SwitchBB)) { |
| std::swap(CB.TrueBB, CB.FalseBB); |
| SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); |
| Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); |
| } |
| |
| SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
| MVT::Other, getControlRoot(), Cond, |
| DAG.getBasicBlock(CB.TrueBB)); |
| |
| // Insert the false branch. Do this even if it's a fall through branch, |
| // this makes it easier to do DAG optimizations which require inverting |
| // the branch condition. |
| BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, |
| DAG.getBasicBlock(CB.FalseBB)); |
| |
| DAG.setRoot(BrCond); |
| } |
| |
| /// visitJumpTable - Emit JumpTable node in the current MBB |
| void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { |
| // Emit the code for the jump table |
| assert(JT.Reg != -1U && "Should lower JT Header first!"); |
| EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); |
| SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), |
| JT.Reg, PTy); |
| SDValue Table = DAG.getJumpTable(JT.JTI, PTy); |
| SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), |
| MVT::Other, Index.getValue(1), |
| Table, Index); |
| DAG.setRoot(BrJumpTable); |
| } |
| |
| /// visitJumpTableHeader - This function emits necessary code to produce index |
| /// in the JumpTable from switch case. |
| void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, |
| JumpTableHeader &JTH, |
| MachineBasicBlock *SwitchBB) { |
| SDLoc dl = getCurSDLoc(); |
| |
| // Subtract the lowest switch case value from the value being switched on and |
| // conditional branch to default mbb if the result is greater than the |
| // difference between smallest and largest cases. |
| SDValue SwitchOp = getValue(JTH.SValue); |
| EVT VT = SwitchOp.getValueType(); |
| SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, |
| DAG.getConstant(JTH.First, dl, VT)); |
| |
| // The SDNode we just created, which holds the value being switched on minus |
| // the smallest case value, needs to be copied to a virtual register so it |
| // can be used as an index into the jump table in a subsequent basic block. |
| // This value may be smaller or larger than the target's pointer type, and |
| // therefore require extension or truncating. |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); |
| |
| unsigned JumpTableReg = |
| FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); |
| SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, |
| JumpTableReg, SwitchOp); |
| JT.Reg = JumpTableReg; |
| |
| // Emit the range check for the jump table, and branch to the default block |
| // for the switch statement if the value being switched on exceeds the largest |
| // case in the switch. |
| SDValue CMP = DAG.getSetCC( |
| dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), |
| Sub.getValueType()), |
| Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); |
| |
| SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
| MVT::Other, CopyTo, CMP, |
| DAG.getBasicBlock(JT.Default)); |
| |
| // Avoid emitting unnecessary branches to the next block. |
| if (JT.MBB != NextBlock(SwitchBB)) |
| BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, |
| DAG.getBasicBlock(JT.MBB)); |
| |
| DAG.setRoot(BrCond); |
| } |
| |
| /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global |
| /// variable if there exists one. |
| static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, |
| SDValue &Chain) { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent()); |
| MachineSDNode *Node = |
| DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); |
| if (Global) { |
| MachinePointerInfo MPInfo(Global); |
| MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1); |
| auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | |
| MachineMemOperand::MODereferenceable; |
| *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8, |
| DAG.getEVTAlignment(PtrTy)); |
| Node->setMemRefs(MemRefs, MemRefs + 1); |
| } |
| return SDValue(Node, 0); |
| } |
| |
| /// Codegen a new tail for a stack protector check ParentMBB which has had its |
| /// tail spliced into a stack protector check success bb. |
| /// |
| /// For a high level explanation of how this fits into the stack protector |
| /// generation see the comment on the declaration of class |
| /// StackProtectorDescriptor. |
| void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, |
| MachineBasicBlock *ParentBB) { |
| |
| // First create the loads to the guard/stack slot for the comparison. |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); |
| |
| MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); |
| int FI = MFI.getStackProtectorIndex(); |
| |
| SDValue Guard; |
| SDLoc dl = getCurSDLoc(); |
| SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); |
| const Module &M = *ParentBB->getParent()->getFunction()->getParent(); |
| unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext())); |
| |
| // Generate code to load the content of the guard slot. |
| SDValue StackSlot = DAG.getLoad( |
| PtrTy, dl, DAG.getEntryNode(), StackSlotPtr, |
| MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, |
| MachineMemOperand::MOVolatile); |
| |
| // Retrieve guard check function, nullptr if instrumentation is inlined. |
| if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) { |
| // The target provides a guard check function to validate the guard value. |
| // Generate a call to that function with the content of the guard slot as |
| // argument. |
| auto *Fn = cast<Function>(GuardCheck); |
| FunctionType *FnTy = Fn->getFunctionType(); |
| assert(FnTy->getNumParams() == 1 && "Invalid function signature"); |
| |
| TargetLowering::ArgListTy Args; |
| TargetLowering::ArgListEntry Entry; |
| Entry.Node = StackSlot; |
| Entry.Ty = FnTy->getParamType(0); |
| if (Fn->hasAttribute(1, Attribute::AttrKind::InReg)) |
| Entry.IsInReg = true; |
| Args.push_back(Entry); |
| |
| TargetLowering::CallLoweringInfo CLI(DAG); |
| CLI.setDebugLoc(getCurSDLoc()) |
| .setChain(DAG.getEntryNode()) |
| .setCallee(Fn->getCallingConv(), FnTy->getReturnType(), |
| getValue(GuardCheck), std::move(Args)); |
| |
| std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); |
| DAG.setRoot(Result.second); |
| return; |
| } |
| |
| // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. |
| // Otherwise, emit a volatile load to retrieve the stack guard value. |
| SDValue Chain = DAG.getEntryNode(); |
| if (TLI.useLoadStackGuardNode()) { |
| Guard = getLoadStackGuard(DAG, dl, Chain); |
| } else { |
| const Value *IRGuard = TLI.getSDagStackGuard(M); |
| SDValue GuardPtr = getValue(IRGuard); |
| |
| Guard = |
| DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0), |
| Align, MachineMemOperand::MOVolatile); |
| } |
| |
| // Perform the comparison via a subtract/getsetcc. |
| EVT VT = Guard.getValueType(); |
| SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot); |
| |
| SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), |
| *DAG.getContext(), |
| Sub.getValueType()), |
| Sub, DAG.getConstant(0, dl, VT), ISD::SETNE); |
| |
| // If the sub is not 0, then we know the guard/stackslot do not equal, so |
| // branch to failure MBB. |
| SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, |
| MVT::Other, StackSlot.getOperand(0), |
| Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); |
| // Otherwise branch to success MBB. |
| SDValue Br = DAG.getNode(ISD::BR, dl, |
| MVT::Other, BrCond, |
| DAG.getBasicBlock(SPD.getSuccessMBB())); |
| |
| DAG.setRoot(Br); |
| } |
| |
| /// Codegen the failure basic block for a stack protector check. |
| /// |
| /// A failure stack protector machine basic block consists simply of a call to |
| /// __stack_chk_fail(). |
| /// |
| /// For a high level explanation of how this fits into the stack protector |
| /// generation see the comment on the declaration of class |
| /// StackProtectorDescriptor. |
| void |
| SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| SDValue Chain = |
| TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, |
| None, false, getCurSDLoc(), false, false).second; |
| DAG.setRoot(Chain); |
| } |
| |
| /// visitBitTestHeader - This function emits necessary code to produce value |
| /// suitable for "bit tests" |
| void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, |
| MachineBasicBlock *SwitchBB) { |
| SDLoc dl = getCurSDLoc(); |
| |
| // Subtract the minimum value |
| SDValue SwitchOp = getValue(B.SValue); |
| EVT VT = SwitchOp.getValueType(); |
| SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, |
| DAG.getConstant(B.First, dl, VT)); |
| |
| // Check range |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| SDValue RangeCmp = DAG.getSetCC( |
| dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), |
| Sub.getValueType()), |
| Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT); |
| |
| // Determine the type of the test operands. |
| bool UsePtrType = false; |
| if (!TLI.isTypeLegal(VT)) |
| UsePtrType = true; |
| else { |
| for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) |
| if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { |
| // Switch table case range are encoded into series of masks. |
| // Just use pointer type, it's guaranteed to fit. |
| UsePtrType = true; |
| break; |
| } |
| } |
| if (UsePtrType) { |
| VT = TLI.getPointerTy(DAG.getDataLayout()); |
| Sub = DAG.getZExtOrTrunc(Sub, dl, VT); |
| } |
| |
| B.RegVT = VT.getSimpleVT(); |
| B.Reg = FuncInfo.CreateReg(B.RegVT); |
| SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); |
| |
| MachineBasicBlock* MBB = B.Cases[0].ThisBB; |
| |
| addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); |
| addSuccessorWithProb(SwitchBB, MBB, B.Prob); |
| SwitchBB->normalizeSuccProbs(); |
| |
| SDValue BrRange = DAG.getNode(ISD::BRCOND, dl, |
| MVT::Other, CopyTo, RangeCmp, |
| DAG.getBasicBlock(B.Default)); |
| |
| // Avoid emitting unnecessary branches to the next block. |
| if (MBB != NextBlock(SwitchBB)) |
| BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange, |
| DAG.getBasicBlock(MBB)); |
| |
| DAG.setRoot(BrRange); |
| } |
| |
| /// visitBitTestCase - this function produces one "bit test" |
| void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, |
| MachineBasicBlock* NextMBB, |
| BranchProbability BranchProbToNext, |
| unsigned Reg, |
| BitTestCase &B, |
| MachineBasicBlock *SwitchBB) { |
| SDLoc dl = getCurSDLoc(); |
| MVT VT = BB.RegVT; |
| SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); |
| SDValue Cmp; |
| unsigned PopCount = countPopulation(B.Mask); |
| const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
| if (PopCount == 1) { |
| // Testing for a single bit; just compare the shift count with what it |
| // would need to be to shift a 1 bit in that position. |
| Cmp = DAG.getSetCC( |
| dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), |
| ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT), |
|