| //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains the X86 implementation of TargetFrameLowering class. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "X86FrameLowering.h" |
| #include "X86InstrBuilder.h" |
| #include "X86InstrInfo.h" |
| #include "X86MachineFunctionInfo.h" |
| #include "X86Subtarget.h" |
| #include "X86TargetMachine.h" |
| #include "llvm/ADT/SmallSet.h" |
| #include "llvm/ADT/Statistic.h" |
| #include "llvm/Analysis/EHPersonalities.h" |
| #include "llvm/CodeGen/MachineFrameInfo.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineModuleInfo.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/WinEHFuncInfo.h" |
| #include "llvm/IR/DataLayout.h" |
| #include "llvm/IR/Function.h" |
| #include "llvm/MC/MCAsmInfo.h" |
| #include "llvm/MC/MCObjectFileInfo.h" |
| #include "llvm/MC/MCSymbol.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Target/TargetOptions.h" |
| #include <cstdlib> |
| |
| #define DEBUG_TYPE "x86-fl" |
| |
| STATISTIC(NumFrameLoopProbe, "Number of loop stack probes used in prologue"); |
| STATISTIC(NumFrameExtraProbe, |
| "Number of extra stack probes generated in prologue"); |
| |
| using namespace llvm; |
| |
| X86FrameLowering::X86FrameLowering(const X86Subtarget &STI, |
| MaybeAlign StackAlignOverride) |
| : TargetFrameLowering(StackGrowsDown, StackAlignOverride.valueOrOne(), |
| STI.is64Bit() ? -8 : -4), |
| STI(STI), TII(*STI.getInstrInfo()), TRI(STI.getRegisterInfo()) { |
| // Cache a bunch of frame-related predicates for this subtarget. |
| SlotSize = TRI->getSlotSize(); |
| Is64Bit = STI.is64Bit(); |
| IsLP64 = STI.isTarget64BitLP64(); |
| // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. |
| Uses64BitFramePtr = STI.isTarget64BitLP64() || STI.isTargetNaCl64(); |
| StackPtr = TRI->getStackRegister(); |
| } |
| |
| bool X86FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { |
| return !MF.getFrameInfo().hasVarSizedObjects() && |
| !MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences() && |
| !MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall(); |
| } |
| |
| /// canSimplifyCallFramePseudos - If there is a reserved call frame, the |
| /// call frame pseudos can be simplified. Having a FP, as in the default |
| /// implementation, is not sufficient here since we can't always use it. |
| /// Use a more nuanced condition. |
| bool |
| X86FrameLowering::canSimplifyCallFramePseudos(const MachineFunction &MF) const { |
| return hasReservedCallFrame(MF) || |
| MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() || |
| (hasFP(MF) && !TRI->hasStackRealignment(MF)) || |
| TRI->hasBasePointer(MF); |
| } |
| |
| // needsFrameIndexResolution - Do we need to perform FI resolution for |
| // this function. Normally, this is required only when the function |
| // has any stack objects. However, FI resolution actually has another job, |
| // not apparent from the title - it resolves callframesetup/destroy |
| // that were not simplified earlier. |
| // So, this is required for x86 functions that have push sequences even |
| // when there are no stack objects. |
| bool |
| X86FrameLowering::needsFrameIndexResolution(const MachineFunction &MF) const { |
| return MF.getFrameInfo().hasStackObjects() || |
| MF.getInfo<X86MachineFunctionInfo>()->getHasPushSequences(); |
| } |
| |
| /// hasFP - Return true if the specified function should have a dedicated frame |
| /// pointer register. This is true if the function has variable sized allocas |
| /// or if frame pointer elimination is disabled. |
| bool X86FrameLowering::hasFP(const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| return (MF.getTarget().Options.DisableFramePointerElim(MF) || |
| TRI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() || |
| MFI.isFrameAddressTaken() || MFI.hasOpaqueSPAdjustment() || |
| MF.getInfo<X86MachineFunctionInfo>()->getForceFramePointer() || |
| MF.getInfo<X86MachineFunctionInfo>()->hasPreallocatedCall() || |
| MF.callsUnwindInit() || MF.hasEHFunclets() || MF.callsEHReturn() || |
| MFI.hasStackMap() || MFI.hasPatchPoint() || |
| MFI.hasCopyImplyingStackAdjustment()); |
| } |
| |
| static unsigned getSUBriOpcode(bool IsLP64, int64_t Imm) { |
| if (IsLP64) { |
| if (isInt<8>(Imm)) |
| return X86::SUB64ri8; |
| return X86::SUB64ri32; |
| } else { |
| if (isInt<8>(Imm)) |
| return X86::SUB32ri8; |
| return X86::SUB32ri; |
| } |
| } |
| |
| static unsigned getADDriOpcode(bool IsLP64, int64_t Imm) { |
| if (IsLP64) { |
| if (isInt<8>(Imm)) |
| return X86::ADD64ri8; |
| return X86::ADD64ri32; |
| } else { |
| if (isInt<8>(Imm)) |
| return X86::ADD32ri8; |
| return X86::ADD32ri; |
| } |
| } |
| |
| static unsigned getSUBrrOpcode(bool IsLP64) { |
| return IsLP64 ? X86::SUB64rr : X86::SUB32rr; |
| } |
| |
| static unsigned getADDrrOpcode(bool IsLP64) { |
| return IsLP64 ? X86::ADD64rr : X86::ADD32rr; |
| } |
| |
| static unsigned getANDriOpcode(bool IsLP64, int64_t Imm) { |
| if (IsLP64) { |
| if (isInt<8>(Imm)) |
| return X86::AND64ri8; |
| return X86::AND64ri32; |
| } |
| if (isInt<8>(Imm)) |
| return X86::AND32ri8; |
| return X86::AND32ri; |
| } |
| |
| static unsigned getLEArOpcode(bool IsLP64) { |
| return IsLP64 ? X86::LEA64r : X86::LEA32r; |
| } |
| |
| static bool isEAXLiveIn(MachineBasicBlock &MBB) { |
| for (MachineBasicBlock::RegisterMaskPair RegMask : MBB.liveins()) { |
| unsigned Reg = RegMask.PhysReg; |
| |
| if (Reg == X86::RAX || Reg == X86::EAX || Reg == X86::AX || |
| Reg == X86::AH || Reg == X86::AL) |
| return true; |
| } |
| |
| return false; |
| } |
| |
| /// Check if the flags need to be preserved before the terminators. |
| /// This would be the case, if the eflags is live-in of the region |
| /// composed by the terminators or live-out of that region, without |
| /// being defined by a terminator. |
| static bool |
| flagsNeedToBePreservedBeforeTheTerminators(const MachineBasicBlock &MBB) { |
| for (const MachineInstr &MI : MBB.terminators()) { |
| bool BreakNext = false; |
| for (const MachineOperand &MO : MI.operands()) { |
| if (!MO.isReg()) |
| continue; |
| Register Reg = MO.getReg(); |
| if (Reg != X86::EFLAGS) |
| continue; |
| |
| // This terminator needs an eflags that is not defined |
| // by a previous another terminator: |
| // EFLAGS is live-in of the region composed by the terminators. |
| if (!MO.isDef()) |
| return true; |
| // This terminator defines the eflags, i.e., we don't need to preserve it. |
| // However, we still need to check this specific terminator does not |
| // read a live-in value. |
| BreakNext = true; |
| } |
| // We found a definition of the eflags, no need to preserve them. |
| if (BreakNext) |
| return false; |
| } |
| |
| // None of the terminators use or define the eflags. |
| // Check if they are live-out, that would imply we need to preserve them. |
| for (const MachineBasicBlock *Succ : MBB.successors()) |
| if (Succ->isLiveIn(X86::EFLAGS)) |
| return true; |
| |
| return false; |
| } |
| |
| /// emitSPUpdate - Emit a series of instructions to increment / decrement the |
| /// stack pointer by a constant value. |
| void X86FrameLowering::emitSPUpdate(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator &MBBI, |
| const DebugLoc &DL, |
| int64_t NumBytes, bool InEpilogue) const { |
| bool isSub = NumBytes < 0; |
| uint64_t Offset = isSub ? -NumBytes : NumBytes; |
| MachineInstr::MIFlag Flag = |
| isSub ? MachineInstr::FrameSetup : MachineInstr::FrameDestroy; |
| |
| uint64_t Chunk = (1LL << 31) - 1; |
| |
| MachineFunction &MF = *MBB.getParent(); |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| const X86TargetLowering &TLI = *STI.getTargetLowering(); |
| const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF); |
| |
| // It's ok to not take into account large chunks when probing, as the |
| // allocation is split in smaller chunks anyway. |
| if (EmitInlineStackProbe && !InEpilogue) { |
| |
| // This pseudo-instruction is going to be expanded, potentially using a |
| // loop, by inlineStackProbe(). |
| BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING)).addImm(Offset); |
| return; |
| } else if (Offset > Chunk) { |
| // Rather than emit a long series of instructions for large offsets, |
| // load the offset into a register and do one sub/add |
| unsigned Reg = 0; |
| unsigned Rax = (unsigned)(Is64Bit ? X86::RAX : X86::EAX); |
| |
| if (isSub && !isEAXLiveIn(MBB)) |
| Reg = Rax; |
| else |
| Reg = TRI->findDeadCallerSavedReg(MBB, MBBI); |
| |
| unsigned MovRIOpc = Is64Bit ? X86::MOV64ri : X86::MOV32ri; |
| unsigned AddSubRROpc = |
| isSub ? getSUBrrOpcode(Is64Bit) : getADDrrOpcode(Is64Bit); |
| if (Reg) { |
| BuildMI(MBB, MBBI, DL, TII.get(MovRIOpc), Reg) |
| .addImm(Offset) |
| .setMIFlag(Flag); |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AddSubRROpc), StackPtr) |
| .addReg(StackPtr) |
| .addReg(Reg); |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| return; |
| } else if (Offset > 8 * Chunk) { |
| // If we would need more than 8 add or sub instructions (a >16GB stack |
| // frame), it's worth spilling RAX to materialize this immediate. |
| // pushq %rax |
| // movabsq +-$Offset+-SlotSize, %rax |
| // addq %rsp, %rax |
| // xchg %rax, (%rsp) |
| // movq (%rsp), %rsp |
| assert(Is64Bit && "can't have 32-bit 16GB stack frame"); |
| BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r)) |
| .addReg(Rax, RegState::Kill) |
| .setMIFlag(Flag); |
| // Subtract is not commutative, so negate the offset and always use add. |
| // Subtract 8 less and add 8 more to account for the PUSH we just did. |
| if (isSub) |
| Offset = -(Offset - SlotSize); |
| else |
| Offset = Offset + SlotSize; |
| BuildMI(MBB, MBBI, DL, TII.get(MovRIOpc), Rax) |
| .addImm(Offset) |
| .setMIFlag(Flag); |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(X86::ADD64rr), Rax) |
| .addReg(Rax) |
| .addReg(StackPtr); |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| // Exchange the new SP in RAX with the top of the stack. |
| addRegOffset( |
| BuildMI(MBB, MBBI, DL, TII.get(X86::XCHG64rm), Rax).addReg(Rax), |
| StackPtr, false, 0); |
| // Load new SP from the top of the stack into RSP. |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), StackPtr), |
| StackPtr, false, 0); |
| return; |
| } |
| } |
| |
| while (Offset) { |
| uint64_t ThisVal = std::min(Offset, Chunk); |
| if (ThisVal == SlotSize) { |
| // Use push / pop for slot sized adjustments as a size optimization. We |
| // need to find a dead register when using pop. |
| unsigned Reg = isSub |
| ? (unsigned)(Is64Bit ? X86::RAX : X86::EAX) |
| : TRI->findDeadCallerSavedReg(MBB, MBBI); |
| if (Reg) { |
| unsigned Opc = isSub |
| ? (Is64Bit ? X86::PUSH64r : X86::PUSH32r) |
| : (Is64Bit ? X86::POP64r : X86::POP32r); |
| BuildMI(MBB, MBBI, DL, TII.get(Opc)) |
| .addReg(Reg, getDefRegState(!isSub) | getUndefRegState(isSub)) |
| .setMIFlag(Flag); |
| Offset -= ThisVal; |
| continue; |
| } |
| } |
| |
| BuildStackAdjustment(MBB, MBBI, DL, isSub ? -ThisVal : ThisVal, InEpilogue) |
| .setMIFlag(Flag); |
| |
| Offset -= ThisVal; |
| } |
| } |
| |
| MachineInstrBuilder X86FrameLowering::BuildStackAdjustment( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, int64_t Offset, bool InEpilogue) const { |
| assert(Offset != 0 && "zero offset stack adjustment requested"); |
| |
| // On Atom, using LEA to adjust SP is preferred, but using it in the epilogue |
| // is tricky. |
| bool UseLEA; |
| if (!InEpilogue) { |
| // Check if inserting the prologue at the beginning |
| // of MBB would require to use LEA operations. |
| // We need to use LEA operations if EFLAGS is live in, because |
| // it means an instruction will read it before it gets defined. |
| UseLEA = STI.useLeaForSP() || MBB.isLiveIn(X86::EFLAGS); |
| } else { |
| // If we can use LEA for SP but we shouldn't, check that none |
| // of the terminators uses the eflags. Otherwise we will insert |
| // a ADD that will redefine the eflags and break the condition. |
| // Alternatively, we could move the ADD, but this may not be possible |
| // and is an optimization anyway. |
| UseLEA = canUseLEAForSPInEpilogue(*MBB.getParent()); |
| if (UseLEA && !STI.useLeaForSP()) |
| UseLEA = flagsNeedToBePreservedBeforeTheTerminators(MBB); |
| // If that assert breaks, that means we do not do the right thing |
| // in canUseAsEpilogue. |
| assert((UseLEA || !flagsNeedToBePreservedBeforeTheTerminators(MBB)) && |
| "We shouldn't have allowed this insertion point"); |
| } |
| |
| MachineInstrBuilder MI; |
| if (UseLEA) { |
| MI = addRegOffset(BuildMI(MBB, MBBI, DL, |
| TII.get(getLEArOpcode(Uses64BitFramePtr)), |
| StackPtr), |
| StackPtr, false, Offset); |
| } else { |
| bool IsSub = Offset < 0; |
| uint64_t AbsOffset = IsSub ? -Offset : Offset; |
| const unsigned Opc = IsSub ? getSUBriOpcode(Uses64BitFramePtr, AbsOffset) |
| : getADDriOpcode(Uses64BitFramePtr, AbsOffset); |
| MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(AbsOffset); |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| } |
| return MI; |
| } |
| |
| int X86FrameLowering::mergeSPUpdates(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator &MBBI, |
| bool doMergeWithPrevious) const { |
| if ((doMergeWithPrevious && MBBI == MBB.begin()) || |
| (!doMergeWithPrevious && MBBI == MBB.end())) |
| return 0; |
| |
| MachineBasicBlock::iterator PI = doMergeWithPrevious ? std::prev(MBBI) : MBBI; |
| |
| PI = skipDebugInstructionsBackward(PI, MBB.begin()); |
| // It is assumed that ADD/SUB/LEA instruction is succeded by one CFI |
| // instruction, and that there are no DBG_VALUE or other instructions between |
| // ADD/SUB/LEA and its corresponding CFI instruction. |
| /* TODO: Add support for the case where there are multiple CFI instructions |
| below the ADD/SUB/LEA, e.g.: |
| ... |
| add |
| cfi_def_cfa_offset |
| cfi_offset |
| ... |
| */ |
| if (doMergeWithPrevious && PI != MBB.begin() && PI->isCFIInstruction()) |
| PI = std::prev(PI); |
| |
| unsigned Opc = PI->getOpcode(); |
| int Offset = 0; |
| |
| if ((Opc == X86::ADD64ri32 || Opc == X86::ADD64ri8 || |
| Opc == X86::ADD32ri || Opc == X86::ADD32ri8) && |
| PI->getOperand(0).getReg() == StackPtr){ |
| assert(PI->getOperand(1).getReg() == StackPtr); |
| Offset = PI->getOperand(2).getImm(); |
| } else if ((Opc == X86::LEA32r || Opc == X86::LEA64_32r) && |
| PI->getOperand(0).getReg() == StackPtr && |
| PI->getOperand(1).getReg() == StackPtr && |
| PI->getOperand(2).getImm() == 1 && |
| PI->getOperand(3).getReg() == X86::NoRegister && |
| PI->getOperand(5).getReg() == X86::NoRegister) { |
| // For LEAs we have: def = lea SP, FI, noreg, Offset, noreg. |
| Offset = PI->getOperand(4).getImm(); |
| } else if ((Opc == X86::SUB64ri32 || Opc == X86::SUB64ri8 || |
| Opc == X86::SUB32ri || Opc == X86::SUB32ri8) && |
| PI->getOperand(0).getReg() == StackPtr) { |
| assert(PI->getOperand(1).getReg() == StackPtr); |
| Offset = -PI->getOperand(2).getImm(); |
| } else |
| return 0; |
| |
| PI = MBB.erase(PI); |
| if (PI != MBB.end() && PI->isCFIInstruction()) { |
| auto CIs = MBB.getParent()->getFrameInstructions(); |
| MCCFIInstruction CI = CIs[PI->getOperand(0).getCFIIndex()]; |
| if (CI.getOperation() == MCCFIInstruction::OpDefCfaOffset || |
| CI.getOperation() == MCCFIInstruction::OpAdjustCfaOffset) |
| PI = MBB.erase(PI); |
| } |
| if (!doMergeWithPrevious) |
| MBBI = skipDebugInstructionsForward(PI, MBB.end()); |
| |
| return Offset; |
| } |
| |
| void X86FrameLowering::BuildCFI(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, |
| const MCCFIInstruction &CFIInst) const { |
| MachineFunction &MF = *MBB.getParent(); |
| unsigned CFIIndex = MF.addFrameInst(CFIInst); |
| BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex); |
| } |
| |
| /// Emits Dwarf Info specifying offsets of callee saved registers and |
| /// frame pointer. This is called only when basic block sections are enabled. |
| void X86FrameLowering::emitCalleeSavedFrameMoves( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { |
| MachineFunction &MF = *MBB.getParent(); |
| if (!hasFP(MF)) { |
| emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true); |
| return; |
| } |
| const MachineModuleInfo &MMI = MF.getMMI(); |
| const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); |
| const Register FramePtr = TRI->getFrameRegister(MF); |
| const Register MachineFramePtr = |
| STI.isTarget64BitILP32() ? Register(getX86SubSuperRegister(FramePtr, 64)) |
| : FramePtr; |
| unsigned DwarfReg = MRI->getDwarfRegNum(MachineFramePtr, true); |
| // Offset = space for return address + size of the frame pointer itself. |
| unsigned Offset = (Is64Bit ? 8 : 4) + (Uses64BitFramePtr ? 8 : 4); |
| BuildCFI(MBB, MBBI, DebugLoc{}, |
| MCCFIInstruction::createOffset(nullptr, DwarfReg, -Offset)); |
| emitCalleeSavedFrameMoves(MBB, MBBI, DebugLoc{}, true); |
| } |
| |
| void X86FrameLowering::emitCalleeSavedFrameMoves( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, bool IsPrologue) const { |
| MachineFunction &MF = *MBB.getParent(); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| MachineModuleInfo &MMI = MF.getMMI(); |
| const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); |
| |
| // Add callee saved registers to move list. |
| const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
| |
| // Calculate offsets. |
| for (const CalleeSavedInfo &I : CSI) { |
| int64_t Offset = MFI.getObjectOffset(I.getFrameIdx()); |
| unsigned Reg = I.getReg(); |
| unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); |
| |
| if (IsPrologue) { |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); |
| } else { |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::createRestore(nullptr, DwarfReg)); |
| } |
| } |
| } |
| |
| void X86FrameLowering::emitStackProbe( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog, |
| Optional<MachineFunction::DebugInstrOperandPair> InstrNum) const { |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| if (STI.isTargetWindowsCoreCLR()) { |
| if (InProlog) { |
| BuildMI(MBB, MBBI, DL, TII.get(X86::STACKALLOC_W_PROBING)) |
| .addImm(0 /* no explicit stack size */); |
| } else { |
| emitStackProbeInline(MF, MBB, MBBI, DL, false); |
| } |
| } else { |
| emitStackProbeCall(MF, MBB, MBBI, DL, InProlog, InstrNum); |
| } |
| } |
| |
| bool X86FrameLowering::stackProbeFunctionModifiesSP() const { |
| return STI.isOSWindows() && !STI.isTargetWin64(); |
| } |
| |
| void X86FrameLowering::inlineStackProbe(MachineFunction &MF, |
| MachineBasicBlock &PrologMBB) const { |
| auto Where = llvm::find_if(PrologMBB, [](MachineInstr &MI) { |
| return MI.getOpcode() == X86::STACKALLOC_W_PROBING; |
| }); |
| if (Where != PrologMBB.end()) { |
| DebugLoc DL = PrologMBB.findDebugLoc(Where); |
| emitStackProbeInline(MF, PrologMBB, Where, DL, true); |
| Where->eraseFromParent(); |
| } |
| } |
| |
| void X86FrameLowering::emitStackProbeInline(MachineFunction &MF, |
| MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, |
| bool InProlog) const { |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| if (STI.isTargetWindowsCoreCLR() && STI.is64Bit()) |
| emitStackProbeInlineWindowsCoreCLR64(MF, MBB, MBBI, DL, InProlog); |
| else |
| emitStackProbeInlineGeneric(MF, MBB, MBBI, DL, InProlog); |
| } |
| |
| void X86FrameLowering::emitStackProbeInlineGeneric( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { |
| MachineInstr &AllocWithProbe = *MBBI; |
| uint64_t Offset = AllocWithProbe.getOperand(0).getImm(); |
| |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| const X86TargetLowering &TLI = *STI.getTargetLowering(); |
| assert(!(STI.is64Bit() && STI.isTargetWindowsCoreCLR()) && |
| "different expansion expected for CoreCLR 64 bit"); |
| |
| const uint64_t StackProbeSize = TLI.getStackProbeSize(MF); |
| uint64_t ProbeChunk = StackProbeSize * 8; |
| |
| uint64_t MaxAlign = |
| TRI->hasStackRealignment(MF) ? calculateMaxStackAlign(MF) : 0; |
| |
| // Synthesize a loop or unroll it, depending on the number of iterations. |
| // BuildStackAlignAND ensures that only MaxAlign % StackProbeSize bits left |
| // between the unaligned rsp and current rsp. |
| if (Offset > ProbeChunk) { |
| emitStackProbeInlineGenericLoop(MF, MBB, MBBI, DL, Offset, |
| MaxAlign % StackProbeSize); |
| } else { |
| emitStackProbeInlineGenericBlock(MF, MBB, MBBI, DL, Offset, |
| MaxAlign % StackProbeSize); |
| } |
| } |
| |
| void X86FrameLowering::emitStackProbeInlineGenericBlock( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset, |
| uint64_t AlignOffset) const { |
| |
| const bool NeedsDwarfCFI = needsDwarfCFI(MF); |
| const bool HasFP = hasFP(MF); |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| const X86TargetLowering &TLI = *STI.getTargetLowering(); |
| const unsigned Opc = getSUBriOpcode(Uses64BitFramePtr, Offset); |
| const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi; |
| const uint64_t StackProbeSize = TLI.getStackProbeSize(MF); |
| |
| uint64_t CurrentOffset = 0; |
| |
| assert(AlignOffset < StackProbeSize); |
| |
| // If the offset is so small it fits within a page, there's nothing to do. |
| if (StackProbeSize < Offset + AlignOffset) { |
| |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(StackProbeSize - AlignOffset) |
| .setMIFlag(MachineInstr::FrameSetup); |
| if (!HasFP && NeedsDwarfCFI) { |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::createAdjustCfaOffset( |
| nullptr, StackProbeSize - AlignOffset)); |
| } |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| NumFrameExtraProbe++; |
| CurrentOffset = StackProbeSize - AlignOffset; |
| } |
| |
| // For the next N - 1 pages, just probe. I tried to take advantage of |
| // natural probes but it implies much more logic and there was very few |
| // interesting natural probes to interleave. |
| while (CurrentOffset + StackProbeSize < Offset) { |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(StackProbeSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| |
| if (!HasFP && NeedsDwarfCFI) { |
| BuildCFI( |
| MBB, MBBI, DL, |
| MCCFIInstruction::createAdjustCfaOffset(nullptr, StackProbeSize)); |
| } |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| NumFrameExtraProbe++; |
| CurrentOffset += StackProbeSize; |
| } |
| |
| // No need to probe the tail, it is smaller than a Page. |
| uint64_t ChunkSize = Offset - CurrentOffset; |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(ChunkSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| // No need to adjust Dwarf CFA offset here, the last position of the stack has |
| // been defined |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| } |
| |
| void X86FrameLowering::emitStackProbeInlineGenericLoop( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, uint64_t Offset, |
| uint64_t AlignOffset) const { |
| assert(Offset && "null offset"); |
| |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| const X86TargetLowering &TLI = *STI.getTargetLowering(); |
| const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi; |
| const uint64_t StackProbeSize = TLI.getStackProbeSize(MF); |
| |
| if (AlignOffset) { |
| if (AlignOffset < StackProbeSize) { |
| // Perform a first smaller allocation followed by a probe. |
| const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr, AlignOffset); |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(SUBOpc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(AlignOffset) |
| .setMIFlag(MachineInstr::FrameSetup); |
| MI->getOperand(3).setIsDead(); // The EFLAGS implicit def is dead. |
| |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| NumFrameExtraProbe++; |
| Offset -= AlignOffset; |
| } |
| } |
| |
| // Synthesize a loop |
| NumFrameLoopProbe++; |
| const BasicBlock *LLVM_BB = MBB.getBasicBlock(); |
| |
| MachineBasicBlock *testMBB = MF.CreateMachineBasicBlock(LLVM_BB); |
| MachineBasicBlock *tailMBB = MF.CreateMachineBasicBlock(LLVM_BB); |
| |
| MachineFunction::iterator MBBIter = ++MBB.getIterator(); |
| MF.insert(MBBIter, testMBB); |
| MF.insert(MBBIter, tailMBB); |
| |
| Register FinalStackProbed = Uses64BitFramePtr ? X86::R11 |
| : Is64Bit ? X86::R11D |
| : X86::EAX; |
| BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::COPY), FinalStackProbed) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // save loop bound |
| { |
| const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr, Offset); |
| BuildMI(MBB, MBBI, DL, TII.get(SUBOpc), FinalStackProbed) |
| .addReg(FinalStackProbed) |
| .addImm(Offset / StackProbeSize * StackProbeSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // allocate a page |
| { |
| const unsigned SUBOpc = getSUBriOpcode(Uses64BitFramePtr, StackProbeSize); |
| BuildMI(testMBB, DL, TII.get(SUBOpc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(StackProbeSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // touch the page |
| addRegOffset(BuildMI(testMBB, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // cmp with stack pointer bound |
| BuildMI(testMBB, DL, TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr)) |
| .addReg(StackPtr) |
| .addReg(FinalStackProbed) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // jump |
| BuildMI(testMBB, DL, TII.get(X86::JCC_1)) |
| .addMBB(testMBB) |
| .addImm(X86::COND_NE) |
| .setMIFlag(MachineInstr::FrameSetup); |
| testMBB->addSuccessor(testMBB); |
| testMBB->addSuccessor(tailMBB); |
| |
| // BB management |
| tailMBB->splice(tailMBB->end(), &MBB, MBBI, MBB.end()); |
| tailMBB->transferSuccessorsAndUpdatePHIs(&MBB); |
| MBB.addSuccessor(testMBB); |
| |
| // handle tail |
| unsigned TailOffset = Offset % StackProbeSize; |
| if (TailOffset) { |
| const unsigned Opc = getSUBriOpcode(Uses64BitFramePtr, TailOffset); |
| BuildMI(*tailMBB, tailMBB->begin(), DL, TII.get(Opc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(TailOffset) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Update Live In information |
| recomputeLiveIns(*testMBB); |
| recomputeLiveIns(*tailMBB); |
| } |
| |
| void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog) const { |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| assert(STI.is64Bit() && "different expansion needed for 32 bit"); |
| assert(STI.isTargetWindowsCoreCLR() && "custom expansion expects CoreCLR"); |
| const TargetInstrInfo &TII = *STI.getInstrInfo(); |
| const BasicBlock *LLVM_BB = MBB.getBasicBlock(); |
| |
| // RAX contains the number of bytes of desired stack adjustment. |
| // The handling here assumes this value has already been updated so as to |
| // maintain stack alignment. |
| // |
| // We need to exit with RSP modified by this amount and execute suitable |
| // page touches to notify the OS that we're growing the stack responsibly. |
| // All stack probing must be done without modifying RSP. |
| // |
| // MBB: |
| // SizeReg = RAX; |
| // ZeroReg = 0 |
| // CopyReg = RSP |
| // Flags, TestReg = CopyReg - SizeReg |
| // FinalReg = !Flags.Ovf ? TestReg : ZeroReg |
| // LimitReg = gs magic thread env access |
| // if FinalReg >= LimitReg goto ContinueMBB |
| // RoundBB: |
| // RoundReg = page address of FinalReg |
| // LoopMBB: |
| // LoopReg = PHI(LimitReg,ProbeReg) |
| // ProbeReg = LoopReg - PageSize |
| // [ProbeReg] = 0 |
| // if (ProbeReg > RoundReg) goto LoopMBB |
| // ContinueMBB: |
| // RSP = RSP - RAX |
| // [rest of original MBB] |
| |
| // Set up the new basic blocks |
| MachineBasicBlock *RoundMBB = MF.CreateMachineBasicBlock(LLVM_BB); |
| MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB); |
| MachineBasicBlock *ContinueMBB = MF.CreateMachineBasicBlock(LLVM_BB); |
| |
| MachineFunction::iterator MBBIter = std::next(MBB.getIterator()); |
| MF.insert(MBBIter, RoundMBB); |
| MF.insert(MBBIter, LoopMBB); |
| MF.insert(MBBIter, ContinueMBB); |
| |
| // Split MBB and move the tail portion down to ContinueMBB. |
| MachineBasicBlock::iterator BeforeMBBI = std::prev(MBBI); |
| ContinueMBB->splice(ContinueMBB->begin(), &MBB, MBBI, MBB.end()); |
| ContinueMBB->transferSuccessorsAndUpdatePHIs(&MBB); |
| |
| // Some useful constants |
| const int64_t ThreadEnvironmentStackLimit = 0x10; |
| const int64_t PageSize = 0x1000; |
| const int64_t PageMask = ~(PageSize - 1); |
| |
| // Registers we need. For the normal case we use virtual |
| // registers. For the prolog expansion we use RAX, RCX and RDX. |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| const TargetRegisterClass *RegClass = &X86::GR64RegClass; |
| const Register SizeReg = InProlog ? X86::RAX |
| : MRI.createVirtualRegister(RegClass), |
| ZeroReg = InProlog ? X86::RCX |
| : MRI.createVirtualRegister(RegClass), |
| CopyReg = InProlog ? X86::RDX |
| : MRI.createVirtualRegister(RegClass), |
| TestReg = InProlog ? X86::RDX |
| : MRI.createVirtualRegister(RegClass), |
| FinalReg = InProlog ? X86::RDX |
| : MRI.createVirtualRegister(RegClass), |
| RoundedReg = InProlog ? X86::RDX |
| : MRI.createVirtualRegister(RegClass), |
| LimitReg = InProlog ? X86::RCX |
| : MRI.createVirtualRegister(RegClass), |
| JoinReg = InProlog ? X86::RCX |
| : MRI.createVirtualRegister(RegClass), |
| ProbeReg = InProlog ? X86::RCX |
| : MRI.createVirtualRegister(RegClass); |
| |
| // SP-relative offsets where we can save RCX and RDX. |
| int64_t RCXShadowSlot = 0; |
| int64_t RDXShadowSlot = 0; |
| |
| // If inlining in the prolog, save RCX and RDX. |
| if (InProlog) { |
| // Compute the offsets. We need to account for things already |
| // pushed onto the stack at this point: return address, frame |
| // pointer (if used), and callee saves. |
| X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| const int64_t CalleeSaveSize = X86FI->getCalleeSavedFrameSize(); |
| const bool HasFP = hasFP(MF); |
| |
| // Check if we need to spill RCX and/or RDX. |
| // Here we assume that no earlier prologue instruction changes RCX and/or |
| // RDX, so checking the block live-ins is enough. |
| const bool IsRCXLiveIn = MBB.isLiveIn(X86::RCX); |
| const bool IsRDXLiveIn = MBB.isLiveIn(X86::RDX); |
| int64_t InitSlot = 8 + CalleeSaveSize + (HasFP ? 8 : 0); |
| // Assign the initial slot to both registers, then change RDX's slot if both |
| // need to be spilled. |
| if (IsRCXLiveIn) |
| RCXShadowSlot = InitSlot; |
| if (IsRDXLiveIn) |
| RDXShadowSlot = InitSlot; |
| if (IsRDXLiveIn && IsRCXLiveIn) |
| RDXShadowSlot += 8; |
| // Emit the saves if needed. |
| if (IsRCXLiveIn) |
| addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, |
| RCXShadowSlot) |
| .addReg(X86::RCX); |
| if (IsRDXLiveIn) |
| addRegOffset(BuildMI(&MBB, DL, TII.get(X86::MOV64mr)), X86::RSP, false, |
| RDXShadowSlot) |
| .addReg(X86::RDX); |
| } else { |
| // Not in the prolog. Copy RAX to a virtual reg. |
| BuildMI(&MBB, DL, TII.get(X86::MOV64rr), SizeReg).addReg(X86::RAX); |
| } |
| |
| // Add code to MBB to check for overflow and set the new target stack pointer |
| // to zero if so. |
| BuildMI(&MBB, DL, TII.get(X86::XOR64rr), ZeroReg) |
| .addReg(ZeroReg, RegState::Undef) |
| .addReg(ZeroReg, RegState::Undef); |
| BuildMI(&MBB, DL, TII.get(X86::MOV64rr), CopyReg).addReg(X86::RSP); |
| BuildMI(&MBB, DL, TII.get(X86::SUB64rr), TestReg) |
| .addReg(CopyReg) |
| .addReg(SizeReg); |
| BuildMI(&MBB, DL, TII.get(X86::CMOV64rr), FinalReg) |
| .addReg(TestReg) |
| .addReg(ZeroReg) |
| .addImm(X86::COND_B); |
| |
| // FinalReg now holds final stack pointer value, or zero if |
| // allocation would overflow. Compare against the current stack |
| // limit from the thread environment block. Note this limit is the |
| // lowest touched page on the stack, not the point at which the OS |
| // will cause an overflow exception, so this is just an optimization |
| // to avoid unnecessarily touching pages that are below the current |
| // SP but already committed to the stack by the OS. |
| BuildMI(&MBB, DL, TII.get(X86::MOV64rm), LimitReg) |
| .addReg(0) |
| .addImm(1) |
| .addReg(0) |
| .addImm(ThreadEnvironmentStackLimit) |
| .addReg(X86::GS); |
| BuildMI(&MBB, DL, TII.get(X86::CMP64rr)).addReg(FinalReg).addReg(LimitReg); |
| // Jump if the desired stack pointer is at or above the stack limit. |
| BuildMI(&MBB, DL, TII.get(X86::JCC_1)).addMBB(ContinueMBB).addImm(X86::COND_AE); |
| |
| // Add code to roundMBB to round the final stack pointer to a page boundary. |
| RoundMBB->addLiveIn(FinalReg); |
| BuildMI(RoundMBB, DL, TII.get(X86::AND64ri32), RoundedReg) |
| .addReg(FinalReg) |
| .addImm(PageMask); |
| BuildMI(RoundMBB, DL, TII.get(X86::JMP_1)).addMBB(LoopMBB); |
| |
| // LimitReg now holds the current stack limit, RoundedReg page-rounded |
| // final RSP value. Add code to loopMBB to decrement LimitReg page-by-page |
| // and probe until we reach RoundedReg. |
| if (!InProlog) { |
| BuildMI(LoopMBB, DL, TII.get(X86::PHI), JoinReg) |
| .addReg(LimitReg) |
| .addMBB(RoundMBB) |
| .addReg(ProbeReg) |
| .addMBB(LoopMBB); |
| } |
| |
| LoopMBB->addLiveIn(JoinReg); |
| addRegOffset(BuildMI(LoopMBB, DL, TII.get(X86::LEA64r), ProbeReg), JoinReg, |
| false, -PageSize); |
| |
| // Probe by storing a byte onto the stack. |
| BuildMI(LoopMBB, DL, TII.get(X86::MOV8mi)) |
| .addReg(ProbeReg) |
| .addImm(1) |
| .addReg(0) |
| .addImm(0) |
| .addReg(0) |
| .addImm(0); |
| |
| LoopMBB->addLiveIn(RoundedReg); |
| BuildMI(LoopMBB, DL, TII.get(X86::CMP64rr)) |
| .addReg(RoundedReg) |
| .addReg(ProbeReg); |
| BuildMI(LoopMBB, DL, TII.get(X86::JCC_1)).addMBB(LoopMBB).addImm(X86::COND_NE); |
| |
| MachineBasicBlock::iterator ContinueMBBI = ContinueMBB->getFirstNonPHI(); |
| |
| // If in prolog, restore RDX and RCX. |
| if (InProlog) { |
| if (RCXShadowSlot) // It means we spilled RCX in the prologue. |
| addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, |
| TII.get(X86::MOV64rm), X86::RCX), |
| X86::RSP, false, RCXShadowSlot); |
| if (RDXShadowSlot) // It means we spilled RDX in the prologue. |
| addRegOffset(BuildMI(*ContinueMBB, ContinueMBBI, DL, |
| TII.get(X86::MOV64rm), X86::RDX), |
| X86::RSP, false, RDXShadowSlot); |
| } |
| |
| // Now that the probing is done, add code to continueMBB to update |
| // the stack pointer for real. |
| ContinueMBB->addLiveIn(SizeReg); |
| BuildMI(*ContinueMBB, ContinueMBBI, DL, TII.get(X86::SUB64rr), X86::RSP) |
| .addReg(X86::RSP) |
| .addReg(SizeReg); |
| |
| // Add the control flow edges we need. |
| MBB.addSuccessor(ContinueMBB); |
| MBB.addSuccessor(RoundMBB); |
| RoundMBB->addSuccessor(LoopMBB); |
| LoopMBB->addSuccessor(ContinueMBB); |
| LoopMBB->addSuccessor(LoopMBB); |
| |
| // Mark all the instructions added to the prolog as frame setup. |
| if (InProlog) { |
| for (++BeforeMBBI; BeforeMBBI != MBB.end(); ++BeforeMBBI) { |
| BeforeMBBI->setFlag(MachineInstr::FrameSetup); |
| } |
| for (MachineInstr &MI : *RoundMBB) { |
| MI.setFlag(MachineInstr::FrameSetup); |
| } |
| for (MachineInstr &MI : *LoopMBB) { |
| MI.setFlag(MachineInstr::FrameSetup); |
| } |
| for (MachineBasicBlock::iterator CMBBI = ContinueMBB->begin(); |
| CMBBI != ContinueMBBI; ++CMBBI) { |
| CMBBI->setFlag(MachineInstr::FrameSetup); |
| } |
| } |
| } |
| |
| void X86FrameLowering::emitStackProbeCall( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, const DebugLoc &DL, bool InProlog, |
| Optional<MachineFunction::DebugInstrOperandPair> InstrNum) const { |
| bool IsLargeCodeModel = MF.getTarget().getCodeModel() == CodeModel::Large; |
| |
| // FIXME: Add indirect thunk support and remove this. |
| if (Is64Bit && IsLargeCodeModel && STI.useIndirectThunkCalls()) |
| report_fatal_error("Emitting stack probe calls on 64-bit with the large " |
| "code model and indirect thunks not yet implemented."); |
| |
| unsigned CallOp; |
| if (Is64Bit) |
| CallOp = IsLargeCodeModel ? X86::CALL64r : X86::CALL64pcrel32; |
| else |
| CallOp = X86::CALLpcrel32; |
| |
| StringRef Symbol = STI.getTargetLowering()->getStackProbeSymbolName(MF); |
| |
| MachineInstrBuilder CI; |
| MachineBasicBlock::iterator ExpansionMBBI = std::prev(MBBI); |
| |
| // All current stack probes take AX and SP as input, clobber flags, and |
| // preserve all registers. x86_64 probes leave RSP unmodified. |
| if (Is64Bit && MF.getTarget().getCodeModel() == CodeModel::Large) { |
| // For the large code model, we have to call through a register. Use R11, |
| // as it is scratch in all supported calling conventions. |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::R11) |
| .addExternalSymbol(MF.createExternalSymbolName(Symbol)); |
| CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)).addReg(X86::R11); |
| } else { |
| CI = BuildMI(MBB, MBBI, DL, TII.get(CallOp)) |
| .addExternalSymbol(MF.createExternalSymbolName(Symbol)); |
| } |
| |
| unsigned AX = Uses64BitFramePtr ? X86::RAX : X86::EAX; |
| unsigned SP = Uses64BitFramePtr ? X86::RSP : X86::ESP; |
| CI.addReg(AX, RegState::Implicit) |
| .addReg(SP, RegState::Implicit) |
| .addReg(AX, RegState::Define | RegState::Implicit) |
| .addReg(SP, RegState::Define | RegState::Implicit) |
| .addReg(X86::EFLAGS, RegState::Define | RegState::Implicit); |
| |
| MachineInstr *ModInst = CI; |
| if (STI.isTargetWin64() || !STI.isOSWindows()) { |
| // MSVC x32's _chkstk and cygwin/mingw's _alloca adjust %esp themselves. |
| // MSVC x64's __chkstk and cygwin/mingw's ___chkstk_ms do not adjust %rsp |
| // themselves. They also does not clobber %rax so we can reuse it when |
| // adjusting %rsp. |
| // All other platforms do not specify a particular ABI for the stack probe |
| // function, so we arbitrarily define it to not adjust %esp/%rsp itself. |
| ModInst = |
| BuildMI(MBB, MBBI, DL, TII.get(getSUBrrOpcode(Uses64BitFramePtr)), SP) |
| .addReg(SP) |
| .addReg(AX); |
| } |
| |
| // DebugInfo variable locations -- if there's an instruction number for the |
| // allocation (i.e., DYN_ALLOC_*), substitute it for the instruction that |
| // modifies SP. |
| if (InstrNum) { |
| if (STI.isTargetWin64() || !STI.isOSWindows()) { |
| // Label destination operand of the subtract. |
| MF.makeDebugValueSubstitution(*InstrNum, |
| {ModInst->getDebugInstrNum(), 0}); |
| } else { |
| // Label the call. The operand number is the penultimate operand, zero |
| // based. |
| unsigned SPDefOperand = ModInst->getNumOperands() - 2; |
| MF.makeDebugValueSubstitution( |
| *InstrNum, {ModInst->getDebugInstrNum(), SPDefOperand}); |
| } |
| } |
| |
| if (InProlog) { |
| // Apply the frame setup flag to all inserted instrs. |
| for (++ExpansionMBBI; ExpansionMBBI != MBBI; ++ExpansionMBBI) |
| ExpansionMBBI->setFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| static unsigned calculateSetFPREG(uint64_t SPAdjust) { |
| // Win64 ABI has a less restrictive limitation of 240; 128 works equally well |
| // and might require smaller successive adjustments. |
| const uint64_t Win64MaxSEHOffset = 128; |
| uint64_t SEHFrameOffset = std::min(SPAdjust, Win64MaxSEHOffset); |
| // Win64 ABI requires 16-byte alignment for the UWOP_SET_FPREG opcode. |
| return SEHFrameOffset & -16; |
| } |
| |
| // If we're forcing a stack realignment we can't rely on just the frame |
| // info, we need to know the ABI stack alignment as well in case we |
| // have a call out. Otherwise just make sure we have some alignment - we'll |
| // go with the minimum SlotSize. |
| uint64_t X86FrameLowering::calculateMaxStackAlign(const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| Align MaxAlign = MFI.getMaxAlign(); // Desired stack alignment. |
| Align StackAlign = getStackAlign(); |
| if (MF.getFunction().hasFnAttribute("stackrealign")) { |
| if (MFI.hasCalls()) |
| MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign; |
| else if (MaxAlign < SlotSize) |
| MaxAlign = Align(SlotSize); |
| } |
| return MaxAlign.value(); |
| } |
| |
| void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, unsigned Reg, |
| uint64_t MaxAlign) const { |
| uint64_t Val = -MaxAlign; |
| unsigned AndOp = getANDriOpcode(Uses64BitFramePtr, Val); |
| |
| MachineFunction &MF = *MBB.getParent(); |
| const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); |
| const X86TargetLowering &TLI = *STI.getTargetLowering(); |
| const uint64_t StackProbeSize = TLI.getStackProbeSize(MF); |
| const bool EmitInlineStackProbe = TLI.hasInlineStackProbe(MF); |
| |
| // We want to make sure that (in worst case) less than StackProbeSize bytes |
| // are not probed after the AND. This assumption is used in |
| // emitStackProbeInlineGeneric. |
| if (Reg == StackPtr && EmitInlineStackProbe && MaxAlign >= StackProbeSize) { |
| { |
| NumFrameLoopProbe++; |
| MachineBasicBlock *entryMBB = |
| MF.CreateMachineBasicBlock(MBB.getBasicBlock()); |
| MachineBasicBlock *headMBB = |
| MF.CreateMachineBasicBlock(MBB.getBasicBlock()); |
| MachineBasicBlock *bodyMBB = |
| MF.CreateMachineBasicBlock(MBB.getBasicBlock()); |
| MachineBasicBlock *footMBB = |
| MF.CreateMachineBasicBlock(MBB.getBasicBlock()); |
| |
| MachineFunction::iterator MBBIter = MBB.getIterator(); |
| MF.insert(MBBIter, entryMBB); |
| MF.insert(MBBIter, headMBB); |
| MF.insert(MBBIter, bodyMBB); |
| MF.insert(MBBIter, footMBB); |
| const unsigned MovMIOpc = Is64Bit ? X86::MOV64mi32 : X86::MOV32mi; |
| Register FinalStackProbed = Uses64BitFramePtr ? X86::R11 |
| : Is64Bit ? X86::R11D |
| : X86::EAX; |
| |
| // Setup entry block |
| { |
| |
| entryMBB->splice(entryMBB->end(), &MBB, MBB.begin(), MBBI); |
| BuildMI(entryMBB, DL, TII.get(TargetOpcode::COPY), FinalStackProbed) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| MachineInstr *MI = |
| BuildMI(entryMBB, DL, TII.get(AndOp), FinalStackProbed) |
| .addReg(FinalStackProbed) |
| .addImm(Val) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // The EFLAGS implicit def is dead. |
| MI->getOperand(3).setIsDead(); |
| |
| BuildMI(entryMBB, DL, |
| TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr)) |
| .addReg(FinalStackProbed) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| BuildMI(entryMBB, DL, TII.get(X86::JCC_1)) |
| .addMBB(&MBB) |
| .addImm(X86::COND_E) |
| .setMIFlag(MachineInstr::FrameSetup); |
| entryMBB->addSuccessor(headMBB); |
| entryMBB->addSuccessor(&MBB); |
| } |
| |
| // Loop entry block |
| |
| { |
| const unsigned SUBOpc = |
| getSUBriOpcode(Uses64BitFramePtr, StackProbeSize); |
| BuildMI(headMBB, DL, TII.get(SUBOpc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(StackProbeSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| BuildMI(headMBB, DL, |
| TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr)) |
| .addReg(FinalStackProbed) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // jump |
| BuildMI(headMBB, DL, TII.get(X86::JCC_1)) |
| .addMBB(footMBB) |
| .addImm(X86::COND_B) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| headMBB->addSuccessor(bodyMBB); |
| headMBB->addSuccessor(footMBB); |
| } |
| |
| // setup loop body |
| { |
| addRegOffset(BuildMI(bodyMBB, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| const unsigned SUBOpc = |
| getSUBriOpcode(Uses64BitFramePtr, StackProbeSize); |
| BuildMI(bodyMBB, DL, TII.get(SUBOpc), StackPtr) |
| .addReg(StackPtr) |
| .addImm(StackProbeSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // cmp with stack pointer bound |
| BuildMI(bodyMBB, DL, |
| TII.get(Uses64BitFramePtr ? X86::CMP64rr : X86::CMP32rr)) |
| .addReg(FinalStackProbed) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // jump |
| BuildMI(bodyMBB, DL, TII.get(X86::JCC_1)) |
| .addMBB(bodyMBB) |
| .addImm(X86::COND_B) |
| .setMIFlag(MachineInstr::FrameSetup); |
| bodyMBB->addSuccessor(bodyMBB); |
| bodyMBB->addSuccessor(footMBB); |
| } |
| |
| // setup loop footer |
| { |
| BuildMI(footMBB, DL, TII.get(TargetOpcode::COPY), StackPtr) |
| .addReg(FinalStackProbed) |
| .setMIFlag(MachineInstr::FrameSetup); |
| addRegOffset(BuildMI(footMBB, DL, TII.get(MovMIOpc)) |
| .setMIFlag(MachineInstr::FrameSetup), |
| StackPtr, false, 0) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| footMBB->addSuccessor(&MBB); |
| } |
| |
| recomputeLiveIns(*headMBB); |
| recomputeLiveIns(*bodyMBB); |
| recomputeLiveIns(*footMBB); |
| recomputeLiveIns(MBB); |
| } |
| } else { |
| MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg) |
| .addReg(Reg) |
| .addImm(Val) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // The EFLAGS implicit def is dead. |
| MI->getOperand(3).setIsDead(); |
| } |
| } |
| |
| bool X86FrameLowering::has128ByteRedZone(const MachineFunction& MF) const { |
| // x86-64 (non Win64) has a 128 byte red zone which is guaranteed not to be |
| // clobbered by any interrupt handler. |
| assert(&STI == &MF.getSubtarget<X86Subtarget>() && |
| "MF used frame lowering for wrong subtarget"); |
| const Function &Fn = MF.getFunction(); |
| const bool IsWin64CC = STI.isCallingConvWin64(Fn.getCallingConv()); |
| return Is64Bit && !IsWin64CC && !Fn.hasFnAttribute(Attribute::NoRedZone); |
| } |
| |
| bool X86FrameLowering::isWin64Prologue(const MachineFunction &MF) const { |
| return MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); |
| } |
| |
| bool X86FrameLowering::needsDwarfCFI(const MachineFunction &MF) const { |
| return !isWin64Prologue(MF) && MF.needsFrameMoves(); |
| } |
| |
| /// emitPrologue - Push callee-saved registers onto the stack, which |
| /// automatically adjust the stack pointer. Adjust the stack pointer to allocate |
| /// space for local variables. Also emit labels used by the exception handler to |
| /// generate the exception handling frames. |
| |
| /* |
| Here's a gist of what gets emitted: |
| |
| ; Establish frame pointer, if needed |
| [if needs FP] |
| push %rbp |
| .cfi_def_cfa_offset 16 |
| .cfi_offset %rbp, -16 |
| .seh_pushreg %rpb |
| mov %rsp, %rbp |
| .cfi_def_cfa_register %rbp |
| |
| ; Spill general-purpose registers |
| [for all callee-saved GPRs] |
| pushq %<reg> |
| [if not needs FP] |
| .cfi_def_cfa_offset (offset from RETADDR) |
| .seh_pushreg %<reg> |
| |
| ; If the required stack alignment > default stack alignment |
| ; rsp needs to be re-aligned. This creates a "re-alignment gap" |
| ; of unknown size in the stack frame. |
| [if stack needs re-alignment] |
| and $MASK, %rsp |
| |
| ; Allocate space for locals |
| [if target is Windows and allocated space > 4096 bytes] |
| ; Windows needs special care for allocations larger |
| ; than one page. |
| mov $NNN, %rax |
| call ___chkstk_ms/___chkstk |
| sub %rax, %rsp |
| [else] |
| sub $NNN, %rsp |
| |
| [if needs FP] |
| .seh_stackalloc (size of XMM spill slots) |
| .seh_setframe %rbp, SEHFrameOffset ; = size of all spill slots |
| [else] |
| .seh_stackalloc NNN |
| |
| ; Spill XMMs |
| ; Note, that while only Windows 64 ABI specifies XMMs as callee-preserved, |
| ; they may get spilled on any platform, if the current function |
| ; calls @llvm.eh.unwind.init |
| [if needs FP] |
| [for all callee-saved XMM registers] |
| movaps %<xmm reg>, -MMM(%rbp) |
| [for all callee-saved XMM registers] |
| .seh_savexmm %<xmm reg>, (-MMM + SEHFrameOffset) |
| ; i.e. the offset relative to (%rbp - SEHFrameOffset) |
| [else] |
| [for all callee-saved XMM registers] |
| movaps %<xmm reg>, KKK(%rsp) |
| [for all callee-saved XMM registers] |
| .seh_savexmm %<xmm reg>, KKK |
| |
| .seh_endprologue |
| |
| [if needs base pointer] |
| mov %rsp, %rbx |
| [if needs to restore base pointer] |
| mov %rsp, -MMM(%rbp) |
| |
| ; Emit CFI info |
| [if needs FP] |
| [for all callee-saved registers] |
| .cfi_offset %<reg>, (offset from %rbp) |
| [else] |
| .cfi_def_cfa_offset (offset from RETADDR) |
| [for all callee-saved registers] |
| .cfi_offset %<reg>, (offset from %rsp) |
| |
| Notes: |
| - .seh directives are emitted only for Windows 64 ABI |
| - .cv_fpo directives are emitted on win32 when emitting CodeView |
| - .cfi directives are emitted for all other ABIs |
| - for 32-bit code, substitute %e?? registers for %r?? |
| */ |
| |
| void X86FrameLowering::emitPrologue(MachineFunction &MF, |
| MachineBasicBlock &MBB) const { |
| assert(&STI == &MF.getSubtarget<X86Subtarget>() && |
| "MF used frame lowering for wrong subtarget"); |
| MachineBasicBlock::iterator MBBI = MBB.begin(); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const Function &Fn = MF.getFunction(); |
| MachineModuleInfo &MMI = MF.getMMI(); |
| X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| uint64_t MaxAlign = calculateMaxStackAlign(MF); // Desired stack alignment. |
| uint64_t StackSize = MFI.getStackSize(); // Number of bytes to allocate. |
| bool IsFunclet = MBB.isEHFuncletEntry(); |
| EHPersonality Personality = EHPersonality::Unknown; |
| if (Fn.hasPersonalityFn()) |
| Personality = classifyEHPersonality(Fn.getPersonalityFn()); |
| bool FnHasClrFunclet = |
| MF.hasEHFunclets() && Personality == EHPersonality::CoreCLR; |
| bool IsClrFunclet = IsFunclet && FnHasClrFunclet; |
| bool HasFP = hasFP(MF); |
| bool IsWin64Prologue = isWin64Prologue(MF); |
| bool NeedsWin64CFI = IsWin64Prologue && Fn.needsUnwindTableEntry(); |
| // FIXME: Emit FPO data for EH funclets. |
| bool NeedsWinFPO = |
| !IsFunclet && STI.isTargetWin32() && MMI.getModule()->getCodeViewFlag(); |
| bool NeedsWinCFI = NeedsWin64CFI || NeedsWinFPO; |
| bool NeedsDwarfCFI = needsDwarfCFI(MF); |
| Register FramePtr = TRI->getFrameRegister(MF); |
| const Register MachineFramePtr = |
| STI.isTarget64BitILP32() |
| ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr; |
| Register BasePtr = TRI->getBaseRegister(); |
| bool HasWinCFI = false; |
| |
| // Debug location must be unknown since the first debug location is used |
| // to determine the end of the prologue. |
| DebugLoc DL; |
| |
| // Space reserved for stack-based arguments when making a (ABI-guaranteed) |
| // tail call. |
| unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta(); |
| if (TailCallArgReserveSize && IsWin64Prologue) |
| report_fatal_error("Can't handle guaranteed tail call under win64 yet"); |
| |
| const bool EmitStackProbeCall = |
| STI.getTargetLowering()->hasStackProbeSymbol(MF); |
| unsigned StackProbeSize = STI.getTargetLowering()->getStackProbeSize(MF); |
| |
| if (HasFP && X86FI->hasSwiftAsyncContext()) { |
| switch (MF.getTarget().Options.SwiftAsyncFramePointer) { |
| case SwiftAsyncFramePointerMode::DeploymentBased: |
| if (STI.swiftAsyncContextIsDynamicallySet()) { |
| // The special symbol below is absolute and has a *value* suitable to be |
| // combined with the frame pointer directly. |
| BuildMI(MBB, MBBI, DL, TII.get(X86::OR64rm), MachineFramePtr) |
| .addUse(MachineFramePtr) |
| .addUse(X86::RIP) |
| .addImm(1) |
| .addUse(X86::NoRegister) |
| .addExternalSymbol("swift_async_extendedFramePointerFlags", |
| X86II::MO_GOTPCREL) |
| .addUse(X86::NoRegister); |
| break; |
| } |
| LLVM_FALLTHROUGH; |
| |
| case SwiftAsyncFramePointerMode::Always: |
| BuildMI(MBB, MBBI, DL, TII.get(X86::BTS64ri8), MachineFramePtr) |
| .addUse(MachineFramePtr) |
| .addImm(60) |
| .setMIFlag(MachineInstr::FrameSetup); |
| break; |
| |
| case SwiftAsyncFramePointerMode::Never: |
| break; |
| } |
| } |
| |
| // Re-align the stack on 64-bit if the x86-interrupt calling convention is |
| // used and an error code was pushed, since the x86-64 ABI requires a 16-byte |
| // stack alignment. |
| if (Fn.getCallingConv() == CallingConv::X86_INTR && Is64Bit && |
| Fn.arg_size() == 2) { |
| StackSize += 8; |
| MFI.setStackSize(StackSize); |
| emitSPUpdate(MBB, MBBI, DL, -8, /*InEpilogue=*/false); |
| } |
| |
| // If this is x86-64 and the Red Zone is not disabled, if we are a leaf |
| // function, and use up to 128 bytes of stack space, don't have a frame |
| // pointer, calls, or dynamic alloca then we do not need to adjust the |
| // stack pointer (we fit in the Red Zone). We also check that we don't |
| // push and pop from the stack. |
| if (has128ByteRedZone(MF) && !TRI->hasStackRealignment(MF) && |
| !MFI.hasVarSizedObjects() && // No dynamic alloca. |
| !MFI.adjustsStack() && // No calls. |
| !EmitStackProbeCall && // No stack probes. |
| !MFI.hasCopyImplyingStackAdjustment() && // Don't push and pop. |
| !MF.shouldSplitStack()) { // Regular stack |
| uint64_t MinSize = |
| X86FI->getCalleeSavedFrameSize() - X86FI->getTCReturnAddrDelta(); |
| if (HasFP) MinSize += SlotSize; |
| X86FI->setUsesRedZone(MinSize > 0 || StackSize > 0); |
| StackSize = std::max(MinSize, StackSize > 128 ? StackSize - 128 : 0); |
| MFI.setStackSize(StackSize); |
| } |
| |
| // Insert stack pointer adjustment for later moving of return addr. Only |
| // applies to tail call optimized functions where the callee argument stack |
| // size is bigger than the callers. |
| if (TailCallArgReserveSize != 0) { |
| BuildStackAdjustment(MBB, MBBI, DL, -(int)TailCallArgReserveSize, |
| /*InEpilogue=*/false) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Mapping for machine moves: |
| // |
| // DST: VirtualFP AND |
| // SRC: VirtualFP => DW_CFA_def_cfa_offset |
| // ELSE => DW_CFA_def_cfa |
| // |
| // SRC: VirtualFP AND |
| // DST: Register => DW_CFA_def_cfa_register |
| // |
| // ELSE |
| // OFFSET < 0 => DW_CFA_offset_extended_sf |
| // REG < 64 => DW_CFA_offset + Reg |
| // ELSE => DW_CFA_offset_extended |
| |
| uint64_t NumBytes = 0; |
| int stackGrowth = -SlotSize; |
| |
| // Find the funclet establisher parameter |
| Register Establisher = X86::NoRegister; |
| if (IsClrFunclet) |
| Establisher = Uses64BitFramePtr ? X86::RCX : X86::ECX; |
| else if (IsFunclet) |
| Establisher = Uses64BitFramePtr ? X86::RDX : X86::EDX; |
| |
| if (IsWin64Prologue && IsFunclet && !IsClrFunclet) { |
| // Immediately spill establisher into the home slot. |
| // The runtime cares about this. |
| // MOV64mr %rdx, 16(%rsp) |
| unsigned MOVmr = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(MOVmr)), StackPtr, true, 16) |
| .addReg(Establisher) |
| .setMIFlag(MachineInstr::FrameSetup); |
| MBB.addLiveIn(Establisher); |
| } |
| |
| if (HasFP) { |
| assert(MF.getRegInfo().isReserved(MachineFramePtr) && "FP reserved"); |
| |
| // Calculate required stack adjustment. |
| uint64_t FrameSize = StackSize - SlotSize; |
| // If required, include space for extra hidden slot for stashing base pointer. |
| if (X86FI->getRestoreBasePointer()) |
| FrameSize += SlotSize; |
| |
| NumBytes = FrameSize - |
| (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize); |
| |
| // Callee-saved registers are pushed on stack before the stack is realigned. |
| if (TRI->hasStackRealignment(MF) && !IsWin64Prologue) |
| NumBytes = alignTo(NumBytes, MaxAlign); |
| |
| // Save EBP/RBP into the appropriate stack slot. |
| BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::PUSH64r : X86::PUSH32r)) |
| .addReg(MachineFramePtr, RegState::Kill) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| if (NeedsDwarfCFI) { |
| // Mark the place where EBP/RBP was saved. |
| // Define the current CFA rule to use the provided offset. |
| assert(StackSize); |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfaOffset(nullptr, -2 * stackGrowth)); |
| |
| // Change the rule for the FramePtr to be an "offset" rule. |
| unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); |
| BuildCFI(MBB, MBBI, DL, MCCFIInstruction::createOffset( |
| nullptr, DwarfFramePtr, 2 * stackGrowth)); |
| } |
| |
| if (NeedsWinCFI) { |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) |
| .addImm(FramePtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| if (!IsFunclet) { |
| if (X86FI->hasSwiftAsyncContext()) { |
| const auto &Attrs = MF.getFunction().getAttributes(); |
| |
| // Before we update the live frame pointer we have to ensure there's a |
| // valid (or null) asynchronous context in its slot just before FP in |
| // the frame record, so store it now. |
| if (Attrs.hasAttrSomewhere(Attribute::SwiftAsync)) { |
| // We have an initial context in r14, store it just before the frame |
| // pointer. |
| MBB.addLiveIn(X86::R14); |
| BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r)) |
| .addReg(X86::R14) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } else { |
| // No initial context, store null so that there's no pointer that |
| // could be misused. |
| BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64i8)) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| if (NeedsWinCFI) { |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) |
| .addImm(X86::R14) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr) |
| .addUse(X86::RSP) |
| .addImm(1) |
| .addUse(X86::NoRegister) |
| .addImm(8) |
| .addUse(X86::NoRegister) |
| .setMIFlag(MachineInstr::FrameSetup); |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SUB64ri8), X86::RSP) |
| .addUse(X86::RSP) |
| .addImm(8) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| if (!IsWin64Prologue && !IsFunclet) { |
| // Update EBP with the new base value. |
| if (!X86FI->hasSwiftAsyncContext()) |
| BuildMI(MBB, MBBI, DL, |
| TII.get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), |
| FramePtr) |
| .addReg(StackPtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| if (NeedsDwarfCFI) { |
| // Mark effective beginning of when frame pointer becomes valid. |
| // Define the current CFA to use the EBP/RBP register. |
| unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); |
| BuildCFI( |
| MBB, MBBI, DL, |
| MCCFIInstruction::createDefCfaRegister(nullptr, DwarfFramePtr)); |
| } |
| |
| if (NeedsWinFPO) { |
| // .cv_fpo_setframe $FramePtr |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame)) |
| .addImm(FramePtr) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| } |
| } else { |
| assert(!IsFunclet && "funclets without FPs not yet implemented"); |
| NumBytes = StackSize - |
| (X86FI->getCalleeSavedFrameSize() + TailCallArgReserveSize); |
| } |
| |
| // Update the offset adjustment, which is mainly used by codeview to translate |
| // from ESP to VFRAME relative local variable offsets. |
| if (!IsFunclet) { |
| if (HasFP && TRI->hasStackRealignment(MF)) |
| MFI.setOffsetAdjustment(-NumBytes); |
| else |
| MFI.setOffsetAdjustment(-StackSize); |
| } |
| |
| // For EH funclets, only allocate enough space for outgoing calls. Save the |
| // NumBytes value that we would've used for the parent frame. |
| unsigned ParentFrameNumBytes = NumBytes; |
| if (IsFunclet) |
| NumBytes = getWinEHFuncletFrameSize(MF); |
| |
| // Skip the callee-saved push instructions. |
| bool PushedRegs = false; |
| int StackOffset = 2 * stackGrowth; |
| |
| while (MBBI != MBB.end() && |
| MBBI->getFlag(MachineInstr::FrameSetup) && |
| (MBBI->getOpcode() == X86::PUSH32r || |
| MBBI->getOpcode() == X86::PUSH64r)) { |
| PushedRegs = true; |
| Register Reg = MBBI->getOperand(0).getReg(); |
| ++MBBI; |
| |
| if (!HasFP && NeedsDwarfCFI) { |
| // Mark callee-saved push instruction. |
| // Define the current CFA rule to use the provided offset. |
| assert(StackSize); |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfaOffset(nullptr, -StackOffset)); |
| StackOffset += stackGrowth; |
| } |
| |
| if (NeedsWinCFI) { |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_PushReg)) |
| .addImm(Reg) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| // Realign stack after we pushed callee-saved registers (so that we'll be |
| // able to calculate their offsets from the frame pointer). |
| // Don't do this for Win64, it needs to realign the stack after the prologue. |
| if (!IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF)) { |
| assert(HasFP && "There should be a frame pointer if stack is realigned."); |
| BuildStackAlignAND(MBB, MBBI, DL, StackPtr, MaxAlign); |
| |
| if (NeedsWinCFI) { |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlign)) |
| .addImm(MaxAlign) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| // If there is an SUB32ri of ESP immediately before this instruction, merge |
| // the two. This can be the case when tail call elimination is enabled and |
| // the callee has more arguments then the caller. |
| NumBytes -= mergeSPUpdates(MBB, MBBI, true); |
| |
| // Adjust stack pointer: ESP -= numbytes. |
| |
| // Windows and cygwin/mingw require a prologue helper routine when allocating |
| // more than 4K bytes on the stack. Windows uses __chkstk and cygwin/mingw |
| // uses __alloca. __alloca and the 32-bit version of __chkstk will probe the |
| // stack and adjust the stack pointer in one go. The 64-bit version of |
| // __chkstk is only responsible for probing the stack. The 64-bit prologue is |
| // responsible for adjusting the stack pointer. Touching the stack at 4K |
| // increments is necessary to ensure that the guard pages used by the OS |
| // virtual memory manager are allocated in correct sequence. |
| uint64_t AlignedNumBytes = NumBytes; |
| if (IsWin64Prologue && !IsFunclet && TRI->hasStackRealignment(MF)) |
| AlignedNumBytes = alignTo(AlignedNumBytes, MaxAlign); |
| if (AlignedNumBytes >= StackProbeSize && EmitStackProbeCall) { |
| assert(!X86FI->getUsesRedZone() && |
| "The Red Zone is not accounted for in stack probes"); |
| |
| // Check whether EAX is livein for this block. |
| bool isEAXAlive = isEAXLiveIn(MBB); |
| |
| if (isEAXAlive) { |
| if (Is64Bit) { |
| // Save RAX |
| BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH64r)) |
| .addReg(X86::RAX, RegState::Kill) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } else { |
| // Save EAX |
| BuildMI(MBB, MBBI, DL, TII.get(X86::PUSH32r)) |
| .addReg(X86::EAX, RegState::Kill) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| if (Is64Bit) { |
| // Handle the 64-bit Windows ABI case where we need to call __chkstk. |
| // Function prologue is responsible for adjusting the stack pointer. |
| int64_t Alloc = isEAXAlive ? NumBytes - 8 : NumBytes; |
| if (isUInt<32>(Alloc)) { |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) |
| .addImm(Alloc) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } else if (isInt<32>(Alloc)) { |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri32), X86::RAX) |
| .addImm(Alloc) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } else { |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64ri), X86::RAX) |
| .addImm(Alloc) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } else { |
| // Allocate NumBytes-4 bytes on stack in case of isEAXAlive. |
| // We'll also use 4 already allocated bytes for EAX. |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) |
| .addImm(isEAXAlive ? NumBytes - 4 : NumBytes) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Call __chkstk, __chkstk_ms, or __alloca. |
| emitStackProbe(MF, MBB, MBBI, DL, true); |
| |
| if (isEAXAlive) { |
| // Restore RAX/EAX |
| MachineInstr *MI; |
| if (Is64Bit) |
| MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV64rm), X86::RAX), |
| StackPtr, false, NumBytes - 8); |
| else |
| MI = addRegOffset(BuildMI(MF, DL, TII.get(X86::MOV32rm), X86::EAX), |
| StackPtr, false, NumBytes - 4); |
| MI->setFlag(MachineInstr::FrameSetup); |
| MBB.insert(MBBI, MI); |
| } |
| } else if (NumBytes) { |
| emitSPUpdate(MBB, MBBI, DL, -(int64_t)NumBytes, /*InEpilogue=*/false); |
| } |
| |
| if (NeedsWinCFI && NumBytes) { |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_StackAlloc)) |
| .addImm(NumBytes) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| int SEHFrameOffset = 0; |
| unsigned SPOrEstablisher; |
| if (IsFunclet) { |
| if (IsClrFunclet) { |
| // The establisher parameter passed to a CLR funclet is actually a pointer |
| // to the (mostly empty) frame of its nearest enclosing funclet; we have |
| // to find the root function establisher frame by loading the PSPSym from |
| // the intermediate frame. |
| unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); |
| MachinePointerInfo NoInfo; |
| MBB.addLiveIn(Establisher); |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rm), Establisher), |
| Establisher, false, PSPSlotOffset) |
| .addMemOperand(MF.getMachineMemOperand( |
| NoInfo, MachineMemOperand::MOLoad, SlotSize, Align(SlotSize))); |
| ; |
| // Save the root establisher back into the current funclet's (mostly |
| // empty) frame, in case a sub-funclet or the GC needs it. |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, |
| false, PSPSlotOffset) |
| .addReg(Establisher) |
| .addMemOperand(MF.getMachineMemOperand( |
| NoInfo, |
| MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, |
| SlotSize, Align(SlotSize))); |
| } |
| SPOrEstablisher = Establisher; |
| } else { |
| SPOrEstablisher = StackPtr; |
| } |
| |
| if (IsWin64Prologue && HasFP) { |
| // Set RBP to a small fixed offset from RSP. In the funclet case, we base |
| // this calculation on the incoming establisher, which holds the value of |
| // RSP from the parent frame at the end of the prologue. |
| SEHFrameOffset = calculateSetFPREG(ParentFrameNumBytes); |
| if (SEHFrameOffset) |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), FramePtr), |
| SPOrEstablisher, false, SEHFrameOffset); |
| else |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64rr), FramePtr) |
| .addReg(SPOrEstablisher); |
| |
| // If this is not a funclet, emit the CFI describing our frame pointer. |
| if (NeedsWinCFI && !IsFunclet) { |
| assert(!NeedsWinFPO && "this setframe incompatible with FPO data"); |
| HasWinCFI = true; |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SetFrame)) |
| .addImm(FramePtr) |
| .addImm(SEHFrameOffset) |
| .setMIFlag(MachineInstr::FrameSetup); |
| if (isAsynchronousEHPersonality(Personality)) |
| MF.getWinEHFuncInfo()->SEHSetFrameOffset = SEHFrameOffset; |
| } |
| } else if (IsFunclet && STI.is32Bit()) { |
| // Reset EBP / ESI to something good for funclets. |
| MBBI = restoreWin32EHStackPointers(MBB, MBBI, DL); |
| // If we're a catch funclet, we can be returned to via catchret. Save ESP |
| // into the registration node so that the runtime will restore it for us. |
| if (!MBB.isCleanupFuncletEntry()) { |
| assert(Personality == EHPersonality::MSVC_CXX); |
| Register FrameReg; |
| int FI = MF.getWinEHFuncInfo()->EHRegNodeFrameIndex; |
| int64_t EHRegOffset = getFrameIndexReference(MF, FI, FrameReg).getFixed(); |
| // ESP is the first field, so no extra displacement is needed. |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32mr)), FrameReg, |
| false, EHRegOffset) |
| .addReg(X86::ESP); |
| } |
| } |
| |
| while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) { |
| const MachineInstr &FrameInstr = *MBBI; |
| ++MBBI; |
| |
| if (NeedsWinCFI) { |
| int FI; |
| if (unsigned Reg = TII.isStoreToStackSlot(FrameInstr, FI)) { |
| if (X86::FR64RegClass.contains(Reg)) { |
| int Offset; |
| Register IgnoredFrameReg; |
| if (IsWin64Prologue && IsFunclet) |
| Offset = getWin64EHFrameIndexRef(MF, FI, IgnoredFrameReg); |
| else |
| Offset = |
| getFrameIndexReference(MF, FI, IgnoredFrameReg).getFixed() + |
| SEHFrameOffset; |
| |
| HasWinCFI = true; |
| assert(!NeedsWinFPO && "SEH_SaveXMM incompatible with FPO data"); |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_SaveXMM)) |
| .addImm(Reg) |
| .addImm(Offset) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| } |
| } |
| |
| if (NeedsWinCFI && HasWinCFI) |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_EndPrologue)) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| if (FnHasClrFunclet && !IsFunclet) { |
| // Save the so-called Initial-SP (i.e. the value of the stack pointer |
| // immediately after the prolog) into the PSPSlot so that funclets |
| // and the GC can recover it. |
| unsigned PSPSlotOffset = getPSPSlotOffsetFromSP(MF); |
| auto PSPInfo = MachinePointerInfo::getFixedStack( |
| MF, MF.getWinEHFuncInfo()->PSPSymFrameIdx); |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(X86::MOV64mr)), StackPtr, false, |
| PSPSlotOffset) |
| .addReg(StackPtr) |
| .addMemOperand(MF.getMachineMemOperand( |
| PSPInfo, MachineMemOperand::MOStore | MachineMemOperand::MOVolatile, |
| SlotSize, Align(SlotSize))); |
| } |
| |
| // Realign stack after we spilled callee-saved registers (so that we'll be |
| // able to calculate their offsets from the frame pointer). |
| // Win64 requires aligning the stack after the prologue. |
| if (IsWin64Prologue && TRI->hasStackRealignment(MF)) { |
| assert(HasFP && "There should be a frame pointer if stack is realigned."); |
| BuildStackAlignAND(MBB, MBBI, DL, SPOrEstablisher, MaxAlign); |
| } |
| |
| // We already dealt with stack realignment and funclets above. |
| if (IsFunclet && STI.is32Bit()) |
| return; |
| |
| // If we need a base pointer, set it up here. It's whatever the value |
| // of the stack pointer is at this point. Any variable size objects |
| // will be allocated after this, so we can still use the base pointer |
| // to reference locals. |
| if (TRI->hasBasePointer(MF)) { |
| // Update the base pointer with the current stack pointer. |
| unsigned Opc = Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr; |
| BuildMI(MBB, MBBI, DL, TII.get(Opc), BasePtr) |
| .addReg(SPOrEstablisher) |
| .setMIFlag(MachineInstr::FrameSetup); |
| if (X86FI->getRestoreBasePointer()) { |
| // Stash value of base pointer. Saving RSP instead of EBP shortens |
| // dependence chain. Used by SjLj EH. |
| unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), |
| FramePtr, true, X86FI->getRestoreBasePointerOffset()) |
| .addReg(SPOrEstablisher) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| if (X86FI->getHasSEHFramePtrSave() && !IsFunclet) { |
| // Stash the value of the frame pointer relative to the base pointer for |
| // Win32 EH. This supports Win32 EH, which does the inverse of the above: |
| // it recovers the frame pointer from the base pointer rather than the |
| // other way around. |
| unsigned Opm = Uses64BitFramePtr ? X86::MOV64mr : X86::MOV32mr; |
| Register UsedReg; |
| int Offset = |
| getFrameIndexReference(MF, X86FI->getSEHFramePtrSaveIndex(), UsedReg) |
| .getFixed(); |
| assert(UsedReg == BasePtr); |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opm)), UsedReg, true, Offset) |
| .addReg(FramePtr) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| if (((!HasFP && NumBytes) || PushedRegs) && NeedsDwarfCFI) { |
| // Mark end of stack pointer adjustment. |
| if (!HasFP && NumBytes) { |
| // Define the current CFA rule to use the provided offset. |
| assert(StackSize); |
| BuildCFI( |
| MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfaOffset(nullptr, StackSize - stackGrowth)); |
| } |
| |
| // Emit DWARF info specifying the offsets of the callee-saved registers. |
| emitCalleeSavedFrameMoves(MBB, MBBI, DL, true); |
| } |
| |
| // X86 Interrupt handling function cannot assume anything about the direction |
| // flag (DF in EFLAGS register). Clear this flag by creating "cld" instruction |
| // in each prologue of interrupt handler function. |
| // |
| // FIXME: Create "cld" instruction only in these cases: |
| // 1. The interrupt handling function uses any of the "rep" instructions. |
| // 2. Interrupt handling function calls another function. |
| // |
| if (Fn.getCallingConv() == CallingConv::X86_INTR) |
| BuildMI(MBB, MBBI, DL, TII.get(X86::CLD)) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // At this point we know if the function has WinCFI or not. |
| MF.setHasWinCFI(HasWinCFI); |
| } |
| |
| bool X86FrameLowering::canUseLEAForSPInEpilogue( |
| const MachineFunction &MF) const { |
| // We can't use LEA instructions for adjusting the stack pointer if we don't |
| // have a frame pointer in the Win64 ABI. Only ADD instructions may be used |
| // to deallocate the stack. |
| // This means that we can use LEA for SP in two situations: |
| // 1. We *aren't* using the Win64 ABI which means we are free to use LEA. |
| // 2. We *have* a frame pointer which means we are permitted to use LEA. |
| return !MF.getTarget().getMCAsmInfo()->usesWindowsCFI() || hasFP(MF); |
| } |
| |
| static bool isFuncletReturnInstr(MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| case X86::CATCHRET: |
| case X86::CLEANUPRET: |
| return true; |
| default: |
| return false; |
| } |
| llvm_unreachable("impossible"); |
| } |
| |
| // CLR funclets use a special "Previous Stack Pointer Symbol" slot on the |
| // stack. It holds a pointer to the bottom of the root function frame. The |
| // establisher frame pointer passed to a nested funclet may point to the |
| // (mostly empty) frame of its parent funclet, but it will need to find |
| // the frame of the root function to access locals. To facilitate this, |
| // every funclet copies the pointer to the bottom of the root function |
| // frame into a PSPSym slot in its own (mostly empty) stack frame. Using the |
| // same offset for the PSPSym in the root function frame that's used in the |
| // funclets' frames allows each funclet to dynamically accept any ancestor |
| // frame as its establisher argument (the runtime doesn't guarantee the |
| // immediate parent for some reason lost to history), and also allows the GC, |
| // which uses the PSPSym for some bookkeeping, to find it in any funclet's |
| // frame with only a single offset reported for the entire method. |
| unsigned |
| X86FrameLowering::getPSPSlotOffsetFromSP(const MachineFunction &MF) const { |
| const WinEHFuncInfo &Info = *MF.getWinEHFuncInfo(); |
| Register SPReg; |
| int Offset = getFrameIndexReferencePreferSP(MF, Info.PSPSymFrameIdx, SPReg, |
| /*IgnoreSPUpdates*/ true) |
| .getFixed(); |
| assert(Offset >= 0 && SPReg == TRI->getStackRegister()); |
| return static_cast<unsigned>(Offset); |
| } |
| |
| unsigned |
| X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const { |
| const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| // This is the size of the pushed CSRs. |
| unsigned CSSize = X86FI->getCalleeSavedFrameSize(); |
| // This is the size of callee saved XMMs. |
| const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); |
| unsigned XMMSize = WinEHXMMSlotInfo.size() * |
| TRI->getSpillSize(X86::VR128RegClass); |
| // This is the amount of stack a funclet needs to allocate. |
| unsigned UsedSize; |
| EHPersonality Personality = |
| classifyEHPersonality(MF.getFunction().getPersonalityFn()); |
| if (Personality == EHPersonality::CoreCLR) { |
| // CLR funclets need to hold enough space to include the PSPSym, at the |
| // same offset from the stack pointer (immediately after the prolog) as it |
| // resides at in the main function. |
| UsedSize = getPSPSlotOffsetFromSP(MF) + SlotSize; |
| } else { |
| // Other funclets just need enough stack for outgoing call arguments. |
| UsedSize = MF.getFrameInfo().getMaxCallFrameSize(); |
| } |
| // RBP is not included in the callee saved register block. After pushing RBP, |
| // everything is 16 byte aligned. Everything we allocate before an outgoing |
| // call must also be 16 byte aligned. |
| unsigned FrameSizeMinusRBP = alignTo(CSSize + UsedSize, getStackAlign()); |
| // Subtract out the size of the callee saved registers. This is how much stack |
| // each funclet will allocate. |
| return FrameSizeMinusRBP + XMMSize - CSSize; |
| } |
| |
| static bool isTailCallOpcode(unsigned Opc) { |
| return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi || |
| Opc == X86::TCRETURNmi || |
| Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNdi64 || |
| Opc == X86::TCRETURNmi64; |
| } |
| |
| void X86FrameLowering::emitEpilogue(MachineFunction &MF, |
| MachineBasicBlock &MBB) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| MachineBasicBlock::iterator Terminator = MBB.getFirstTerminator(); |
| MachineBasicBlock::iterator MBBI = Terminator; |
| DebugLoc DL; |
| if (MBBI != MBB.end()) |
| DL = MBBI->getDebugLoc(); |
| // standard x86_64 and NaCl use 64-bit frame/stack pointers, x32 - 32-bit. |
| const bool Is64BitILP32 = STI.isTarget64BitILP32(); |
| Register FramePtr = TRI->getFrameRegister(MF); |
| Register MachineFramePtr = |
| Is64BitILP32 ? Register(getX86SubSuperRegister(FramePtr, 64)) : FramePtr; |
| |
| bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); |
| bool NeedsWin64CFI = |
| IsWin64Prologue && MF.getFunction().needsUnwindTableEntry(); |
| bool IsFunclet = MBBI == MBB.end() ? false : isFuncletReturnInstr(*MBBI); |
| |
| // Get the number of bytes to allocate from the FrameInfo. |
| uint64_t StackSize = MFI.getStackSize(); |
| uint64_t MaxAlign = calculateMaxStackAlign(MF); |
| unsigned CSSize = X86FI->getCalleeSavedFrameSize(); |
| unsigned TailCallArgReserveSize = -X86FI->getTCReturnAddrDelta(); |
| bool HasFP = hasFP(MF); |
| uint64_t NumBytes = 0; |
| |
| bool NeedsDwarfCFI = (!MF.getTarget().getTargetTriple().isOSDarwin() && |
| !MF.getTarget().getTargetTriple().isOSWindows()) && |
| MF.needsFrameMoves(); |
| |
| if (IsFunclet) { |
| assert(HasFP && "EH funclets without FP not yet implemented"); |
| NumBytes = getWinEHFuncletFrameSize(MF); |
| } else if (HasFP) { |
| // Calculate required stack adjustment. |
| uint64_t FrameSize = StackSize - SlotSize; |
| NumBytes = FrameSize - CSSize - TailCallArgReserveSize; |
| |
| // Callee-saved registers were pushed on stack before the stack was |
| // realigned. |
| if (TRI->hasStackRealignment(MF) && !IsWin64Prologue) |
| NumBytes = alignTo(FrameSize, MaxAlign); |
| } else { |
| NumBytes = StackSize - CSSize - TailCallArgReserveSize; |
| } |
| uint64_t SEHStackAllocAmt = NumBytes; |
| |
| // AfterPop is the position to insert .cfi_restore. |
| MachineBasicBlock::iterator AfterPop = MBBI; |
| if (HasFP) { |
| if (X86FI->hasSwiftAsyncContext()) { |
| // Discard the context. |
| int Offset = 16 + mergeSPUpdates(MBB, MBBI, true); |
| emitSPUpdate(MBB, MBBI, DL, Offset, /*InEpilogue*/true); |
| } |
| // Pop EBP. |
| BuildMI(MBB, MBBI, DL, TII.get(Is64Bit ? X86::POP64r : X86::POP32r), |
| MachineFramePtr) |
| .setMIFlag(MachineInstr::FrameDestroy); |
| |
| // We need to reset FP to its untagged state on return. Bit 60 is currently |
| // used to show the presence of an extended frame. |
| if (X86FI->hasSwiftAsyncContext()) { |
| BuildMI(MBB, MBBI, DL, TII.get(X86::BTR64ri8), |
| MachineFramePtr) |
| .addUse(MachineFramePtr) |
| .addImm(60) |
| .setMIFlag(MachineInstr::FrameDestroy); |
| } |
| |
| if (NeedsDwarfCFI) { |
| unsigned DwarfStackPtr = |
| TRI->getDwarfRegNum(Is64Bit ? X86::RSP : X86::ESP, true); |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfa(nullptr, DwarfStackPtr, SlotSize)); |
| if (!MBB.succ_empty() && !MBB.isReturnBlock()) { |
| unsigned DwarfFramePtr = TRI->getDwarfRegNum(MachineFramePtr, true); |
| BuildCFI(MBB, AfterPop, DL, |
| MCCFIInstruction::createRestore(nullptr, DwarfFramePtr)); |
| --MBBI; |
| --AfterPop; |
| } |
| --MBBI; |
| } |
| } |
| |
| MachineBasicBlock::iterator FirstCSPop = MBBI; |
| // Skip the callee-saved pop instructions. |
| while (MBBI != MBB.begin()) { |
| MachineBasicBlock::iterator PI = std::prev(MBBI); |
| unsigned Opc = PI->getOpcode(); |
| |
| if (Opc != X86::DBG_VALUE && !PI->isTerminator()) { |
| if ((Opc != X86::POP32r || !PI->getFlag(MachineInstr::FrameDestroy)) && |
| (Opc != X86::POP64r || !PI->getFlag(MachineInstr::FrameDestroy)) && |
| (Opc != X86::BTR64ri8 || !PI->getFlag(MachineInstr::FrameDestroy)) && |
| (Opc != X86::ADD64ri8 || !PI->getFlag(MachineInstr::FrameDestroy))) |
| break; |
| FirstCSPop = PI; |
| } |
| |
| --MBBI; |
| } |
| MBBI = FirstCSPop; |
| |
| if (IsFunclet && Terminator->getOpcode() == X86::CATCHRET) |
| emitCatchRetReturnValue(MBB, FirstCSPop, &*Terminator); |
| |
| if (MBBI != MBB.end()) |
| DL = MBBI->getDebugLoc(); |
| // If there is an ADD32ri or SUB32ri of ESP immediately before this |
| // instruction, merge the two instructions. |
| if (NumBytes || MFI.hasVarSizedObjects()) |
| NumBytes += mergeSPUpdates(MBB, MBBI, true); |
| |
| // If dynamic alloca is used, then reset esp to point to the last callee-saved |
| // slot before popping them off! Same applies for the case, when stack was |
| // realigned. Don't do this if this was a funclet epilogue, since the funclets |
| // will not do realignment or dynamic stack allocation. |
| if (((TRI->hasStackRealignment(MF)) || MFI.hasVarSizedObjects()) && |
| !IsFunclet) { |
| if (TRI->hasStackRealignment(MF)) |
| MBBI = FirstCSPop; |
| unsigned SEHFrameOffset = calculateSetFPREG(SEHStackAllocAmt); |
| uint64_t LEAAmount = |
| IsWin64Prologue ? SEHStackAllocAmt - SEHFrameOffset : -CSSize; |
| |
| if (X86FI->hasSwiftAsyncContext()) |
| LEAAmount -= 16; |
| |
| // There are only two legal forms of epilogue: |
| // - add SEHAllocationSize, %rsp |
| // - lea SEHAllocationSize(%FramePtr), %rsp |
| // |
| // 'mov %FramePtr, %rsp' will not be recognized as an epilogue sequence. |
| // However, we may use this sequence if we have a frame pointer because the |
| // effects of the prologue can safely be undone. |
| if (LEAAmount != 0) { |
| unsigned Opc = getLEArOpcode(Uses64BitFramePtr); |
| addRegOffset(BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr), |
| FramePtr, false, LEAAmount); |
| --MBBI; |
| } else { |
| unsigned Opc = (Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr); |
| BuildMI(MBB, MBBI, DL, TII.get(Opc), StackPtr) |
| .addReg(FramePtr); |
| --MBBI; |
| } |
| } else if (NumBytes) { |
| // Adjust stack pointer back: ESP += numbytes. |
| emitSPUpdate(MBB, MBBI, DL, NumBytes, /*InEpilogue=*/true); |
| if (!HasFP && NeedsDwarfCFI) { |
| // Define the current CFA rule to use the provided offset. |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfaOffset( |
| nullptr, CSSize + TailCallArgReserveSize + SlotSize)); |
| } |
| --MBBI; |
| } |
| |
| // Windows unwinder will not invoke function's exception handler if IP is |
| // either in prologue or in epilogue. This behavior causes a problem when a |
| // call immediately precedes an epilogue, because the return address points |
| // into the epilogue. To cope with that, we insert an epilogue marker here, |
| // then replace it with a 'nop' if it ends up immediately after a CALL in the |
| // final emitted code. |
| if (NeedsWin64CFI && MF.hasWinCFI()) |
| BuildMI(MBB, MBBI, DL, TII.get(X86::SEH_Epilogue)); |
| |
| if (!HasFP && NeedsDwarfCFI) { |
| MBBI = FirstCSPop; |
| int64_t Offset = -CSSize - SlotSize; |
| // Mark callee-saved pop instruction. |
| // Define the current CFA rule to use the provided offset. |
| while (MBBI != MBB.end()) { |
| MachineBasicBlock::iterator PI = MBBI; |
| unsigned Opc = PI->getOpcode(); |
| ++MBBI; |
| if (Opc == X86::POP32r || Opc == X86::POP64r) { |
| Offset += SlotSize; |
| BuildCFI(MBB, MBBI, DL, |
| MCCFIInstruction::cfiDefCfaOffset(nullptr, -Offset)); |
| } |
| } |
| } |
| |
| // Emit DWARF info specifying the restores of the callee-saved registers. |
| // For epilogue with return inside or being other block without successor, |
| // no need to generate .cfi_restore for callee-saved registers. |
| if (NeedsDwarfCFI && !MBB.succ_empty()) |
| emitCalleeSavedFrameMoves(MBB, AfterPop, DL, false); |
| |
| if (Terminator == MBB.end() || !isTailCallOpcode(Terminator->getOpcode())) { |
| // Add the return addr area delta back since we are not tail calling. |
| int Offset = -1 * X86FI->getTCReturnAddrDelta(); |
| assert(Offset >= 0 && "TCDelta should never be positive"); |
| if (Offset) { |
| // Check for possible merge with preceding ADD instruction. |
| Offset += mergeSPUpdates(MBB, Terminator, true); |
| emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true); |
| } |
| } |
| |
| // Emit tilerelease for AMX kernel. |
| if (X86FI->hasVirtualTileReg()) |
| BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE)); |
| } |
| |
| StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, |
| int FI, |
| Register &FrameReg) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| |
| bool IsFixed = MFI.isFixedObjectIndex(FI); |
| // We can't calculate offset from frame pointer if the stack is realigned, |
| // so enforce usage of stack/base pointer. The base pointer is used when we |
| // have dynamic allocas in addition to dynamic realignment. |
| if (TRI->hasBasePointer(MF)) |
| FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getBaseRegister(); |
| else if (TRI->hasStackRealignment(MF)) |
| FrameReg = IsFixed ? TRI->getFramePtr() : TRI->getStackRegister(); |
| else |
| FrameReg = TRI->getFrameRegister(MF); |
| |
| // Offset will hold the offset from the stack pointer at function entry to the |
| // object. |
| // We need to factor in additional offsets applied during the prologue to the |
| // frame, base, and stack pointer depending on which is used. |
| int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea(); |
| const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| unsigned CSSize = X86FI->getCalleeSavedFrameSize(); |
| uint64_t StackSize = MFI.getStackSize(); |
| bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI(); |
| int64_t FPDelta = 0; |
| |
| // In an x86 interrupt, remove the offset we added to account for the return |
| // address from any stack object allocated in the caller's frame. Interrupts |
| // do not have a standard return address. Fixed objects in the current frame, |
| // such as SSE register spills, should not get this treatment. |
| if (MF.getFunction().getCallingConv() == CallingConv::X86_INTR && |
| Offset >= 0) { |
| Offset += getOffsetOfLocalArea(); |
| } |
| |
| if (IsWin64Prologue) { |
| assert(!MFI.hasCalls() || (StackSize % 16) == 8); |
| |
| // Calculate required stack adjustment. |
| uint64_t FrameSize = StackSize - SlotSize; |
| // If required, include space for extra hidden slot for stashing base pointer. |
| if (X86FI->getRestoreBasePointer()) |
| FrameSize += SlotSize; |
| uint64_t NumBytes = FrameSize - CSSize; |
| |
| uint64_t SEHFrameOffset = calculateSetFPREG(NumBytes); |
| if (FI && FI == X86FI->getFAIndex()) |
| return StackOffset::getFixed(-SEHFrameOffset); |
| |
| // FPDelta is the offset from the "traditional" FP location of the old base |
| // pointer followed by return address and the location required by the |
| // restricted Win64 prologue. |
| // Add FPDelta to all offsets below that go through the frame pointer. |
| FPDelta = FrameSize - SEHFrameOffset; |
| assert((!MFI.hasCalls() || (FPDelta % 16) == 0) && |
| "FPDelta isn't aligned per the Win64 ABI!"); |
| } |
| |
| if (FrameReg == TRI->getFramePtr()) { |
| // Skip saved EBP/RBP |
| Offset += SlotSize; |
| |
| // Account for restricted Windows prologue. |
| Offset += FPDelta; |
| |
| // Skip the RETADDR move area |
| int TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); |
| if (TailCallReturnAddrDelta < 0) |
| Offset -= TailCallReturnAddrDelta; |
| |
| return StackOffset::getFixed(Offset); |
| } |
| |
| // FrameReg is either the stack pointer or a base pointer. But the base is |
| // located at the end of the statically known StackSize so the distinction |
| // doesn't really matter. |
| if (TRI->hasStackRealignment(MF) || TRI->hasBasePointer(MF)) |
| assert(isAligned(MFI.getObjectAlign(FI), -(Offset + StackSize))); |
| return StackOffset::getFixed(Offset + StackSize); |
| } |
| |
| int X86FrameLowering::getWin64EHFrameIndexRef(const MachineFunction &MF, int FI, |
| Register &FrameReg) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| const auto& WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); |
| const auto it = WinEHXMMSlotInfo.find(FI); |
| |
| if (it == WinEHXMMSlotInfo.end()) |
| return getFrameIndexReference(MF, FI, FrameReg).getFixed(); |
| |
| FrameReg = TRI->getStackRegister(); |
| return alignDown(MFI.getMaxCallFrameSize(), getStackAlign().value()) + |
| it->second; |
| } |
| |
| StackOffset |
| X86FrameLowering::getFrameIndexReferenceSP(const MachineFunction &MF, int FI, |
| Register &FrameReg, |
| int Adjustment) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| FrameReg = TRI->getStackRegister(); |
| return StackOffset::getFixed(MFI.getObjectOffset(FI) - |
| getOffsetOfLocalArea() + Adjustment); |
| } |
| |
| StackOffset |
| X86FrameLowering::getFrameIndexReferencePreferSP(const MachineFunction &MF, |
| int FI, Register &FrameReg, |
| bool IgnoreSPUpdates) const { |
| |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| // Does not include any dynamic realign. |
| const uint64_t StackSize = MFI.getStackSize(); |
| // LLVM arranges the stack as follows: |
| // ... |
| // ARG2 |
| // ARG1 |
| // RETADDR |
| // PUSH RBP <-- RBP points here |
| // PUSH CSRs |
| // ~~~~~~~ <-- possible stack realignment (non-win64) |
| // ... |
| // STACK OBJECTS |
| // ... <-- RSP after prologue points here |
| // ~~~~~~~ <-- possible stack realignment (win64) |
| // |
| // if (hasVarSizedObjects()): |
| // ... <-- "base pointer" (ESI/RBX) points here |
| // DYNAMIC ALLOCAS |
| // ... <-- RSP points here |
| // |
| // Case 1: In the simple case of no stack realignment and no dynamic |
| // allocas, both "fixed" stack objects (arguments and CSRs) are addressable |
| // with fixed offsets from RSP. |
| // |
| // Case 2: In the case of stack realignment with no dynamic allocas, fixed |
| // stack objects are addressed with RBP and regular stack objects with RSP. |
| // |
| // Case 3: In the case of dynamic allocas and stack realignment, RSP is used |
| // to address stack arguments for outgoing calls and nothing else. The "base |
| // pointer" points to local variables, and RBP points to fixed objects. |
| // |
| // In cases 2 and 3, we can only answer for non-fixed stack objects, and the |
| // answer we give is relative to the SP after the prologue, and not the |
| // SP in the middle of the function. |
| |
| if (MFI.isFixedObjectIndex(FI) && TRI->hasStackRealignment(MF) && |
| !STI.isTargetWin64()) |
| return getFrameIndexReference(MF, FI, FrameReg); |
| |
| // If !hasReservedCallFrame the function might have SP adjustement in the |
| // body. So, even though the offset is statically known, it depends on where |
| // we are in the function. |
| if (!IgnoreSPUpdates && !hasReservedCallFrame(MF)) |
| return getFrameIndexReference(MF, FI, FrameReg); |
| |
| // We don't handle tail calls, and shouldn't be seeing them either. |
| assert(MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta() >= 0 && |
| "we don't handle this case!"); |
| |
| // This is how the math works out: |
| // |
| // %rsp grows (i.e. gets lower) left to right. Each box below is |
| // one word (eight bytes). Obj0 is the stack slot we're trying to |
| // get to. |
| // |
| // ---------------------------------- |
| // | BP | Obj0 | Obj1 | ... | ObjN | |
| // ---------------------------------- |
| // ^ ^ ^ ^ |
| // A B C E |
| // |
| // A is the incoming stack pointer. |
| // (B - A) is the local area offset (-8 for x86-64) [1] |
| // (C - A) is the Offset returned by MFI.getObjectOffset for Obj0 [2] |
| // |
| // |(E - B)| is the StackSize (absolute value, positive). For a |
| // stack that grown down, this works out to be (B - E). [3] |
| // |
| // E is also the value of %rsp after stack has been set up, and we |
| // want (C - E) -- the value we can add to %rsp to get to Obj0. Now |
| // (C - E) == (C - A) - (B - A) + (B - E) |
| // { Using [1], [2] and [3] above } |
| // == getObjectOffset - LocalAreaOffset + StackSize |
| |
| return getFrameIndexReferenceSP(MF, FI, FrameReg, StackSize); |
| } |
| |
| bool X86FrameLowering::assignCalleeSavedSpillSlots( |
| MachineFunction &MF, const TargetRegisterInfo *TRI, |
| std::vector<CalleeSavedInfo> &CSI) const { |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>(); |
| |
| unsigned CalleeSavedFrameSize = 0; |
| unsigned XMMCalleeSavedFrameSize = 0; |
| auto &WinEHXMMSlotInfo = X86FI->getWinEHXMMSlotInfo(); |
| int SpillSlotOffset = getOffsetOfLocalArea() + X86FI->getTCReturnAddrDelta(); |
| |
| int64_t TailCallReturnAddrDelta = X86FI->getTCReturnAddrDelta(); |
| |
| if (TailCallReturnAddrDelta < 0) { |
| // create RETURNADDR area |
| // arg |
| // arg |
| // RETADDR |
| // { ... |
| // RETADDR area |
| // ... |
| // } |
| // [EBP] |
| MFI.CreateFixedObject(-TailCallReturnAddrDelta, |
| TailCallReturnAddrDelta - SlotSize, true); |
| } |
| |
| // Spill the BasePtr if it's used. |
| if (this->TRI->hasBasePointer(MF)) { |
| // Allocate a spill slot for EBP if we have a base pointer and EH funclets. |
| if (MF.hasEHFunclets()) { |
| int FI = MFI.CreateSpillStackObject(SlotSize, Align(SlotSize)); |
| X86FI->setHasSEHFramePtrSave(true); |
| X86FI->setSEHFramePtrSaveIndex(FI); |
| } |
| } |
| |
| if (hasFP(MF)) { |
| // emitPrologue always spills frame register the first thing. |
| SpillSlotOffset -= SlotSize; |
| MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); |
| |
| // The async context lives directly before the frame pointer, and we |
| // allocate a second slot to preserve stack alignment. |
| if (X86FI->hasSwiftAsyncContext()) { |
| SpillSlotOffset -= SlotSize; |
| MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); |
| SpillSlotOffset -= SlotSize; |
| } |
| |
| // Since emitPrologue and emitEpilogue will handle spilling and restoring of |
| // the frame register, we can delete it from CSI list and not have to worry |
| // about avoiding it later. |
| Register FPReg = TRI->getFrameRegister(MF); |
| for (unsigned i = 0; i < CSI.size(); ++i) { |
| if (TRI->regsOverlap(CSI[i].getReg(),FPReg)) { |
| CSI.erase(CSI.begin() + i); |
| break; |
| } |
| } |
| } |
| |
| // Assign slots for GPRs. It increases frame size. |
| for (unsigned i = CSI.size(); i != 0; --i) { |
| unsigned Reg = CSI[i - 1].getReg(); |
| |
| if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) |
| continue; |
| |
| SpillSlotOffset -= SlotSize; |
| CalleeSavedFrameSize += SlotSize; |
| |
| int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); |
| CSI[i - 1].setFrameIdx(SlotIndex); |
| } |
| |
| X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize); |
| MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize); |
| |
| // Assign slots for XMMs. |
| for (unsigned i = CSI.size(); i != 0; --i) { |
| unsigned Reg = CSI[i - 1].getReg(); |
| if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) |
| continue; |
| |
| // If this is k-register make sure we lookup via the largest legal type. |
| MVT VT = MVT::Other; |
| if (X86::VK16RegClass.contains(Reg)) |
| VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1; |
| |
| const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); |
| unsigned Size = TRI->getSpillSize(*RC); |
| Align Alignment = TRI->getSpillAlign(*RC); |
| // ensure alignment |
| assert(SpillSlotOffset < 0 && "SpillSlotOffset should always < 0 on X86"); |
| SpillSlotOffset = -alignTo(-SpillSlotOffset, Alignment); |
| |
| // spill into slot |
| SpillSlotOffset -= Size; |
| int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset); |
| CSI[i - 1].setFrameIdx(SlotIndex); |
| MFI.ensureMaxAlignment(Alignment); |
| |
| // Save the start offset and size of XMM in stack frame for funclets. |
| if (X86::VR128RegClass.contains(Reg)) { |
| WinEHXMMSlotInfo[SlotIndex] = XMMCalleeSavedFrameSize; |
| XMMCalleeSavedFrameSize += Size; |
| } |
| } |
| |
| return true; |
| } |
| |
| bool X86FrameLowering::spillCalleeSavedRegisters( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
| DebugLoc DL = MBB.findDebugLoc(MI); |
| |
| // Don't save CSRs in 32-bit EH funclets. The caller saves EBX, EBP, ESI, EDI |
| // for us, and there are no XMM CSRs on Win32. |
| if (MBB.isEHFuncletEntry() && STI.is32Bit() && STI.isOSWindows()) |
| return true; |
| |
| // Push GPRs. It increases frame size. |
| const MachineFunction &MF = *MBB.getParent(); |
| unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r; |
| for (unsigned i = CSI.size(); i != 0; --i) { |
| unsigned Reg = CSI[i - 1].getReg(); |
| |
| if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) |
| continue; |
| |
| const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| bool isLiveIn = MRI.isLiveIn(Reg); |
| if (!isLiveIn) |
| MBB.addLiveIn(Reg); |
| |
| // Decide whether we can add a kill flag to the use. |
| bool CanKill = !isLiveIn; |
| // Check if any subregister is live-in |
| if (CanKill) { |
| for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) { |
| if (MRI.isLiveIn(*AReg)) { |
| CanKill = false; |
| break; |
| } |
| } |
| } |
| |
| // Do not set a kill flag on values that are also marked as live-in. This |
| // happens with the @llvm-returnaddress intrinsic and with arguments |
| // passed in callee saved registers. |
| // Omitting the kill flags is conservatively correct even if the live-in |
| // is not used after all. |
| BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, getKillRegState(CanKill)) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Make XMM regs spilled. X86 does not have ability of push/pop XMM. |
| // It can be done by spilling XMMs to stack frame. |
| for (unsigned i = CSI.size(); i != 0; --i) { |
| unsigned Reg = CSI[i-1].getReg(); |
| if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) |
| continue; |
| |
| // If this is k-register make sure we lookup via the largest legal type. |
| MVT VT = MVT::Other; |
| if (X86::VK16RegClass.contains(Reg)) |
| VT = STI.hasBWI() ? MVT::v64i1 : MVT::v16i1; |
| |
| // Add the callee-saved register as live-in. It's killed at the spill. |
| MBB.addLiveIn(Reg); |
| const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); |
| |
| TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC, |
| TRI); |
| --MI; |
| MI->setFlag(MachineInstr::FrameSetup); |
| ++MI; |
| } |
| |
| return true; |
| } |
| |
| void X86FrameLowering::emitCatchRetReturnValue(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| MachineInstr *CatchRet) const { |
| // SEH shouldn't use catchret. |
| assert(!isAsynchronousEHPersonality(classifyEHPersonality( |
| MBB.getParent()->getFunction().getPersonalityFn())) && |
| "SEH should not use CATCHRET"); |
| const DebugLoc &DL = CatchRet->getDebugLoc(); |
| MachineBasicBlock *CatchRetTarget = CatchRet->getOperand(0).getMBB(); |
| |
| // Fill EAX/RAX with the address of the target block. |
| if (STI.is64Bit()) { |
| // LEA64r CatchRetTarget(%rip), %rax |
| BuildMI(MBB, MBBI, DL, TII.get(X86::LEA64r), X86::RAX) |
| .addReg(X86::RIP) |
| .addImm(0) |
| .addReg(0) |
| .addMBB(CatchRetTarget) |
| .addReg(0); |
| } else { |
| // MOV32ri $CatchRetTarget, %eax |
| BuildMI(MBB, MBBI, DL, TII.get(X86::MOV32ri), X86::EAX) |
| .addMBB(CatchRetTarget); |
| } |
| |
| // Record that we've taken the address of CatchRetTarget and no longer just |
| // reference it in a terminator. |
| CatchRetTarget->setHasAddressTaken(); |
| } |
| |
| bool X86FrameLowering::restoreCalleeSavedRegisters( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
| if (CSI.empty()) |
| return false; |
| |
|