| //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file defines a DAG pattern matching instruction selector for X86, |
| // converting from a legalized dag to a X86 dag. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "X86.h" |
| #include "X86MachineFunctionInfo.h" |
| #include "X86RegisterInfo.h" |
| #include "X86Subtarget.h" |
| #include "X86TargetMachine.h" |
| #include "llvm/ADT/Statistic.h" |
| #include "llvm/CodeGen/MachineModuleInfo.h" |
| #include "llvm/CodeGen/SelectionDAGISel.h" |
| #include "llvm/Config/llvm-config.h" |
| #include "llvm/IR/ConstantRange.h" |
| #include "llvm/IR/Function.h" |
| #include "llvm/IR/Instructions.h" |
| #include "llvm/IR/Intrinsics.h" |
| #include "llvm/IR/IntrinsicsX86.h" |
| #include "llvm/IR/Type.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/KnownBits.h" |
| #include "llvm/Support/MathExtras.h" |
| #include <cstdint> |
| |
| using namespace llvm; |
| |
| #define DEBUG_TYPE "x86-isel" |
| |
| STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); |
| |
| static cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(true), |
| cl::desc("Enable setting constant bits to reduce size of mask immediates"), |
| cl::Hidden); |
| |
| static cl::opt<bool> EnablePromoteAnyextLoad( |
| "x86-promote-anyext-load", cl::init(true), |
| cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden); |
| |
| extern cl::opt<bool> IndirectBranchTracking; |
| |
| //===----------------------------------------------------------------------===// |
| // Pattern Matcher Implementation |
| //===----------------------------------------------------------------------===// |
| |
| namespace { |
| /// This corresponds to X86AddressMode, but uses SDValue's instead of register |
| /// numbers for the leaves of the matched tree. |
| struct X86ISelAddressMode { |
| enum { |
| RegBase, |
| FrameIndexBase |
| } BaseType; |
| |
| // This is really a union, discriminated by BaseType! |
| SDValue Base_Reg; |
| int Base_FrameIndex; |
| |
| unsigned Scale; |
| SDValue IndexReg; |
| int32_t Disp; |
| SDValue Segment; |
| const GlobalValue *GV; |
| const Constant *CP; |
| const BlockAddress *BlockAddr; |
| const char *ES; |
| MCSymbol *MCSym; |
| int JT; |
| Align Alignment; // CP alignment. |
| unsigned char SymbolFlags; // X86II::MO_* |
| bool NegateIndex = false; |
| |
| X86ISelAddressMode() |
| : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), |
| Segment(), GV(nullptr), CP(nullptr), BlockAddr(nullptr), ES(nullptr), |
| MCSym(nullptr), JT(-1), SymbolFlags(X86II::MO_NO_FLAG) {} |
| |
| bool hasSymbolicDisplacement() const { |
| return GV != nullptr || CP != nullptr || ES != nullptr || |
| MCSym != nullptr || JT != -1 || BlockAddr != nullptr; |
| } |
| |
| bool hasBaseOrIndexReg() const { |
| return BaseType == FrameIndexBase || |
| IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr; |
| } |
| |
| /// Return true if this addressing mode is already RIP-relative. |
| bool isRIPRelative() const { |
| if (BaseType != RegBase) return false; |
| if (RegisterSDNode *RegNode = |
| dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) |
| return RegNode->getReg() == X86::RIP; |
| return false; |
| } |
| |
| void setBaseReg(SDValue Reg) { |
| BaseType = RegBase; |
| Base_Reg = Reg; |
| } |
| |
| #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
| void dump(SelectionDAG *DAG = nullptr) { |
| dbgs() << "X86ISelAddressMode " << this << '\n'; |
| dbgs() << "Base_Reg "; |
| if (Base_Reg.getNode()) |
| Base_Reg.getNode()->dump(DAG); |
| else |
| dbgs() << "nul\n"; |
| if (BaseType == FrameIndexBase) |
| dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n'; |
| dbgs() << " Scale " << Scale << '\n' |
| << "IndexReg "; |
| if (NegateIndex) |
| dbgs() << "negate "; |
| if (IndexReg.getNode()) |
| IndexReg.getNode()->dump(DAG); |
| else |
| dbgs() << "nul\n"; |
| dbgs() << " Disp " << Disp << '\n' |
| << "GV "; |
| if (GV) |
| GV->dump(); |
| else |
| dbgs() << "nul"; |
| dbgs() << " CP "; |
| if (CP) |
| CP->dump(); |
| else |
| dbgs() << "nul"; |
| dbgs() << '\n' |
| << "ES "; |
| if (ES) |
| dbgs() << ES; |
| else |
| dbgs() << "nul"; |
| dbgs() << " MCSym "; |
| if (MCSym) |
| dbgs() << MCSym; |
| else |
| dbgs() << "nul"; |
| dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n'; |
| } |
| #endif |
| }; |
| } |
| |
| namespace { |
| //===--------------------------------------------------------------------===// |
| /// ISel - X86-specific code to select X86 machine instructions for |
| /// SelectionDAG operations. |
| /// |
| class X86DAGToDAGISel final : public SelectionDAGISel { |
| /// Keep a pointer to the X86Subtarget around so that we can |
| /// make the right decision when generating code for different targets. |
| const X86Subtarget *Subtarget; |
| |
| /// If true, selector should try to optimize for minimum code size. |
| bool OptForMinSize; |
| |
| /// Disable direct TLS access through segment registers. |
| bool IndirectTlsSegRefs; |
| |
| public: |
| explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) |
| : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr), |
| OptForMinSize(false), IndirectTlsSegRefs(false) {} |
| |
| StringRef getPassName() const override { |
| return "X86 DAG->DAG Instruction Selection"; |
| } |
| |
| bool runOnMachineFunction(MachineFunction &MF) override { |
| // Reset the subtarget each time through. |
| Subtarget = &MF.getSubtarget<X86Subtarget>(); |
| IndirectTlsSegRefs = MF.getFunction().hasFnAttribute( |
| "indirect-tls-seg-refs"); |
| |
| // OptFor[Min]Size are used in pattern predicates that isel is matching. |
| OptForMinSize = MF.getFunction().hasMinSize(); |
| assert((!OptForMinSize || MF.getFunction().hasOptSize()) && |
| "OptForMinSize implies OptForSize"); |
| |
| SelectionDAGISel::runOnMachineFunction(MF); |
| return true; |
| } |
| |
| void emitFunctionEntryCode() override; |
| |
| bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; |
| |
| void PreprocessISelDAG() override; |
| void PostprocessISelDAG() override; |
| |
| // Include the pieces autogenerated from the target description. |
| #include "X86GenDAGISel.inc" |
| |
| private: |
| void Select(SDNode *N) override; |
| |
| bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM); |
| bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM, |
| bool AllowSegmentRegForX32 = false); |
| bool matchWrapper(SDValue N, X86ISelAddressMode &AM); |
| bool matchAddress(SDValue N, X86ISelAddressMode &AM); |
| bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM); |
| bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth); |
| bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, |
| unsigned Depth); |
| bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM, |
| unsigned Depth); |
| bool matchAddressBase(SDValue N, X86ISelAddressMode &AM); |
| bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, |
| SDValue &Scale, SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp, |
| SDValue ScaleOp, SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, SDValue &Segment); |
| bool selectMOV64Imm32(SDValue N, SDValue &Imm); |
| bool selectLEAAddr(SDValue N, SDValue &Base, |
| SDValue &Scale, SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| bool selectLEA64_32Addr(SDValue N, SDValue &Base, |
| SDValue &Scale, SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| bool selectTLSADDRAddr(SDValue N, SDValue &Base, |
| SDValue &Scale, SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| bool selectRelocImm(SDValue N, SDValue &Op); |
| |
| bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N, |
| SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| |
| // Convenience method where P is also root. |
| bool tryFoldLoad(SDNode *P, SDValue N, |
| SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, |
| SDValue &Segment) { |
| return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment); |
| } |
| |
| bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N, |
| SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, |
| SDValue &Segment); |
| |
| bool isProfitableToFormMaskedOp(SDNode *N) const; |
| |
| /// Implement addressing mode selection for inline asm expressions. |
| bool SelectInlineAsmMemoryOperand(const SDValue &Op, |
| unsigned ConstraintID, |
| std::vector<SDValue> &OutOps) override; |
| |
| void emitSpecialCodeForMain(); |
| |
| inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL, |
| MVT VT, SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, |
| SDValue &Segment) { |
| if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) |
| Base = CurDAG->getTargetFrameIndex( |
| AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout())); |
| else if (AM.Base_Reg.getNode()) |
| Base = AM.Base_Reg; |
| else |
| Base = CurDAG->getRegister(0, VT); |
| |
| Scale = getI8Imm(AM.Scale, DL); |
| |
| // Negate the index if needed. |
| if (AM.NegateIndex) { |
| unsigned NegOpc = VT == MVT::i64 ? X86::NEG64r : X86::NEG32r; |
| SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32, |
| AM.IndexReg), 0); |
| AM.IndexReg = Neg; |
| } |
| |
| if (AM.IndexReg.getNode()) |
| Index = AM.IndexReg; |
| else |
| Index = CurDAG->getRegister(0, VT); |
| |
| // These are 32-bit even in 64-bit mode since RIP-relative offset |
| // is 32-bit. |
| if (AM.GV) |
| Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(), |
| MVT::i32, AM.Disp, |
| AM.SymbolFlags); |
| else if (AM.CP) |
| Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment, |
| AM.Disp, AM.SymbolFlags); |
| else if (AM.ES) { |
| assert(!AM.Disp && "Non-zero displacement is ignored with ES."); |
| Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); |
| } else if (AM.MCSym) { |
| assert(!AM.Disp && "Non-zero displacement is ignored with MCSym."); |
| assert(AM.SymbolFlags == 0 && "oo"); |
| Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32); |
| } else if (AM.JT != -1) { |
| assert(!AM.Disp && "Non-zero displacement is ignored with JT."); |
| Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); |
| } else if (AM.BlockAddr) |
| Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp, |
| AM.SymbolFlags); |
| else |
| Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32); |
| |
| if (AM.Segment.getNode()) |
| Segment = AM.Segment; |
| else |
| Segment = CurDAG->getRegister(0, MVT::i16); |
| } |
| |
| // Utility function to determine whether we should avoid selecting |
| // immediate forms of instructions for better code size or not. |
| // At a high level, we'd like to avoid such instructions when |
| // we have similar constants used within the same basic block |
| // that can be kept in a register. |
| // |
| bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const { |
| uint32_t UseCount = 0; |
| |
| // Do not want to hoist if we're not optimizing for size. |
| // TODO: We'd like to remove this restriction. |
| // See the comment in X86InstrInfo.td for more info. |
| if (!CurDAG->shouldOptForSize()) |
| return false; |
| |
| // Walk all the users of the immediate. |
| for (const SDNode *User : N->uses()) { |
| if (UseCount >= 2) |
| break; |
| |
| // This user is already selected. Count it as a legitimate use and |
| // move on. |
| if (User->isMachineOpcode()) { |
| UseCount++; |
| continue; |
| } |
| |
| // We want to count stores of immediates as real uses. |
| if (User->getOpcode() == ISD::STORE && |
| User->getOperand(1).getNode() == N) { |
| UseCount++; |
| continue; |
| } |
| |
| // We don't currently match users that have > 2 operands (except |
| // for stores, which are handled above) |
| // Those instruction won't match in ISEL, for now, and would |
| // be counted incorrectly. |
| // This may change in the future as we add additional instruction |
| // types. |
| if (User->getNumOperands() != 2) |
| continue; |
| |
| // If this is a sign-extended 8-bit integer immediate used in an ALU |
| // instruction, there is probably an opcode encoding to save space. |
| auto *C = dyn_cast<ConstantSDNode>(N); |
| if (C && isInt<8>(C->getSExtValue())) |
| continue; |
| |
| // Immediates that are used for offsets as part of stack |
| // manipulation should be left alone. These are typically |
| // used to indicate SP offsets for argument passing and |
| // will get pulled into stores/pushes (implicitly). |
| if (User->getOpcode() == X86ISD::ADD || |
| User->getOpcode() == ISD::ADD || |
| User->getOpcode() == X86ISD::SUB || |
| User->getOpcode() == ISD::SUB) { |
| |
| // Find the other operand of the add/sub. |
| SDValue OtherOp = User->getOperand(0); |
| if (OtherOp.getNode() == N) |
| OtherOp = User->getOperand(1); |
| |
| // Don't count if the other operand is SP. |
| RegisterSDNode *RegNode; |
| if (OtherOp->getOpcode() == ISD::CopyFromReg && |
| (RegNode = dyn_cast_or_null<RegisterSDNode>( |
| OtherOp->getOperand(1).getNode()))) |
| if ((RegNode->getReg() == X86::ESP) || |
| (RegNode->getReg() == X86::RSP)) |
| continue; |
| } |
| |
| // ... otherwise, count this and move on. |
| UseCount++; |
| } |
| |
| // If we have more than 1 use, then recommend for hoisting. |
| return (UseCount > 1); |
| } |
| |
| /// Return a target constant with the specified value of type i8. |
| inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) { |
| return CurDAG->getTargetConstant(Imm, DL, MVT::i8); |
| } |
| |
| /// Return a target constant with the specified value, of type i32. |
| inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) { |
| return CurDAG->getTargetConstant(Imm, DL, MVT::i32); |
| } |
| |
| /// Return a target constant with the specified value, of type i64. |
| inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) { |
| return CurDAG->getTargetConstant(Imm, DL, MVT::i64); |
| } |
| |
| SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth, |
| const SDLoc &DL) { |
| assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width"); |
| uint64_t Index = N->getConstantOperandVal(1); |
| MVT VecVT = N->getOperand(0).getSimpleValueType(); |
| return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL); |
| } |
| |
| SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth, |
| const SDLoc &DL) { |
| assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width"); |
| uint64_t Index = N->getConstantOperandVal(2); |
| MVT VecVT = N->getSimpleValueType(0); |
| return getI8Imm((Index * VecVT.getScalarSizeInBits()) / VecWidth, DL); |
| } |
| |
| SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth, |
| const SDLoc &DL) { |
| assert(VecWidth == 128 && "Unexpected vector width"); |
| uint64_t Index = N->getConstantOperandVal(2); |
| MVT VecVT = N->getSimpleValueType(0); |
| uint64_t InsertIdx = (Index * VecVT.getScalarSizeInBits()) / VecWidth; |
| assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index"); |
| // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub) |
| // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub) |
| return getI8Imm(InsertIdx ? 0x02 : 0x30, DL); |
| } |
| |
| // Helper to detect unneeded and instructions on shift amounts. Called |
| // from PatFrags in tablegen. |
| bool isUnneededShiftMask(SDNode *N, unsigned Width) const { |
| assert(N->getOpcode() == ISD::AND && "Unexpected opcode"); |
| const APInt &Val = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue(); |
| |
| if (Val.countTrailingOnes() >= Width) |
| return true; |
| |
| APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero; |
| return Mask.countTrailingOnes() >= Width; |
| } |
| |
| /// Return an SDNode that returns the value of the global base register. |
| /// Output instructions required to initialize the global base register, |
| /// if necessary. |
| SDNode *getGlobalBaseReg(); |
| |
| /// Return a reference to the TargetMachine, casted to the target-specific |
| /// type. |
| const X86TargetMachine &getTargetMachine() const { |
| return static_cast<const X86TargetMachine &>(TM); |
| } |
| |
| /// Return a reference to the TargetInstrInfo, casted to the target-specific |
| /// type. |
| const X86InstrInfo *getInstrInfo() const { |
| return Subtarget->getInstrInfo(); |
| } |
| |
| /// Address-mode matching performs shift-of-and to and-of-shift |
| /// reassociation in order to expose more scaled addressing |
| /// opportunities. |
| bool ComplexPatternFuncMutatesDAG() const override { |
| return true; |
| } |
| |
| bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const; |
| |
| // Indicates we should prefer to use a non-temporal load for this load. |
| bool useNonTemporalLoad(LoadSDNode *N) const { |
| if (!N->isNonTemporal()) |
| return false; |
| |
| unsigned StoreSize = N->getMemoryVT().getStoreSize(); |
| |
| if (N->getAlignment() < StoreSize) |
| return false; |
| |
| switch (StoreSize) { |
| default: llvm_unreachable("Unsupported store size"); |
| case 4: |
| case 8: |
| return false; |
| case 16: |
| return Subtarget->hasSSE41(); |
| case 32: |
| return Subtarget->hasAVX2(); |
| case 64: |
| return Subtarget->hasAVX512(); |
| } |
| } |
| |
| bool foldLoadStoreIntoMemOperand(SDNode *Node); |
| MachineSDNode *matchBEXTRFromAndImm(SDNode *Node); |
| bool matchBitExtract(SDNode *Node); |
| bool shrinkAndImmediate(SDNode *N); |
| bool isMaskZeroExtended(SDNode *N) const; |
| bool tryShiftAmountMod(SDNode *N); |
| bool tryShrinkShlLogicImm(SDNode *N); |
| bool tryVPTERNLOG(SDNode *N); |
| bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB, |
| SDNode *ParentC, SDValue A, SDValue B, SDValue C, |
| uint8_t Imm); |
| bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask); |
| bool tryMatchBitSelect(SDNode *N); |
| |
| MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad, |
| const SDLoc &dl, MVT VT, SDNode *Node); |
| MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad, |
| const SDLoc &dl, MVT VT, SDNode *Node, |
| SDValue &InFlag); |
| |
| bool tryOptimizeRem8Extend(SDNode *N); |
| |
| bool onlyUsesZeroFlag(SDValue Flags) const; |
| bool hasNoSignFlagUses(SDValue Flags) const; |
| bool hasNoCarryFlagUses(SDValue Flags) const; |
| }; |
| } |
| |
| |
| // Returns true if this masked compare can be implemented legally with this |
| // type. |
| static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget) { |
| unsigned Opcode = N->getOpcode(); |
| if (Opcode == X86ISD::CMPM || Opcode == X86ISD::CMPMM || |
| Opcode == X86ISD::STRICT_CMPM || Opcode == ISD::SETCC || |
| Opcode == X86ISD::CMPMM_SAE || Opcode == X86ISD::VFPCLASS) { |
| // We can get 256-bit 8 element types here without VLX being enabled. When |
| // this happens we will use 512-bit operations and the mask will not be |
| // zero extended. |
| EVT OpVT = N->getOperand(0).getValueType(); |
| // The first operand of X86ISD::STRICT_CMPM is chain, so we need to get the |
| // second operand. |
| if (Opcode == X86ISD::STRICT_CMPM) |
| OpVT = N->getOperand(1).getValueType(); |
| if (OpVT.is256BitVector() || OpVT.is128BitVector()) |
| return Subtarget->hasVLX(); |
| |
| return true; |
| } |
| // Scalar opcodes use 128 bit registers, but aren't subject to the VLX check. |
| if (Opcode == X86ISD::VFPCLASSS || Opcode == X86ISD::FSETCCM || |
| Opcode == X86ISD::FSETCCM_SAE) |
| return true; |
| |
| return false; |
| } |
| |
| // Returns true if we can assume the writer of the mask has zero extended it |
| // for us. |
| bool X86DAGToDAGISel::isMaskZeroExtended(SDNode *N) const { |
| // If this is an AND, check if we have a compare on either side. As long as |
| // one side guarantees the mask is zero extended, the AND will preserve those |
| // zeros. |
| if (N->getOpcode() == ISD::AND) |
| return isLegalMaskCompare(N->getOperand(0).getNode(), Subtarget) || |
| isLegalMaskCompare(N->getOperand(1).getNode(), Subtarget); |
| |
| return isLegalMaskCompare(N, Subtarget); |
| } |
| |
| bool |
| X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { |
| if (OptLevel == CodeGenOpt::None) return false; |
| |
| if (!N.hasOneUse()) |
| return false; |
| |
| if (N.getOpcode() != ISD::LOAD) |
| return true; |
| |
| // Don't fold non-temporal loads if we have an instruction for them. |
| if (useNonTemporalLoad(cast<LoadSDNode>(N))) |
| return false; |
| |
| // If N is a load, do additional profitability checks. |
| if (U == Root) { |
| switch (U->getOpcode()) { |
| default: break; |
| case X86ISD::ADD: |
| case X86ISD::ADC: |
| case X86ISD::SUB: |
| case X86ISD::SBB: |
| case X86ISD::AND: |
| case X86ISD::XOR: |
| case X86ISD::OR: |
| case ISD::ADD: |
| case ISD::ADDCARRY: |
| case ISD::AND: |
| case ISD::OR: |
| case ISD::XOR: { |
| SDValue Op1 = U->getOperand(1); |
| |
| // If the other operand is a 8-bit immediate we should fold the immediate |
| // instead. This reduces code size. |
| // e.g. |
| // movl 4(%esp), %eax |
| // addl $4, %eax |
| // vs. |
| // movl $4, %eax |
| // addl 4(%esp), %eax |
| // The former is 2 bytes shorter. In case where the increment is 1, then |
| // the saving can be 4 bytes (by using incl %eax). |
| if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) { |
| if (Imm->getAPIntValue().isSignedIntN(8)) |
| return false; |
| |
| // If this is a 64-bit AND with an immediate that fits in 32-bits, |
| // prefer using the smaller and over folding the load. This is needed to |
| // make sure immediates created by shrinkAndImmediate are always folded. |
| // Ideally we would narrow the load during DAG combine and get the |
| // best of both worlds. |
| if (U->getOpcode() == ISD::AND && |
| Imm->getAPIntValue().getBitWidth() == 64 && |
| Imm->getAPIntValue().isIntN(32)) |
| return false; |
| |
| // If this really a zext_inreg that can be represented with a movzx |
| // instruction, prefer that. |
| // TODO: We could shrink the load and fold if it is non-volatile. |
| if (U->getOpcode() == ISD::AND && |
| (Imm->getAPIntValue() == UINT8_MAX || |
| Imm->getAPIntValue() == UINT16_MAX || |
| Imm->getAPIntValue() == UINT32_MAX)) |
| return false; |
| |
| // ADD/SUB with can negate the immediate and use the opposite operation |
| // to fit 128 into a sign extended 8 bit immediate. |
| if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB) && |
| (-Imm->getAPIntValue()).isSignedIntN(8)) |
| return false; |
| |
| if ((U->getOpcode() == X86ISD::ADD || U->getOpcode() == X86ISD::SUB) && |
| (-Imm->getAPIntValue()).isSignedIntN(8) && |
| hasNoCarryFlagUses(SDValue(U, 1))) |
| return false; |
| } |
| |
| // If the other operand is a TLS address, we should fold it instead. |
| // This produces |
| // movl %gs:0, %eax |
| // leal i@NTPOFF(%eax), %eax |
| // instead of |
| // movl $i@NTPOFF, %eax |
| // addl %gs:0, %eax |
| // if the block also has an access to a second TLS address this will save |
| // a load. |
| // FIXME: This is probably also true for non-TLS addresses. |
| if (Op1.getOpcode() == X86ISD::Wrapper) { |
| SDValue Val = Op1.getOperand(0); |
| if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) |
| return false; |
| } |
| |
| // Don't fold load if this matches the BTS/BTR/BTC patterns. |
| // BTS: (or X, (shl 1, n)) |
| // BTR: (and X, (rotl -2, n)) |
| // BTC: (xor X, (shl 1, n)) |
| if (U->getOpcode() == ISD::OR || U->getOpcode() == ISD::XOR) { |
| if (U->getOperand(0).getOpcode() == ISD::SHL && |
| isOneConstant(U->getOperand(0).getOperand(0))) |
| return false; |
| |
| if (U->getOperand(1).getOpcode() == ISD::SHL && |
| isOneConstant(U->getOperand(1).getOperand(0))) |
| return false; |
| } |
| if (U->getOpcode() == ISD::AND) { |
| SDValue U0 = U->getOperand(0); |
| SDValue U1 = U->getOperand(1); |
| if (U0.getOpcode() == ISD::ROTL) { |
| auto *C = dyn_cast<ConstantSDNode>(U0.getOperand(0)); |
| if (C && C->getSExtValue() == -2) |
| return false; |
| } |
| |
| if (U1.getOpcode() == ISD::ROTL) { |
| auto *C = dyn_cast<ConstantSDNode>(U1.getOperand(0)); |
| if (C && C->getSExtValue() == -2) |
| return false; |
| } |
| } |
| |
| break; |
| } |
| case ISD::SHL: |
| case ISD::SRA: |
| case ISD::SRL: |
| // Don't fold a load into a shift by immediate. The BMI2 instructions |
| // support folding a load, but not an immediate. The legacy instructions |
| // support folding an immediate, but can't fold a load. Folding an |
| // immediate is preferable to folding a load. |
| if (isa<ConstantSDNode>(U->getOperand(1))) |
| return false; |
| |
| break; |
| } |
| } |
| |
| // Prevent folding a load if this can implemented with an insert_subreg or |
| // a move that implicitly zeroes. |
| if (Root->getOpcode() == ISD::INSERT_SUBVECTOR && |
| isNullConstant(Root->getOperand(2)) && |
| (Root->getOperand(0).isUndef() || |
| ISD::isBuildVectorAllZeros(Root->getOperand(0).getNode()))) |
| return false; |
| |
| return true; |
| } |
| |
| // Indicates it is profitable to form an AVX512 masked operation. Returning |
| // false will favor a masked register-register masked move or vblendm and the |
| // operation will be selected separately. |
| bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const { |
| assert( |
| (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::SELECTS) && |
| "Unexpected opcode!"); |
| |
| // If the operation has additional users, the operation will be duplicated. |
| // Check the use count to prevent that. |
| // FIXME: Are there cheap opcodes we might want to duplicate? |
| return N->getOperand(1).hasOneUse(); |
| } |
| |
| /// Replace the original chain operand of the call with |
| /// load's chain operand and move load below the call's chain operand. |
| static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, |
| SDValue Call, SDValue OrigChain) { |
| SmallVector<SDValue, 8> Ops; |
| SDValue Chain = OrigChain.getOperand(0); |
| if (Chain.getNode() == Load.getNode()) |
| Ops.push_back(Load.getOperand(0)); |
| else { |
| assert(Chain.getOpcode() == ISD::TokenFactor && |
| "Unexpected chain operand"); |
| for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) |
| if (Chain.getOperand(i).getNode() == Load.getNode()) |
| Ops.push_back(Load.getOperand(0)); |
| else |
| Ops.push_back(Chain.getOperand(i)); |
| SDValue NewChain = |
| CurDAG->getNode(ISD::TokenFactor, SDLoc(Load), MVT::Other, Ops); |
| Ops.clear(); |
| Ops.push_back(NewChain); |
| } |
| Ops.append(OrigChain->op_begin() + 1, OrigChain->op_end()); |
| CurDAG->UpdateNodeOperands(OrigChain.getNode(), Ops); |
| CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), |
| Load.getOperand(1), Load.getOperand(2)); |
| |
| Ops.clear(); |
| Ops.push_back(SDValue(Load.getNode(), 1)); |
| Ops.append(Call->op_begin() + 1, Call->op_end()); |
| CurDAG->UpdateNodeOperands(Call.getNode(), Ops); |
| } |
| |
| /// Return true if call address is a load and it can be |
| /// moved below CALLSEQ_START and the chains leading up to the call. |
| /// Return the CALLSEQ_START by reference as a second output. |
| /// In the case of a tail call, there isn't a callseq node between the call |
| /// chain and the load. |
| static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { |
| // The transformation is somewhat dangerous if the call's chain was glued to |
| // the call. After MoveBelowOrigChain the load is moved between the call and |
| // the chain, this can create a cycle if the load is not folded. So it is |
| // *really* important that we are sure the load will be folded. |
| if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) |
| return false; |
| LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); |
| if (!LD || |
| !LD->isSimple() || |
| LD->getAddressingMode() != ISD::UNINDEXED || |
| LD->getExtensionType() != ISD::NON_EXTLOAD) |
| return false; |
| |
| // Now let's find the callseq_start. |
| while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { |
| if (!Chain.hasOneUse()) |
| return false; |
| Chain = Chain.getOperand(0); |
| } |
| |
| if (!Chain.getNumOperands()) |
| return false; |
| // Since we are not checking for AA here, conservatively abort if the chain |
| // writes to memory. It's not safe to move the callee (a load) across a store. |
| if (isa<MemSDNode>(Chain.getNode()) && |
| cast<MemSDNode>(Chain.getNode())->writeMem()) |
| return false; |
| if (Chain.getOperand(0).getNode() == Callee.getNode()) |
| return true; |
| if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && |
| Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && |
| Callee.getValue(1).hasOneUse()) |
| return true; |
| return false; |
| } |
| |
| static bool isEndbrImm64(uint64_t Imm) { |
| // There may be some other prefix bytes between 0xF3 and 0x0F1EFA. |
| // i.g: 0xF3660F1EFA, 0xF3670F1EFA |
| if ((Imm & 0x00FFFFFF) != 0x0F1EFA) |
| return false; |
| |
| uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64, |
| 0x65, 0x66, 0x67, 0xf0, 0xf2}; |
| int i = 24; // 24bit 0x0F1EFA has matched |
| while (i < 64) { |
| uint8_t Byte = (Imm >> i) & 0xFF; |
| if (Byte == 0xF3) |
| return true; |
| if (!llvm::is_contained(OptionalPrefixBytes, Byte)) |
| return false; |
| i += 8; |
| } |
| |
| return false; |
| } |
| |
| void X86DAGToDAGISel::PreprocessISelDAG() { |
| bool MadeChange = false; |
| for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), |
| E = CurDAG->allnodes_end(); I != E; ) { |
| SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues. |
| |
| // This is for CET enhancement. |
| // |
| // ENDBR32 and ENDBR64 have specific opcodes: |
| // ENDBR32: F3 0F 1E FB |
| // ENDBR64: F3 0F 1E FA |
| // And we want that attackers won’t find unintended ENDBR32/64 |
| // opcode matches in the binary |
| // Here’s an example: |
| // If the compiler had to generate asm for the following code: |
| // a = 0xF30F1EFA |
| // it could, for example, generate: |
| // mov 0xF30F1EFA, dword ptr[a] |
| // In such a case, the binary would include a gadget that starts |
| // with a fake ENDBR64 opcode. Therefore, we split such generation |
| // into multiple operations, let it not shows in the binary |
| if (N->getOpcode() == ISD::Constant) { |
| MVT VT = N->getSimpleValueType(0); |
| int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue(); |
| int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB; |
| if (Imm == EndbrImm || isEndbrImm64(Imm)) { |
| // Check that the cf-protection-branch is enabled. |
| Metadata *CFProtectionBranch = |
| MF->getMMI().getModule()->getModuleFlag("cf-protection-branch"); |
| if (CFProtectionBranch || IndirectBranchTracking) { |
| SDLoc dl(N); |
| SDValue Complement = CurDAG->getConstant(~Imm, dl, VT, false, true); |
| Complement = CurDAG->getNOT(dl, Complement, VT); |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Complement); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| } |
| } |
| |
| // If this is a target specific AND node with no flag usages, turn it back |
| // into ISD::AND to enable test instruction matching. |
| if (N->getOpcode() == X86ISD::AND && !N->hasAnyUseOfValue(1)) { |
| SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0), |
| N->getOperand(0), N->getOperand(1)); |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| |
| // Convert vector increment or decrement to sub/add with an all-ones |
| // constant: |
| // add X, <1, 1...> --> sub X, <-1, -1...> |
| // sub X, <1, 1...> --> add X, <-1, -1...> |
| // The all-ones vector constant can be materialized using a pcmpeq |
| // instruction that is commonly recognized as an idiom (has no register |
| // dependency), so that's better/smaller than loading a splat 1 constant. |
| // |
| // But don't do this if it would inhibit a potentially profitable load |
| // folding opportunity for the other operand. That only occurs with the |
| // intersection of: |
| // (1) The other operand (op0) is load foldable. |
| // (2) The op is an add (otherwise, we are *creating* an add and can still |
| // load fold the other op). |
| // (3) The target has AVX (otherwise, we have a destructive add and can't |
| // load fold the other op without killing the constant op). |
| // (4) The constant 1 vector has multiple uses (so it is profitable to load |
| // into a register anyway). |
| auto mayPreventLoadFold = [&]() { |
| return X86::mayFoldLoad(N->getOperand(0), *Subtarget) && |
| N->getOpcode() == ISD::ADD && Subtarget->hasAVX() && |
| !N->getOperand(1).hasOneUse(); |
| }; |
| if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && |
| N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) { |
| APInt SplatVal; |
| if (X86::isConstantSplat(N->getOperand(1), SplatVal) && |
| SplatVal.isOne()) { |
| SDLoc DL(N); |
| |
| MVT VT = N->getSimpleValueType(0); |
| unsigned NumElts = VT.getSizeInBits() / 32; |
| SDValue AllOnes = |
| CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts)); |
| AllOnes = CurDAG->getBitcast(VT, AllOnes); |
| |
| unsigned NewOpcode = N->getOpcode() == ISD::ADD ? ISD::SUB : ISD::ADD; |
| SDValue Res = |
| CurDAG->getNode(NewOpcode, DL, VT, N->getOperand(0), AllOnes); |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Res.getNode()); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| } |
| |
| switch (N->getOpcode()) { |
| case X86ISD::VBROADCAST: { |
| MVT VT = N->getSimpleValueType(0); |
| // Emulate v32i16/v64i8 broadcast without BWI. |
| if (!Subtarget->hasBWI() && (VT == MVT::v32i16 || VT == MVT::v64i8)) { |
| MVT NarrowVT = VT == MVT::v32i16 ? MVT::v16i16 : MVT::v32i8; |
| SDLoc dl(N); |
| SDValue NarrowBCast = |
| CurDAG->getNode(X86ISD::VBROADCAST, dl, NarrowVT, N->getOperand(0)); |
| SDValue Res = |
| CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), |
| NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); |
| unsigned Index = VT == MVT::v32i16 ? 16 : 32; |
| Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, |
| CurDAG->getIntPtrConstant(Index, dl)); |
| |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Res.getNode()); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| |
| break; |
| } |
| case X86ISD::VBROADCAST_LOAD: { |
| MVT VT = N->getSimpleValueType(0); |
| // Emulate v32i16/v64i8 broadcast without BWI. |
| if (!Subtarget->hasBWI() && (VT == MVT::v32i16 || VT == MVT::v64i8)) { |
| MVT NarrowVT = VT == MVT::v32i16 ? MVT::v16i16 : MVT::v32i8; |
| auto *MemNode = cast<MemSDNode>(N); |
| SDLoc dl(N); |
| SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other); |
| SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()}; |
| SDValue NarrowBCast = CurDAG->getMemIntrinsicNode( |
| X86ISD::VBROADCAST_LOAD, dl, VTs, Ops, MemNode->getMemoryVT(), |
| MemNode->getMemOperand()); |
| SDValue Res = |
| CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, CurDAG->getUNDEF(VT), |
| NarrowBCast, CurDAG->getIntPtrConstant(0, dl)); |
| unsigned Index = VT == MVT::v32i16 ? 16 : 32; |
| Res = CurDAG->getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, NarrowBCast, |
| CurDAG->getIntPtrConstant(Index, dl)); |
| |
| --I; |
| SDValue To[] = {Res, NarrowBCast.getValue(1)}; |
| CurDAG->ReplaceAllUsesWith(N, To); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| |
| break; |
| } |
| case ISD::VSELECT: { |
| // Replace VSELECT with non-mask conditions with with BLENDV. |
| if (N->getOperand(0).getValueType().getVectorElementType() == MVT::i1) |
| break; |
| |
| assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!"); |
| SDValue Blendv = |
| CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0), |
| N->getOperand(0), N->getOperand(1), N->getOperand(2)); |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Blendv.getNode()); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| case ISD::FP_ROUND: |
| case ISD::STRICT_FP_ROUND: |
| case ISD::FP_TO_SINT: |
| case ISD::FP_TO_UINT: |
| case ISD::STRICT_FP_TO_SINT: |
| case ISD::STRICT_FP_TO_UINT: { |
| // Replace vector fp_to_s/uint with their X86 specific equivalent so we |
| // don't need 2 sets of patterns. |
| if (!N->getSimpleValueType(0).isVector()) |
| break; |
| |
| unsigned NewOpc; |
| switch (N->getOpcode()) { |
| default: llvm_unreachable("Unexpected opcode!"); |
| case ISD::FP_ROUND: NewOpc = X86ISD::VFPROUND; break; |
| case ISD::STRICT_FP_ROUND: NewOpc = X86ISD::STRICT_VFPROUND; break; |
| case ISD::STRICT_FP_TO_SINT: NewOpc = X86ISD::STRICT_CVTTP2SI; break; |
| case ISD::FP_TO_SINT: NewOpc = X86ISD::CVTTP2SI; break; |
| case ISD::STRICT_FP_TO_UINT: NewOpc = X86ISD::STRICT_CVTTP2UI; break; |
| case ISD::FP_TO_UINT: NewOpc = X86ISD::CVTTP2UI; break; |
| } |
| SDValue Res; |
| if (N->isStrictFPOpcode()) |
| Res = |
| CurDAG->getNode(NewOpc, SDLoc(N), {N->getValueType(0), MVT::Other}, |
| {N->getOperand(0), N->getOperand(1)}); |
| else |
| Res = |
| CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), |
| N->getOperand(0)); |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Res.getNode()); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| case ISD::SHL: |
| case ISD::SRA: |
| case ISD::SRL: { |
| // Replace vector shifts with their X86 specific equivalent so we don't |
| // need 2 sets of patterns. |
| if (!N->getValueType(0).isVector()) |
| break; |
| |
| unsigned NewOpc; |
| switch (N->getOpcode()) { |
| default: llvm_unreachable("Unexpected opcode!"); |
| case ISD::SHL: NewOpc = X86ISD::VSHLV; break; |
| case ISD::SRA: NewOpc = X86ISD::VSRAV; break; |
| case ISD::SRL: NewOpc = X86ISD::VSRLV; break; |
| } |
| SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), |
| N->getOperand(0), N->getOperand(1)); |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| case ISD::ANY_EXTEND: |
| case ISD::ANY_EXTEND_VECTOR_INREG: { |
| // Replace vector any extend with the zero extend equivalents so we don't |
| // need 2 sets of patterns. Ignore vXi1 extensions. |
| if (!N->getValueType(0).isVector()) |
| break; |
| |
| unsigned NewOpc; |
| if (N->getOperand(0).getScalarValueSizeInBits() == 1) { |
| assert(N->getOpcode() == ISD::ANY_EXTEND && |
| "Unexpected opcode for mask vector!"); |
| NewOpc = ISD::SIGN_EXTEND; |
| } else { |
| NewOpc = N->getOpcode() == ISD::ANY_EXTEND |
| ? ISD::ZERO_EXTEND |
| : ISD::ZERO_EXTEND_VECTOR_INREG; |
| } |
| |
| SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0), |
| N->getOperand(0)); |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| case ISD::FCEIL: |
| case ISD::STRICT_FCEIL: |
| case ISD::FFLOOR: |
| case ISD::STRICT_FFLOOR: |
| case ISD::FTRUNC: |
| case ISD::STRICT_FTRUNC: |
| case ISD::FROUNDEVEN: |
| case ISD::STRICT_FROUNDEVEN: |
| case ISD::FNEARBYINT: |
| case ISD::STRICT_FNEARBYINT: |
| case ISD::FRINT: |
| case ISD::STRICT_FRINT: { |
| // Replace fp rounding with their X86 specific equivalent so we don't |
| // need 2 sets of patterns. |
| unsigned Imm; |
| switch (N->getOpcode()) { |
| default: llvm_unreachable("Unexpected opcode!"); |
| case ISD::STRICT_FCEIL: |
| case ISD::FCEIL: Imm = 0xA; break; |
| case ISD::STRICT_FFLOOR: |
| case ISD::FFLOOR: Imm = 0x9; break; |
| case ISD::STRICT_FTRUNC: |
| case ISD::FTRUNC: Imm = 0xB; break; |
| case ISD::STRICT_FROUNDEVEN: |
| case ISD::FROUNDEVEN: Imm = 0x8; break; |
| case ISD::STRICT_FNEARBYINT: |
| case ISD::FNEARBYINT: Imm = 0xC; break; |
| case ISD::STRICT_FRINT: |
| case ISD::FRINT: Imm = 0x4; break; |
| } |
| SDLoc dl(N); |
| bool IsStrict = N->isStrictFPOpcode(); |
| SDValue Res; |
| if (IsStrict) |
| Res = CurDAG->getNode(X86ISD::STRICT_VRNDSCALE, dl, |
| {N->getValueType(0), MVT::Other}, |
| {N->getOperand(0), N->getOperand(1), |
| CurDAG->getTargetConstant(Imm, dl, MVT::i32)}); |
| else |
| Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl, N->getValueType(0), |
| N->getOperand(0), |
| CurDAG->getTargetConstant(Imm, dl, MVT::i32)); |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Res.getNode()); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| case X86ISD::FANDN: |
| case X86ISD::FAND: |
| case X86ISD::FOR: |
| case X86ISD::FXOR: { |
| // Widen scalar fp logic ops to vector to reduce isel patterns. |
| // FIXME: Can we do this during lowering/combine. |
| MVT VT = N->getSimpleValueType(0); |
| if (VT.isVector() || VT == MVT::f128) |
| break; |
| |
| MVT VecVT = VT == MVT::f64 ? MVT::v2f64 |
| : VT == MVT::f32 ? MVT::v4f32 |
| : MVT::v8f16; |
| |
| SDLoc dl(N); |
| SDValue Op0 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, |
| N->getOperand(0)); |
| SDValue Op1 = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, |
| N->getOperand(1)); |
| |
| SDValue Res; |
| if (Subtarget->hasSSE2()) { |
| EVT IntVT = EVT(VecVT).changeVectorElementTypeToInteger(); |
| Op0 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op0); |
| Op1 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op1); |
| unsigned Opc; |
| switch (N->getOpcode()) { |
| default: llvm_unreachable("Unexpected opcode!"); |
| case X86ISD::FANDN: Opc = X86ISD::ANDNP; break; |
| case X86ISD::FAND: Opc = ISD::AND; break; |
| case X86ISD::FOR: Opc = ISD::OR; break; |
| case X86ISD::FXOR: Opc = ISD::XOR; break; |
| } |
| Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1); |
| Res = CurDAG->getNode(ISD::BITCAST, dl, VecVT, Res); |
| } else { |
| Res = CurDAG->getNode(N->getOpcode(), dl, VecVT, Op0, Op1); |
| } |
| Res = CurDAG->getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, |
| CurDAG->getIntPtrConstant(0, dl)); |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); |
| ++I; |
| MadeChange = true; |
| continue; |
| } |
| } |
| |
| if (OptLevel != CodeGenOpt::None && |
| // Only do this when the target can fold the load into the call or |
| // jmp. |
| !Subtarget->useIndirectThunkCalls() && |
| ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) || |
| (N->getOpcode() == X86ISD::TC_RETURN && |
| (Subtarget->is64Bit() || |
| !getTargetMachine().isPositionIndependent())))) { |
| /// Also try moving call address load from outside callseq_start to just |
| /// before the call to allow it to be folded. |
| /// |
| /// [Load chain] |
| /// ^ |
| /// | |
| /// [Load] |
| /// ^ ^ |
| /// | | |
| /// / \-- |
| /// / | |
| ///[CALLSEQ_START] | |
| /// ^ | |
| /// | | |
| /// [LOAD/C2Reg] | |
| /// | | |
| /// \ / |
| /// \ / |
| /// [CALL] |
| bool HasCallSeq = N->getOpcode() == X86ISD::CALL; |
| SDValue Chain = N->getOperand(0); |
| SDValue Load = N->getOperand(1); |
| if (!isCalleeLoad(Load, Chain, HasCallSeq)) |
| continue; |
| moveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); |
| ++NumLoadMoved; |
| MadeChange = true; |
| continue; |
| } |
| |
| // Lower fpround and fpextend nodes that target the FP stack to be store and |
| // load to the stack. This is a gross hack. We would like to simply mark |
| // these as being illegal, but when we do that, legalize produces these when |
| // it expands calls, then expands these in the same legalize pass. We would |
| // like dag combine to be able to hack on these between the call expansion |
| // and the node legalization. As such this pass basically does "really |
| // late" legalization of these inline with the X86 isel pass. |
| // FIXME: This should only happen when not compiled with -O0. |
| switch (N->getOpcode()) { |
| default: continue; |
| case ISD::FP_ROUND: |
| case ISD::FP_EXTEND: |
| { |
| MVT SrcVT = N->getOperand(0).getSimpleValueType(); |
| MVT DstVT = N->getSimpleValueType(0); |
| |
| // If any of the sources are vectors, no fp stack involved. |
| if (SrcVT.isVector() || DstVT.isVector()) |
| continue; |
| |
| // If the source and destination are SSE registers, then this is a legal |
| // conversion that should not be lowered. |
| const X86TargetLowering *X86Lowering = |
| static_cast<const X86TargetLowering *>(TLI); |
| bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); |
| bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); |
| if (SrcIsSSE && DstIsSSE) |
| continue; |
| |
| if (!SrcIsSSE && !DstIsSSE) { |
| // If this is an FPStack extension, it is a noop. |
| if (N->getOpcode() == ISD::FP_EXTEND) |
| continue; |
| // If this is a value-preserving FPStack truncation, it is a noop. |
| if (N->getConstantOperandVal(1)) |
| continue; |
| } |
| |
| // Here we could have an FP stack truncation or an FPStack <-> SSE convert. |
| // FPStack has extload and truncstore. SSE can fold direct loads into other |
| // operations. Based on this, decide what we want to do. |
| MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT; |
| SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); |
| int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); |
| MachinePointerInfo MPI = |
| MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); |
| SDLoc dl(N); |
| |
| // FIXME: optimize the case where the src/dest is a load or store? |
| |
| SDValue Store = CurDAG->getTruncStore( |
| CurDAG->getEntryNode(), dl, N->getOperand(0), MemTmp, MPI, MemVT); |
| SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, |
| MemTmp, MPI, MemVT); |
| |
| // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the |
| // extload we created. This will cause general havok on the dag because |
| // anything below the conversion could be folded into other existing nodes. |
| // To avoid invalidating 'I', back it up to the convert node. |
| --I; |
| CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); |
| break; |
| } |
| |
| //The sequence of events for lowering STRICT_FP versions of these nodes requires |
| //dealing with the chain differently, as there is already a preexisting chain. |
| case ISD::STRICT_FP_ROUND: |
| case ISD::STRICT_FP_EXTEND: |
| { |
| MVT SrcVT = N->getOperand(1).getSimpleValueType(); |
| MVT DstVT = N->getSimpleValueType(0); |
| |
| // If any of the sources are vectors, no fp stack involved. |
| if (SrcVT.isVector() || DstVT.isVector()) |
| continue; |
| |
| // If the source and destination are SSE registers, then this is a legal |
| // conversion that should not be lowered. |
| const X86TargetLowering *X86Lowering = |
| static_cast<const X86TargetLowering *>(TLI); |
| bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(SrcVT); |
| bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(DstVT); |
| if (SrcIsSSE && DstIsSSE) |
| continue; |
| |
| if (!SrcIsSSE && !DstIsSSE) { |
| // If this is an FPStack extension, it is a noop. |
| if (N->getOpcode() == ISD::STRICT_FP_EXTEND) |
| continue; |
| // If this is a value-preserving FPStack truncation, it is a noop. |
| if (N->getConstantOperandVal(2)) |
| continue; |
| } |
| |
| // Here we could have an FP stack truncation or an FPStack <-> SSE convert. |
| // FPStack has extload and truncstore. SSE can fold direct loads into other |
| // operations. Based on this, decide what we want to do. |
| MVT MemVT = (N->getOpcode() == ISD::STRICT_FP_ROUND) ? DstVT : SrcVT; |
| SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); |
| int SPFI = cast<FrameIndexSDNode>(MemTmp)->getIndex(); |
| MachinePointerInfo MPI = |
| MachinePointerInfo::getFixedStack(CurDAG->getMachineFunction(), SPFI); |
| SDLoc dl(N); |
| |
| // FIXME: optimize the case where the src/dest is a load or store? |
| |
| //Since the operation is StrictFP, use the preexisting chain. |
| SDValue Store, Result; |
| if (!SrcIsSSE) { |
| SDVTList VTs = CurDAG->getVTList(MVT::Other); |
| SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp}; |
| Store = CurDAG->getMemIntrinsicNode(X86ISD::FST, dl, VTs, Ops, MemVT, |
| MPI, /*Align*/ None, |
| MachineMemOperand::MOStore); |
| if (N->getFlags().hasNoFPExcept()) { |
| SDNodeFlags Flags = Store->getFlags(); |
| Flags.setNoFPExcept(true); |
| Store->setFlags(Flags); |
| } |
| } else { |
| assert(SrcVT == MemVT && "Unexpected VT!"); |
| Store = CurDAG->getStore(N->getOperand(0), dl, N->getOperand(1), MemTmp, |
| MPI); |
| } |
| |
| if (!DstIsSSE) { |
| SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other); |
| SDValue Ops[] = {Store, MemTmp}; |
| Result = CurDAG->getMemIntrinsicNode( |
| X86ISD::FLD, dl, VTs, Ops, MemVT, MPI, |
| /*Align*/ None, MachineMemOperand::MOLoad); |
| if (N->getFlags().hasNoFPExcept()) { |
| SDNodeFlags Flags = Result->getFlags(); |
| Flags.setNoFPExcept(true); |
| Result->setFlags(Flags); |
| } |
| } else { |
| assert(DstVT == MemVT && "Unexpected VT!"); |
| Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI); |
| } |
| |
| // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the |
| // extload we created. This will cause general havok on the dag because |
| // anything below the conversion could be folded into other existing nodes. |
| // To avoid invalidating 'I', back it up to the convert node. |
| --I; |
| CurDAG->ReplaceAllUsesWith(N, Result.getNode()); |
| break; |
| } |
| } |
| |
| |
| // Now that we did that, the node is dead. Increment the iterator to the |
| // next node to process, then delete N. |
| ++I; |
| MadeChange = true; |
| } |
| |
| // Remove any dead nodes that may have been left behind. |
| if (MadeChange) |
| CurDAG->RemoveDeadNodes(); |
| } |
| |
| // Look for a redundant movzx/movsx that can occur after an 8-bit divrem. |
| bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) { |
| unsigned Opc = N->getMachineOpcode(); |
| if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 && |
| Opc != X86::MOVSX64rr8) |
| return false; |
| |
| SDValue N0 = N->getOperand(0); |
| |
| // We need to be extracting the lower bit of an extend. |
| if (!N0.isMachineOpcode() || |
| N0.getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG || |
| N0.getConstantOperandVal(1) != X86::sub_8bit) |
| return false; |
| |
| // We're looking for either a movsx or movzx to match the original opcode. |
| unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX |
| : X86::MOVSX32rr8_NOREX; |
| SDValue N00 = N0.getOperand(0); |
| if (!N00.isMachineOpcode() || N00.getMachineOpcode() != ExpectedOpc) |
| return false; |
| |
| if (Opc == X86::MOVSX64rr8) { |
| // If we had a sign extend from 8 to 64 bits. We still need to go from 32 |
| // to 64. |
| MachineSDNode *Extend = CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N), |
| MVT::i64, N00); |
| ReplaceUses(N, Extend); |
| } else { |
| // Ok we can drop this extend and just use the original extend. |
| ReplaceUses(N, N00.getNode()); |
| } |
| |
| return true; |
| } |
| |
| void X86DAGToDAGISel::PostprocessISelDAG() { |
| // Skip peepholes at -O0. |
| if (TM.getOptLevel() == CodeGenOpt::None) |
| return; |
| |
| SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); |
| |
| bool MadeChange = false; |
| while (Position != CurDAG->allnodes_begin()) { |
| SDNode *N = &*--Position; |
| // Skip dead nodes and any non-machine opcodes. |
| if (N->use_empty() || !N->isMachineOpcode()) |
| continue; |
| |
| if (tryOptimizeRem8Extend(N)) { |
| MadeChange = true; |
| continue; |
| } |
| |
| // Look for a TESTrr+ANDrr pattern where both operands of the test are |
| // the same. Rewrite to remove the AND. |
| unsigned Opc = N->getMachineOpcode(); |
| if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr || |
| Opc == X86::TEST32rr || Opc == X86::TEST64rr) && |
| N->getOperand(0) == N->getOperand(1) && |
| N->isOnlyUserOf(N->getOperand(0).getNode()) && |
| N->getOperand(0).isMachineOpcode()) { |
| SDValue And = N->getOperand(0); |
| unsigned N0Opc = And.getMachineOpcode(); |
| if (N0Opc == X86::AND8rr || N0Opc == X86::AND16rr || |
| N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) { |
| MachineSDNode *Test = CurDAG->getMachineNode(Opc, SDLoc(N), |
| MVT::i32, |
| And.getOperand(0), |
| And.getOperand(1)); |
| ReplaceUses(N, Test); |
| MadeChange = true; |
| continue; |
| } |
| if (N0Opc == X86::AND8rm || N0Opc == X86::AND16rm || |
| N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) { |
| unsigned NewOpc; |
| switch (N0Opc) { |
| case X86::AND8rm: NewOpc = X86::TEST8mr; break; |
| case X86::AND16rm: NewOpc = X86::TEST16mr; break; |
| case X86::AND32rm: NewOpc = X86::TEST32mr; break; |
| case X86::AND64rm: NewOpc = X86::TEST64mr; break; |
| } |
| |
| // Need to swap the memory and register operand. |
| SDValue Ops[] = { And.getOperand(1), |
| And.getOperand(2), |
| And.getOperand(3), |
| And.getOperand(4), |
| And.getOperand(5), |
| And.getOperand(0), |
| And.getOperand(6) /* Chain */ }; |
| MachineSDNode *Test = CurDAG->getMachineNode(NewOpc, SDLoc(N), |
| MVT::i32, MVT::Other, Ops); |
| CurDAG->setNodeMemRefs( |
| Test, cast<MachineSDNode>(And.getNode())->memoperands()); |
| ReplaceUses(N, Test); |
| MadeChange = true; |
| continue; |
| } |
| } |
| |
| // Look for a KAND+KORTEST and turn it into KTEST if only the zero flag is |
| // used. We're doing this late so we can prefer to fold the AND into masked |
| // comparisons. Doing that can be better for the live range of the mask |
| // register. |
| if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr || |
| Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) && |
| N->getOperand(0) == N->getOperand(1) && |
| N->isOnlyUserOf(N->getOperand(0).getNode()) && |
| N->getOperand(0).isMachineOpcode() && |
| onlyUsesZeroFlag(SDValue(N, 0))) { |
| SDValue And = N->getOperand(0); |
| unsigned N0Opc = And.getMachineOpcode(); |
| // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other |
| // KAND instructions and KTEST use the same ISA feature. |
| if (N0Opc == X86::KANDBrr || |
| (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) || |
| N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) { |
| unsigned NewOpc; |
| switch (Opc) { |
| default: llvm_unreachable("Unexpected opcode!"); |
| case X86::KORTESTBrr: NewOpc = X86::KTESTBrr; break; |
| case X86::KORTESTWrr: NewOpc = X86::KTESTWrr; break; |
| case X86::KORTESTDrr: NewOpc = X86::KTESTDrr; break; |
| case X86::KORTESTQrr: NewOpc = X86::KTESTQrr; break; |
| } |
| MachineSDNode *KTest = CurDAG->getMachineNode(NewOpc, SDLoc(N), |
| MVT::i32, |
| And.getOperand(0), |
| And.getOperand(1)); |
| ReplaceUses(N, KTest); |
| MadeChange = true; |
| continue; |
| } |
| } |
| |
| // Attempt to remove vectors moves that were inserted to zero upper bits. |
| if (Opc != TargetOpcode::SUBREG_TO_REG) |
| continue; |
| |
| unsigned SubRegIdx = N->getConstantOperandVal(2); |
| if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm) |
| continue; |
| |
| SDValue Move = N->getOperand(1); |
| if (!Move.isMachineOpcode()) |
| continue; |
| |
| // Make sure its one of the move opcodes we recognize. |
| switch (Move.getMachineOpcode()) { |
| default: |
| continue; |
| case X86::VMOVAPDrr: case X86::VMOVUPDrr: |
| case X86::VMOVAPSrr: case X86::VMOVUPSrr: |
| case X86::VMOVDQArr: case X86::VMOVDQUrr: |
| case X86::VMOVAPDYrr: case X86::VMOVUPDYrr: |
| case X86::VMOVAPSYrr: case X86::VMOVUPSYrr: |
| case X86::VMOVDQAYrr: case X86::VMOVDQUYrr: |
| case X86::VMOVAPDZ128rr: case X86::VMOVUPDZ128rr: |
| case X86::VMOVAPSZ128rr: case X86::VMOVUPSZ128rr: |
| case X86::VMOVDQA32Z128rr: case X86::VMOVDQU32Z128rr: |
| case X86::VMOVDQA64Z128rr: case X86::VMOVDQU64Z128rr: |
| case X86::VMOVAPDZ256rr: case X86::VMOVUPDZ256rr: |
| case X86::VMOVAPSZ256rr: case X86::VMOVUPSZ256rr: |
| case X86::VMOVDQA32Z256rr: case X86::VMOVDQU32Z256rr: |
| case X86::VMOVDQA64Z256rr: case X86::VMOVDQU64Z256rr: |
| break; |
| } |
| |
| SDValue In = Move.getOperand(0); |
| if (!In.isMachineOpcode() || |
| In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END) |
| continue; |
| |
| // Make sure the instruction has a VEX, XOP, or EVEX prefix. This covers |
| // the SHA instructions which use a legacy encoding. |
| uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags; |
| if ((TSFlags & X86II::EncodingMask) != X86II::VEX && |
| (TSFlags & X86II::EncodingMask) != X86II::EVEX && |
| (TSFlags & X86II::EncodingMask) != X86II::XOP) |
| continue; |
| |
| // Producing instruction is another vector instruction. We can drop the |
| // move. |
| CurDAG->UpdateNodeOperands(N, N->getOperand(0), In, N->getOperand(2)); |
| MadeChange = true; |
| } |
| |
| if (MadeChange) |
| CurDAG->RemoveDeadNodes(); |
| } |
| |
| |
| /// Emit any code that needs to be executed only in the main function. |
| void X86DAGToDAGISel::emitSpecialCodeForMain() { |
| if (Subtarget->isTargetCygMing()) { |
| TargetLowering::ArgListTy Args; |
| auto &DL = CurDAG->getDataLayout(); |
| |
| TargetLowering::CallLoweringInfo CLI(*CurDAG); |
| CLI.setChain(CurDAG->getRoot()) |
| .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()), |
| CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)), |
| std::move(Args)); |
| const TargetLowering &TLI = CurDAG->getTargetLoweringInfo(); |
| std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); |
| CurDAG->setRoot(Result.second); |
| } |
| } |
| |
| void X86DAGToDAGISel::emitFunctionEntryCode() { |
| // If this is main, emit special code for main. |
| const Function &F = MF->getFunction(); |
| if (F.hasExternalLinkage() && F.getName() == "main") |
| emitSpecialCodeForMain(); |
| } |
| |
| static bool isDispSafeForFrameIndex(int64_t Val) { |
| // On 64-bit platforms, we can run into an issue where a frame index |
| // includes a displacement that, when added to the explicit displacement, |
| // will overflow the displacement field. Assuming that the frame index |
| // displacement fits into a 31-bit integer (which is only slightly more |
| // aggressive than the current fundamental assumption that it fits into |
| // a 32-bit integer), a 31-bit disp should always be safe. |
| return isInt<31>(Val); |
| } |
| |
| bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset, |
| X86ISelAddressMode &AM) { |
| // We may have already matched a displacement and the caller just added the |
| // symbolic displacement. So we still need to do the checks even if Offset |
| // is zero. |
| |
| int64_t Val = AM.Disp + Offset; |
| |
| // Cannot combine ExternalSymbol displacements with integer offsets. |
| if (Val != 0 && (AM.ES || AM.MCSym)) |
| return true; |
| |
| CodeModel::Model M = TM.getCodeModel(); |
| if (Subtarget->is64Bit()) { |
| if (Val != 0 && |
| !X86::isOffsetSuitableForCodeModel(Val, M, |
| AM.hasSymbolicDisplacement())) |
| return true; |
| // In addition to the checks required for a register base, check that |
| // we do not try to use an unsafe Disp with a frame index. |
| if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && |
| !isDispSafeForFrameIndex(Val)) |
| return true; |
| } |
| AM.Disp = Val; |
| return false; |
| |
| } |
| |
| bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM, |
| bool AllowSegmentRegForX32) { |
| SDValue Address = N->getOperand(1); |
| |
| // load gs:0 -> GS segment register. |
| // load fs:0 -> FS segment register. |
| // |
| // This optimization is generally valid because the GNU TLS model defines that |
| // gs:0 (or fs:0 on X86-64) contains its own address. However, for X86-64 mode |
| // with 32-bit registers, as we get in ILP32 mode, those registers are first |
| // zero-extended to 64 bits and then added it to the base address, which gives |
| // unwanted results when the register holds a negative value. |
| // For more information see http://people.redhat.com/drepper/tls.pdf |
| if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) { |
| if (C->getSExtValue() == 0 && AM.Segment.getNode() == nullptr && |
| !IndirectTlsSegRefs && |
| (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() || |
| Subtarget->isTargetFuchsia())) { |
| if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32) |
| return true; |
| switch (N->getPointerInfo().getAddrSpace()) { |
| case X86AS::GS: |
| AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); |
| return false; |
| case X86AS::FS: |
| AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); |
| return false; |
| // Address space X86AS::SS is not handled here, because it is not used to |
| // address TLS areas. |
| } |
| } |
| } |
| |
| return true; |
| } |
| |
| /// Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes into an addressing |
| /// mode. These wrap things that will resolve down into a symbol reference. |
| /// If no match is possible, this returns true, otherwise it returns false. |
| bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) { |
| // If the addressing mode already has a symbol as the displacement, we can |
| // never match another symbol. |
| if (AM.hasSymbolicDisplacement()) |
| return true; |
| |
| bool IsRIPRelTLS = false; |
| bool IsRIPRel = N.getOpcode() == X86ISD::WrapperRIP; |
| if (IsRIPRel) { |
| SDValue Val = N.getOperand(0); |
| if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) |
| IsRIPRelTLS = true; |
| } |
| |
| // We can't use an addressing mode in the 64-bit large code model. |
| // Global TLS addressing is an exception. In the medium code model, |
| // we use can use a mode when RIP wrappers are present. |
| // That signifies access to globals that are known to be "near", |
| // such as the GOT itself. |
| CodeModel::Model M = TM.getCodeModel(); |
| if (Subtarget->is64Bit() && |
| ((M == CodeModel::Large && !IsRIPRelTLS) || |
| (M == CodeModel::Medium && !IsRIPRel))) |
| return true; |
| |
| // Base and index reg must be 0 in order to use %rip as base. |
| if (IsRIPRel && AM.hasBaseOrIndexReg()) |
| return true; |
| |
| // Make a local copy in case we can't do this fold. |
| X86ISelAddressMode Backup = AM; |
| |
| int64_t Offset = 0; |
| SDValue N0 = N.getOperand(0); |
| if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { |
| AM.GV = G->getGlobal(); |
| AM.SymbolFlags = G->getTargetFlags(); |
| Offset = G->getOffset(); |
| } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { |
| AM.CP = CP->getConstVal(); |
| AM.Alignment = CP->getAlign(); |
| AM.SymbolFlags = CP->getTargetFlags(); |
| Offset = CP->getOffset(); |
| } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { |
| AM.ES = S->getSymbol(); |
| AM.SymbolFlags = S->getTargetFlags(); |
| } else if (auto *S = dyn_cast<MCSymbolSDNode>(N0)) { |
| AM.MCSym = S->getMCSymbol(); |
| } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { |
| AM.JT = J->getIndex(); |
| AM.SymbolFlags = J->getTargetFlags(); |
| } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) { |
| AM.BlockAddr = BA->getBlockAddress(); |
| AM.SymbolFlags = BA->getTargetFlags(); |
| Offset = BA->getOffset(); |
| } else |
| llvm_unreachable("Unhandled symbol reference node."); |
| |
| if (foldOffsetIntoAddress(Offset, AM)) { |
| AM = Backup; |
| return true; |
| } |
| |
| if (IsRIPRel) |
| AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); |
| |
| // Commit the changes now that we know this fold is safe. |
| return false; |
| } |
| |
| /// Add the specified node to the specified addressing mode, returning true if |
| /// it cannot be done. This just pattern matches for the addressing mode. |
| bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) { |
| if (matchAddressRecursively(N, AM, 0)) |
| return true; |
| |
| // Post-processing: Make a second attempt to fold a load, if we now know |
| // that there will not be any other register. This is only performed for |
| // 64-bit ILP32 mode since 32-bit mode and 64-bit LP64 mode will have folded |
| // any foldable load the first time. |
| if (Subtarget->isTarget64BitILP32() && |
| AM.BaseType == X86ISelAddressMode::RegBase && |
| AM.Base_Reg.getNode() != nullptr && AM.IndexReg.getNode() == nullptr) { |
| SDValue Save_Base_Reg = AM.Base_Reg; |
| if (auto *LoadN = dyn_cast<LoadSDNode>(Save_Base_Reg)) { |
| AM.Base_Reg = SDValue(); |
| if (matchLoadInAddress(LoadN, AM, /*AllowSegmentRegForX32=*/true)) |
| AM.Base_Reg = Save_Base_Reg; |
| } |
| } |
| |
| // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has |
| // a smaller encoding and avoids a scaled-index. |
| if (AM.Scale == 2 && |
| AM.BaseType == X86ISelAddressMode::RegBase && |
| AM.Base_Reg.getNode() == nullptr) { |
| AM.Base_Reg = AM.IndexReg; |
| AM.Scale = 1; |
| } |
| |
| // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, |
| // because it has a smaller encoding. |
| // TODO: Which other code models can use this? |
| switch (TM.getCodeModel()) { |
| default: break; |
| case CodeModel::Small: |
| case CodeModel::Kernel: |
| if (Subtarget->is64Bit() && |
| AM.Scale == 1 && |
| AM.BaseType == X86ISelAddressMode::RegBase && |
| AM.Base_Reg.getNode() == nullptr && |
| AM.IndexReg.getNode() == nullptr && |
| AM.SymbolFlags == X86II::MO_NO_FLAG && |
| AM.hasSymbolicDisplacement()) |
| AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); |
| break; |
| } |
| |
| return false; |
| } |
| |
| bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM, |
| unsigned Depth) { |
| // Add an artificial use to this node so that we can keep track of |
| // it if it gets CSE'd with a different node. |
| HandleSDNode Handle(N); |
| |
| X86ISelAddressMode Backup = AM; |
| if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) && |
| !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) |
| return false; |
| AM = Backup; |
| |
| // Try again after commutating the operands. |
| if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM, |
| Depth + 1) && |
| !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth + 1)) |
| return false; |
| AM = Backup; |
| |
| // If we couldn't fold both operands into the address at the same time, |
| // see if we can just put each operand into a register and fold at least |
| // the add. |
| if (AM.BaseType == X86ISelAddressMode::RegBase && |
| !AM.Base_Reg.getNode() && |
| !AM.IndexReg.getNode()) { |
| N = Handle.getValue(); |
| AM.Base_Reg = N.getOperand(0); |
| AM.IndexReg = N.getOperand(1); |
| AM.Scale = 1; |
| return false; |
| } |
| N = Handle.getValue(); |
| return true; |
| } |
| |
| // Insert a node into the DAG at least before the Pos node's position. This |
| // will reposition the node as needed, and will assign it a node ID that is <= |
| // the Pos node's ID. Note that this does *not* preserve the uniqueness of node |
| // IDs! The selection DAG must no longer depend on their uniqueness when this |
| // is used. |
| static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) { |
| if (N->getNodeId() == -1 || |
| (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) > |
| SelectionDAGISel::getUninvalidatedNodeId(Pos.getNode()))) { |
| DAG.RepositionNode(Pos->getIterator(), N.getNode()); |
| // Mark Node as invalid for pruning as after this it may be a successor to a |
| // selected node but otherwise be in the same position of Pos. |
| // Conservatively mark it with the same -abs(Id) to assure node id |
| // invariant is preserved. |
| N->setNodeId(Pos->getNodeId()); |
| SelectionDAGISel::InvalidateNodeId(N.getNode()); |
| } |
| } |
| |
| // Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if |
| // safe. This allows us to convert the shift and and into an h-register |
| // extract and a scaled index. Returns false if the simplification is |
| // performed. |
| static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, |
| uint64_t Mask, |
| SDValue Shift, SDValue X, |
| X86ISelAddressMode &AM) { |
| if (Shift.getOpcode() != ISD::SRL || |
| !isa<ConstantSDNode>(Shift.getOperand(1)) || |
| !Shift.hasOneUse()) |
| return true; |
| |
| int ScaleLog = 8 - Shift.getConstantOperandVal(1); |
| if (ScaleLog <= 0 || ScaleLog >= 4 || |
| Mask != (0xffu << ScaleLog)) |
| return true; |
| |
| MVT VT = N.getSimpleValueType(); |
| SDLoc DL(N); |
| SDValue Eight = DAG.getConstant(8, DL, MVT::i8); |
| SDValue NewMask = DAG.getConstant(0xff, DL, VT); |
| SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, X, Eight); |
| SDValue And = DAG.getNode(ISD::AND, DL, VT, Srl, NewMask); |
| SDValue ShlCount = DAG.getConstant(ScaleLog, DL, MVT::i8); |
| SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And, ShlCount); |
| |
| // Insert the new nodes into the topological ordering. We must do this in |
| // a valid topological ordering as nothing is going to go back and re-sort |
| // these nodes. We continually insert before 'N' in sequence as this is |
| // essentially a pre-flattened and pre-sorted sequence of nodes. There is no |
| // hierarchy left to express. |
| insertDAGNode(DAG, N, Eight); |
| insertDAGNode(DAG, N, Srl); |
| insertDAGNode(DAG, N, NewMask); |
| insertDAGNode(DAG, N, And); |
| insertDAGNode(DAG, N, ShlCount); |
| insertDAGNode(DAG, N, Shl); |
| DAG.ReplaceAllUsesWith(N, Shl); |
| DAG.RemoveDeadNode(N.getNode()); |
| AM.IndexReg = And; |
| AM.Scale = (1 << ScaleLog); |
| return false; |
| } |
| |
| // Transforms "(X << C1) & C2" to "(X & (C2>>C1)) << C1" if safe and if this |
| // allows us to fold the shift into this addressing mode. Returns false if the |
| // transform succeeded. |
| static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, |
| X86ISelAddressMode &AM) { |
| SDValue Shift = N.getOperand(0); |
| |
| // Use a signed mask so that shifting right will insert sign bits. These |
| // bits will be removed when we shift the result left so it doesn't matter |
| // what we use. This might allow a smaller immediate encoding. |
| int64_t Mask = cast<ConstantSDNode>(N->getOperand(1))->getSExtValue(); |
| |
| // If we have an any_extend feeding the AND, look through it to see if there |
| // is a shift behind it. But only if the AND doesn't use the extended bits. |
| // FIXME: Generalize this to other ANY_EXTEND than i32 to i64? |
| bool FoundAnyExtend = false; |
| if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() && |
| Shift.getOperand(0).getSimpleValueType() == MVT::i32 && |
| isUInt<32>(Mask)) { |
| FoundAnyExtend = true; |
| Shift = Shift.getOperand(0); |
| } |
| |
| if (Shift.getOpcode() != ISD::SHL || |
| !isa<ConstantSDNode>(Shift.getOperand(1))) |
| return true; |
| |
| SDValue X = Shift.getOperand(0); |
| |
| // Not likely to be profitable if either the AND or SHIFT node has more |
| // than one use (unless all uses are for address computation). Besides, |
| // isel mechanism requires their node ids to be reused. |
| if (!N.hasOneUse() || !Shift.hasOneUse()) |
| return true; |
| |
| // Verify that the shift amount is something we can fold. |
| unsigned ShiftAmt = Shift.getConstantOperandVal(1); |
| if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3) |
| return true; |
| |
| MVT VT = N.getSimpleValueType(); |
| SDLoc DL(N); |
| if (FoundAnyExtend) { |
| SDValue NewX = DAG.getNode(ISD::ANY_EXTEND, DL, VT, X); |
| insertDAGNode(DAG, N, NewX); |
| X = NewX; |
| } |
| |
| SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT); |
| SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask); |
| SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1)); |
| |
| // Insert the new nodes into the topological ordering. We must do this in |
| // a valid topological ordering as nothing is going to go back and re-sort |
| // these nodes. We continually insert before 'N' in sequence as this is |
| // essentially a pre-flattened and pre-sorted sequence of nodes. There is no |
| // hierarchy left to express. |
| insertDAGNode(DAG, N, NewMask); |
| insertDAGNode(DAG, N, NewAnd); |
| insertDAGNode(DAG, N, NewShift); |
| DAG.ReplaceAllUsesWith(N, NewShift); |
| DAG.RemoveDeadNode(N.getNode()); |
| |
| AM.Scale = 1 << ShiftAmt; |
| AM.IndexReg = NewAnd; |
| return false; |
| } |
| |
| // Implement some heroics to detect shifts of masked values where the mask can |
| // be replaced by extending the shift and undoing that in the addressing mode |
| // scale. Patterns such as (shl (srl x, c1), c2) are canonicalized into (and |
| // (srl x, SHIFT), MASK) by DAGCombines that don't know the shl can be done in |
| // the addressing mode. This results in code such as: |
| // |
| // int f(short *y, int *lookup_table) { |
| // ... |
| // return *y + lookup_table[*y >> 11]; |
| // } |
| // |
| // Turning into: |
| // movzwl (%rdi), %eax |
| // movl %eax, %ecx |
| // shrl $11, %ecx |
| // addl (%rsi,%rcx,4), %eax |
| // |
| // Instead of: |
| // movzwl (%rdi), %eax |
| // movl %eax, %ecx |
| // shrl $9, %ecx |
| // andl $124, %rcx |
| // addl (%rsi,%rcx), %eax |
| // |
| // Note that this function assumes the mask is provided as a mask *after* the |
| // value is shifted. The input chain may or may not match that, but computing |
| // such a mask is trivial. |
| static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, |
| uint64_t Mask, |
| SDValue Shift, SDValue X, |
| X86ISelAddressMode &AM) { |
| if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() || |
| !isa<ConstantSDNode>(Shift.getOperand(1))) |
| return true; |
| |
| unsigned ShiftAmt = Shift.getConstantOperandVal(1); |
| unsigned MaskLZ = countLeadingZeros(Mask); |
| unsigned MaskTZ = countTrailingZeros(Mask); |
| |
| // The amount of shift we're trying to fit into the addressing mode is taken |
| // from the trailing zeros of the mask. |
| unsigned AMShiftAmt = MaskTZ; |
| |
| // There is nothing we can do here unless the mask is removing some bits. |
| // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. |
| if (AMShiftAmt == 0 || AMShiftAmt > 3) return true; |
| |
| // We also need to ensure that mask is a continuous run of bits. |
| if (countTrailingOnes(Mask >> MaskTZ) + MaskTZ + MaskLZ != 64) return true; |
| |
| // Scale the leading zero count down based on the actual size of the value. |
| // Also scale it down based on the size of the shift. |
| unsigned ScaleDown = (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt; |
| if (MaskLZ < ScaleDown) |
| return true; |
| MaskLZ -= ScaleDown; |
| |
| // The final check is to ensure that any masked out high bits of X are |
| // already known to be zero. Otherwise, the mask has a semantic impact |
| // other than masking out a couple of low bits. Unfortunately, because of |
| // the mask, zero extensions will be removed from operands in some cases. |
| // This code works extra hard to look through extensions because we can |
| // replace them with zero extensions cheaply if necessary. |
| bool ReplacingAnyExtend = false; |
| if (X.getOpcode() == ISD::ANY_EXTEND) { |
| unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() - |
| X.getOperand(0).getSimpleValueType().getSizeInBits(); |
| // Assume that we'll replace the any-extend with a zero-extend, and |
| // narrow the search to the extended value. |
| X = X.getOperand(0); |
| MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits; |
| ReplacingAnyExtend = true; |
| } |
| APInt MaskedHighBits = |
| APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ); |
| KnownBits Known = DAG.computeKnownBits(X); |
| if (MaskedHighBits != Known.Zero) return true; |
| |
| // We've identified a pattern that can be transformed into a single shift |
| // and an addressing mode. Make it so. |
| MVT VT = N.getSimpleValueType(); |
| if (ReplacingAnyExtend) { |
| assert(X.getValueType() != VT); |
| // We looked through an ANY_EXTEND node, insert a ZERO_EXTEND. |
| SDValue NewX = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(X), VT, X); |
| insertDAGNode(DAG, N, NewX); |
| X = NewX; |
| } |
| SDLoc DL(N); |
| SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); |
| SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt); |
| SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); |
| SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewSRL, NewSHLAmt); |
| |
| // Insert the new nodes into the topological ordering. We must do this in |
| // a valid topological ordering as nothing is going to go back and re-sort |
| // these nodes. We continually insert before 'N' in sequence as this is |
| // essentially a pre-flattened and pre-sorted sequence of nodes. There is no |
| // hierarchy left to express. |
| insertDAGNode(DAG, N, NewSRLAmt); |
| insertDAGNode(DAG, N, NewSRL); |
| insertDAGNode(DAG, N, NewSHLAmt); |
| insertDAGNode(DAG, N, NewSHL); |
| DAG.ReplaceAllUsesWith(N, NewSHL); |
| DAG.RemoveDeadNode(N.getNode()); |
| |
| AM.Scale = 1 << AMShiftAmt; |
| AM.IndexReg = NewSRL; |
| return false; |
| } |
| |
| // Transform "(X >> SHIFT) & (MASK << C1)" to |
| // "((X >> (SHIFT + C1)) & (MASK)) << C1". Everything before the SHL will be |
| // matched to a BEXTR later. Returns false if the simplification is performed. |
| static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, |
| uint64_t Mask, |
| SDValue Shift, SDValue X, |
| X86ISelAddressMode &AM, |
| const X86Subtarget &Subtarget) { |
| if (Shift.getOpcode() != ISD::SRL || |
| !isa<ConstantSDNode>(Shift.getOperand(1)) || |
| !Shift.hasOneUse() || !N.hasOneUse()) |
| return true; |
| |
| // Only do this if BEXTR will be matched by matchBEXTRFromAndImm. |
| if (!Subtarget.hasTBM() && |
| !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR())) |
| return true; |
| |
| // We need to ensure that mask is a continuous run of bits. |
| if (!isShiftedMask_64(Mask)) return true; |
| |
| unsigned ShiftAmt = Shift.getConstantOperandVal(1); |
| |
| // The amount of shift we're trying to fit into the addressing mode is taken |
| // from the trailing zeros of the mask. |
| unsigned AMShiftAmt = countTrailingZeros(Mask); |
| |
| // There is nothing we can do here unless the mask is removing some bits. |
| // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits. |
| if (AMShiftAmt == 0 || AMShiftAmt > 3) return true; |
| |
| MVT VT = N.getSimpleValueType(); |
| SDLoc DL(N); |
| SDValue NewSRLAmt = DAG.getConstant(ShiftAmt + AMShiftAmt, DL, MVT::i8); |
| SDValue NewSRL = DAG.getNode(ISD::SRL, DL, VT, X, NewSRLAmt); |
| SDValue NewMask = DAG.getConstant(Mask >> AMShiftAmt, DL, VT); |
| SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, NewSRL, NewMask); |
| SDValue NewSHLAmt = DAG.getConstant(AMShiftAmt, DL, MVT::i8); |
| SDValue NewSHL = DAG.getNode(ISD::SHL, DL, VT, NewAnd, NewSHLAmt); |
| |
| // Insert the new nodes into the topological ordering. We must do this in |
| // a valid topological ordering as nothing is going to go back and re-sort |
| // these nodes. We continually insert before 'N' in sequence as this is |
| // essentially a pre-flattened and pre-sorted sequence of nodes. There is no |
| // hierarchy left to express. |
| insertDAGNode(DAG, N, NewSRLAmt); |
| insertDAGNode(DAG, N, NewSRL); |
| insertDAGNode(DAG, N, NewMask); |
| insertDAGNode(DAG, N, NewAnd); |
| insertDAGNode(DAG, N, NewSHLAmt); |
| insertDAGNode(DAG, N, NewSHL); |
| DAG.ReplaceAllUsesWith(N, NewSHL); |
| DAG.RemoveDeadNode(N.getNode()); |
| |
| AM.Scale = 1 << AMShiftAmt; |
| AM.IndexReg = NewAnd; |
| return false; |
| } |
| |
| bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM, |
| unsigned Depth) { |
| SDLoc dl(N); |
| LLVM_DEBUG({ |
| dbgs() << "MatchAddress: "; |
| AM.dump(CurDAG); |
| }); |
| // Limit recursion. |
| if (Depth > 5) |
| return matchAddressBase(N, AM); |
| |
| // If this is already a %rip relative address, we can only merge immediates |
| // into it. Instead of handling this in every case, we handle it here. |
| // RIP relative addressing: %rip + 32-bit displacement! |
| if (AM.isRIPRelative()) { |
| // FIXME: JumpTable and ExternalSymbol address currently don't like |
| // displacements. It isn't very important, but this should be fixed for |
| // consistency. |
| if (!(AM.ES || AM.MCSym) && AM.JT != -1) |
| return true; |
| |
| if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) |
| if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM)) |
| return false; |
| return true; |
| } |
| |
| switch (N.getOpcode()) { |
| default: break; |
| case ISD::LOCAL_RECOVER: { |
| if (!AM.hasSymbolicDisplacement() && AM.Disp == 0) |
| if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(N.getOperand(0))) { |
| // Use the symbol and don't prefix it. |
| AM.MCSym = ESNode->getMCSymbol(); |
| return false; |
| } |
| break; |
| } |
| case ISD::Constant: { |
| uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); |
| if (!foldOffsetIntoAddress(Val, AM)) |
| return false; |
| break; |
| } |
| |
| case X86ISD::Wrapper: |
| case X86ISD::WrapperRIP: |
| if (!matchWrapper(N, AM)) |
| return false; |
| break; |
| |
| case ISD::LOAD: |
| if (!matchLoadInAddress(cast<LoadSDNode>(N), AM)) |
| return false; |
| break; |
| |
| case ISD::FrameIndex: |
| if (AM.BaseType == X86ISelAddressMode::RegBase && |
| AM.Base_Reg.getNode() == nullptr && |
| (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { |
| AM.BaseType = X86ISelAddressMode::FrameIndexBase; |
| AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); |
| return false; |
| } |
| break; |
| |
| case ISD::SHL: |
| if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) |
| break; |
| |
| if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) { |
| unsigned Val = CN->getZExtValue(); |
| // Note that we handle x<<1 as (,x,2) rather than (x,x) here so |
| // that the base operand remains free for further matching. If |
| // the base doesn't end up getting used, a post-processing step |
| // in MatchAddress turns (,x,2) into (x,x), which is cheaper. |
| if (Val == 1 || Val == 2 || Val == 3) { |
| AM.Scale = 1 << Val; |
| SDValue ShVal = N.getOperand(0); |
| |
| // Okay, we know that we have a scale by now. However, if the scaled |
| // value is an add of something and a constant, we can fold the |
| // constant into the disp field here. |
| if (CurDAG->isBaseWithConstantOffset(ShVal)) { |
| AM.IndexReg = ShVal.getOperand(0); |
| ConstantSDNode *AddVal = cast<ConstantSDNode>(ShVal.getOperand(1)); |
| uint64_t Disp = (uint64_t)AddVal->getSExtValue() << Val; |
| if (!foldOffsetIntoAddress(Disp, AM)) |
| return false; |
| } |
| |
| AM.IndexReg = ShVal; |
| return false; |
| } |
| } |
| break; |
| |
| case ISD::SRL: { |
| // Scale must not be used already. |
| if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; |
| |
| // We only handle up to 64-bit values here as those are what matter for |
| // addressing mode optimizations. |
| assert(N.getSimpleValueType().getSizeInBits() <= 64 && |
| "Unexpected value size!"); |
| |
| SDValue And = N.getOperand(0); |
| if (And.getOpcode() != ISD::AND) break; |
| SDValue X = And.getOperand(0); |
| |
| // The mask used for the transform is expected to be post-shift, but we |
| // found the shift first so just apply the shift to the mask before passing |
| // it down. |
| if (!isa<ConstantSDNode>(N.getOperand(1)) || |
| !isa<ConstantSDNode>(And.getOperand(1))) |
| break; |
| uint64_t Mask = And.getConstantOperandVal(1) >> N.getConstantOperandVal(1); |
| |
| // Try to fold the mask and shift into the scale, and return false if we |
| // succeed. |
| if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, N, X, AM)) |
| return false; |
| break; |
| } |
| |
| case ISD::SMUL_LOHI: |
| case ISD::UMUL_LOHI: |
| // A mul_lohi where we need the low part can be folded as a plain multiply. |
| if (N.getResNo() != 0) break; |
| LLVM_FALLTHROUGH; |
| case ISD::MUL: |
| case X86ISD::MUL_IMM: |
| // X*[3,5,9] -> X+X*[2,4,8] |
| if (AM.BaseType == X86ISelAddressMode::RegBase && |
| AM.Base_Reg.getNode() == nullptr && |
| AM.IndexReg.getNode() == nullptr) { |
| if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) |
| if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || |
| CN->getZExtValue() == 9) { |
| AM.Scale = unsigned(CN->getZExtValue())-1; |
| |
| SDValue MulVal = N.getOperand(0); |
| SDValue Reg; |
| |
| // Okay, we know that we have a scale by now. However, if the scaled |
| // value is an add of something and a constant, we can fold the |
| // constant into the disp field here. |
| if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && |
| isa<ConstantSDNode>(MulVal.getOperand(1))) { |
| Reg = MulVal.getOperand(0); |
| ConstantSDNode *AddVal = |
| cast<ConstantSDNode>(MulVal.getOperand(1)); |
| uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); |
| if (foldOffsetIntoAddress(Disp, AM)) |
| Reg = N.getOperand(0); |
| } else { |
| Reg = N.getOperand(0); |
| } |
| |
| AM.IndexReg = AM.Base_Reg = Reg; |
| return false; |
| } |
| } |
| break; |
| |
| case ISD::SUB: { |
| // Given A-B, if A can be completely folded into the address and |
| // the index field with the index field unused, use -B as the index. |
| // This is a win if a has multiple parts that can be folded into |
| // the address. Also, this saves a mov if the base register has |
| // other uses, since it avoids a two-address sub instruction, however |
| // it costs an additional mov if the index register has other uses. |
| |
| // Add an artificial use to this node so that we can keep track of |
| // it if it gets CSE'd with a different node. |
| HandleSDNode Handle(N); |
| |
| // Test if the LHS of the sub can be folded. |
| X86ISelAddressMode Backup = AM; |
| if (matchAddressRecursively(N.getOperand(0), AM, Depth+1)) { |
| N = Handle.getValue(); |
| AM = Backup; |
| break; |
| } |
| N = Handle.getValue(); |
| // Test if the index field is free for use. |
| if (AM.IndexReg.getNode() || AM.isRIPRelative()) { |
| AM = Backup; |
| break; |
| } |
| |
| int Cost = 0; |
| SDValue RHS = N.getOperand(1); |
| // If the RHS involves a register with multiple uses, this |
| // transformation incurs an extra mov, due to the neg instruction |
| // clobbering its operand. |
| if (!RHS.getNode()->hasOneUse() || |
| RHS.getNode()->getOpcode() == ISD::CopyFromReg || |
| RHS.getNode()->getOpcode() == ISD::TRUNCATE || |
| RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || |
| (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && |
| RHS.getOperand(0).getValueType() == MVT::i32)) |
| ++Cost; |
| // If the base is a register with multiple uses, this |
| // transformation may save a mov. |
| if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() && |
| !AM.Base_Reg.getNode()->hasOneUse()) || |
| AM.BaseType == X86ISelAddressMode::FrameIndexBase) |
| --Cost; |
| // If the folded LHS was interesting, this transformation saves |
| // address arithmetic. |
| if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + |
| ((AM.Disp != 0) && (Backup.Disp == 0)) + |
| (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) |
| --Cost; |
| // If it doesn't look like it may be an overall win, don't do it. |
| if (Cost >= 0) { |
| AM = Backup; |
| break; |
| } |
| |
| // Ok, the transformation is legal and appears profitable. Go for it. |
| // Negation will be emitted later to avoid creating dangling nodes if this |
| // was an unprofitable LEA. |
| AM.IndexReg = RHS; |
| AM.NegateIndex = true; |
| AM.Scale = 1; |
| return false; |
| } |
| |
| case ISD::ADD: |
| if (!matchAdd(N, AM, Depth)) |
| return false; |
| break; |
| |
| case ISD::OR: |
| // We want to look through a transform in InstCombine and DAGCombiner that |
| // turns 'add' into 'or', so we can treat this 'or' exactly like an 'add'. |
| // Example: (or (and x, 1), (shl y, 3)) --> (add (and x, 1), (shl y, 3)) |
| // An 'lea' can then be used to match the shift (multiply) and add: |
| // and $1, %esi |
| // lea (%rsi, %rdi, 8), %rax |
| if (CurDAG->haveNoCommonBitsSet(N.getOperand(0), N.getOperand(1)) && |
| !matchAdd(N, AM, Depth)) |
| return false; |
| break; |
| |
| case ISD::AND: { |
| // Perform some heroic transforms on an and of a constant-count shift |
| // with a constant to enable use of the scaled offset field. |
| |
| // Scale must not be used already. |
| if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break; |
| |
| // We only handle up to 64-bit values here as those are what matter for |
| // addressing mode optimizations. |
| assert(N.getSimpleValueType().getSizeInBits() <= 64 && |
| "Unexpected value size!"); |
| |
| if (!isa<ConstantSDNode>(N.getOperand(1))) |
| break; |
| |
| if (N.getOperand(0).getOpcode() == ISD::SRL) { |
| SDValue Shift = N.getOperand(0); |
| SDValue X = Shift.getOperand(0); |
| |
| uint64_t Mask = N.getConstantOperandVal(1); |
| |
| // Try to fold the mask and shift into an extract and scale. |
| if (!foldMaskAndShiftToExtract(*CurDAG, N, Mask, Shift, X, AM)) |
| return false; |
| |
| // Try to fold the mask and shift directly into the scale. |
| if (!foldMaskAndShiftToScale(*CurDAG, N, Mask, Shift, X, AM)) |
| return false; |
| |
| // Try to fold the mask and shift into BEXTR and scale. |
| if (!foldMaskedShiftToBEXTR(*CurDAG, N, Mask, Shift, X, AM, *Subtarget)) |
| return false; |
| } |
| |
| // Try to swap the mask and shift to place shifts which can be done as |
| // a scale on the outside of the mask. |
| if (!foldMaskedShiftToScaledMask(*CurDAG, N, AM)) |
| return false; |
| |
| break; |
| } |
| case ISD::ZERO_EXTEND: { |
| // Try to widen a zexted shift left to the same size as its use, so we can |
| // match the shift as a scale factor. |
| if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) |
| break; |
| if (N.getOperand(0).getOpcode() != ISD::SHL || !N.getOperand(0).hasOneUse()) |
| break; |
| |
| // Give up if the shift is not a valid scale factor [1,2,3]. |
| SDValue Shl = N.getOperand(0); |
| auto *ShAmtC = dyn_cast<ConstantSDNode>(Shl.getOperand(1)); |
| if (!ShAmtC || ShAmtC->getZExtValue() > 3) |
| break; |
| |
| // The narrow shift must only shift out zero bits (it must be 'nuw'). |
| // That makes it safe to widen to the destination type. |
| APInt HighZeros = APInt::getHighBitsSet(Shl.getValueSizeInBits(), |
| ShAmtC->getZExtValue()); |
| if (!CurDAG->MaskedValueIsZero(Shl.getOperand(0), HighZeros)) |
| break; |
| |
| // zext (shl nuw i8 %x, C) to i32 --> shl (zext i8 %x to i32), (zext C) |
| MVT VT = N.getSimpleValueType(); |
| SDLoc DL(N); |
| SDValue Zext = CurDAG->getNode(ISD::ZERO_EXTEND, DL, VT, Shl.getOperand(0)); |
| SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, Shl.getOperand(1)); |
| |
| // Convert the shift to scale factor. |
| AM.Scale = 1 << ShAmtC->getZExtValue(); |
| AM.IndexReg = Zext; |
| |
| insertDAGNode(*CurDAG, N, Zext); |
| insertDAGNode(*CurDAG, N, NewShl); |
| CurDAG->ReplaceAllUsesWith(N, NewShl); |
| CurDAG->RemoveDeadNode(N.getNode()); |
| return false; |
| } |
| } |
| |
| return matchAddressBase(N, AM); |
| } |
| |
| /// Helper for MatchAddress. Add the specified node to the |
| /// specified addressing mode without any further recursion. |
| bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) { |
| // Is the base register already occupied? |
| if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { |
| // If so, check to see if the scale index register is set. |
| if (!AM.IndexReg.getNode()) { |
| AM.IndexReg = N; |
| AM.Scale = 1; |
| return false; |
| } |
| |
| // Otherwise, we cannot select it. |
| return true; |
| } |
| |
| // Default, generate it as a register. |
| AM.BaseType = X86ISelAddressMode::RegBase; |
| AM.Base_Reg = N; |
| return false; |
| } |
| |
| bool X86DAGToDAGISel::matchVectorAddressRecursively(SDValue N, |
| X86ISelAddressMode &AM, |
| unsigned Depth) { |
| SDLoc dl(N); |
| LLVM_DEBUG({ |
| dbgs() << "MatchVectorAddress: "; |
| AM.dump(CurDAG); |
| }); |
| // Limit recursion. |
| if (Depth > 5) |
| return matchAddressBase(N, AM); |
| |
| // TODO: Support other operations. |
| switch (N.getOpcode()) { |
| case ISD::Constant: { |
| uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); |
| if (!foldOffsetIntoAddress(Val, AM)) |
| return false; |
| break; |
| } |
| case X86ISD::Wrapper: |
| if (!matchWrapper(N, AM)) |
| return false; |
| break; |
| case ISD::ADD: { |
| // Add an artificial use to this node so that we can keep track of |
| // it if it gets CSE'd with a different node. |
| HandleSDNode Handle(N); |
| |
| X86ISelAddressMode Backup = AM; |
| if (!matchVectorAddressRecursively(N.getOperand(0), AM, Depth + 1) && |
| !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, |
| Depth + 1)) |
| return false; |
| AM = Backup; |
| |
| // Try again after commuting the operands. |
| if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM, |
| Depth + 1) && |
| !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM, |
| Depth + 1)) |
| return false; |
| AM = Backup; |
| |
| N = Handle.getValue(); |
| break; |
| } |
| } |
| |
| return matchAddressBase(N, AM); |
| } |
| |
| /// Helper for selectVectorAddr. Handles things that can be folded into a |
| /// gather/scatter address. The index register and scale should have already |
| /// been handled. |
| bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) { |
| return matchVectorAddressRecursively(N, AM, 0); |
| } |
| |
| bool X86DAGToDAGISel::selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, |
| SDValue IndexOp, SDValue ScaleOp, |
| SDValue &Base, SDValue &Scale, |
| SDValue &Index, SDValue &Disp, |
| SDValue &Segment) { |
| X86ISelAddressMode AM; |
| AM.IndexReg = IndexOp; |
| AM.Scale = cast<ConstantSDNode>(ScaleOp)->getZExtValue(); |
| |
| unsigned AddrSpace = Parent->getPointerInfo().getAddrSpace(); |
| if (AddrSpace == X86AS::GS) |
| AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); |
| if (AddrSpace == X86AS::FS) |
| AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); |
| if (AddrSpace == X86AS::SS) |
| AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16); |
| |
| SDLoc DL(BasePtr); |
| MVT VT = BasePtr.getSimpleValueType(); |
| |
| // Try to match into the base and displacement fields. |
| if (matchVectorAddress(BasePtr, AM)) |
| return false; |
| |
| getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment); |
| return true; |
| } |
| |
| /// Returns true if it is able to pattern match an addressing mode. |
| /// It returns the operands which make up the maximal addressing mode it can |
| /// match by reference. |
| /// |
| /// Parent is the parent node of the addr operand that is being matched. It |
| /// is always a load, store, atomic node, or null. It is only null when |
| /// checking memory operands for inline asm nodes. |
| bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, |
| SDValue &Scale, SDValue &Index, |
| SDValue &Disp, SDValue &Segment) { |
| X86ISelAddressMode AM; |
| |
| if (Parent && |
| // This list of opcodes are all the nodes that have an "addr:$ptr" operand |
| // that are not a MemSDNode, and thus don't have proper addrspace info. |
| Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme |
| Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores |
| Parent->getOpcode() != X86ISD::TLSCALL && // Fixme |
| Parent->getOpcode() != X86ISD::ENQCMD && // Fixme |
| Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme |
| Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp |
| Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp |
| unsigned AddrSpace = |
| cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); |
| if (AddrSpace == X86AS::GS) |
| AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); |
| if (AddrSpace == X86AS::FS) |
| AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); |
|