Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 1 | //===- SelectionDAGBuilder.h - Selection-DAG building -----------*- C++ -*-===// |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This implements routines for translating from LLVM IR into SelectionDAG IR. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
Benjamin Kramer | a7c40ef | 2014-08-13 16:26:38 +0000 | [diff] [blame] | 13 | #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_SELECTIONDAGBUILDER_H |
| 14 | #define LLVM_LIB_CODEGEN_SELECTIONDAG_SELECTIONDAGBUILDER_H |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 15 | |
Chandler Carruth | d990388 | 2015-01-14 11:23:27 +0000 | [diff] [blame] | 16 | #include "StatepointLowering.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 17 | #include "llvm/ADT/ArrayRef.h" |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 18 | #include "llvm/ADT/DenseMap.h" |
Yi Kong | 74b874a | 2019-03-26 12:18:08 +0000 | [diff] [blame] | 19 | #include "llvm/ADT/MapVector.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 20 | #include "llvm/ADT/SmallVector.h" |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 21 | #include "llvm/CodeGen/AssignmentTrackingAnalysis.h" |
Amara Emerson | cfef180 | 2021-09-25 22:55:37 -0700 | [diff] [blame] | 22 | #include "llvm/CodeGen/CodeGenCommonISel.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 23 | #include "llvm/CodeGen/ISDOpcodes.h" |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 24 | #include "llvm/CodeGen/SelectionDAGNodes.h" |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 25 | #include "llvm/CodeGen/SwitchLoweringUtils.h" |
David Blaikie | b3bde2e | 2017-11-17 01:07:10 +0000 | [diff] [blame] | 26 | #include "llvm/CodeGen/TargetLowering.h" |
Craig Topper | 2fa1436 | 2018-03-29 17:21:10 +0000 | [diff] [blame] | 27 | #include "llvm/CodeGen/ValueTypes.h" |
Nico Weber | 184ca39 | 2024-01-25 12:01:31 -0500 | [diff] [blame] | 28 | #include "llvm/CodeGenTypes/MachineValueType.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 29 | #include "llvm/IR/DebugLoc.h" |
| 30 | #include "llvm/IR/Instruction.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 31 | #include "llvm/Support/BranchProbability.h" |
| 32 | #include "llvm/Support/CodeGen.h" |
Torok Edwin | 56d0659 | 2009-07-11 20:10:48 +0000 | [diff] [blame] | 33 | #include "llvm/Support/ErrorHandling.h" |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 34 | #include <algorithm> |
| 35 | #include <cassert> |
| 36 | #include <cstdint> |
Krzysztof Parzyszek | 864aaa2 | 2022-12-01 16:18:32 -0800 | [diff] [blame] | 37 | #include <optional> |
Benjamin Kramer | 82de7d3 | 2016-05-27 14:27:24 +0000 | [diff] [blame] | 38 | #include <utility> |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 39 | #include <vector> |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 40 | |
| 41 | namespace llvm { |
| 42 | |
Simon Pilgrim | bee79cd | 2020-09-15 11:44:47 +0100 | [diff] [blame] | 43 | class AAResults; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 44 | class AllocaInst; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 45 | class AtomicCmpXchgInst; |
| 46 | class AtomicRMWInst; |
Matt Arsenault | bcb931c | 2022-09-19 15:25:37 -0400 | [diff] [blame] | 47 | class AssumptionCache; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 48 | class BasicBlock; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 49 | class BranchInst; |
| 50 | class CallInst; |
Craig Topper | 784929d | 2019-02-08 20:48:56 +0000 | [diff] [blame] | 51 | class CallBrInst; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 52 | class CatchPadInst; |
| 53 | class CatchReturnInst; |
| 54 | class CatchSwitchInst; |
| 55 | class CleanupPadInst; |
| 56 | class CleanupReturnInst; |
| 57 | class Constant; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 58 | class ConstrainedFPIntrinsic; |
Devang Patel | b12ff59 | 2010-08-26 23:35:15 +0000 | [diff] [blame] | 59 | class DbgValueInst; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 60 | class DataLayout; |
| 61 | class DIExpression; |
| 62 | class DILocalVariable; |
| 63 | class DILocation; |
| 64 | class FenceInst; |
Dan Gohman | a3624b6 | 2009-11-23 17:16:22 +0000 | [diff] [blame] | 65 | class FunctionLoweringInfo; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 66 | class GCFunctionInfo; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 67 | class GCRelocateInst; |
| 68 | class GCResultInst; |
Simon Pilgrim | bee79cd | 2020-09-15 11:44:47 +0100 | [diff] [blame] | 69 | class GCStatepointInst; |
Chris Lattner | d04cb6d | 2009-10-28 00:19:10 +0000 | [diff] [blame] | 70 | class IndirectBrInst; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 71 | class InvokeInst; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 72 | class LandingPadInst; |
| 73 | class LLVMContext; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 74 | class LoadInst; |
| 75 | class MachineBasicBlock; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 76 | class PHINode; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 77 | class ResumeInst; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 78 | class ReturnInst; |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 79 | class SDDbgValue; |
Simon Pilgrim | 46de0d5 | 2020-04-19 12:38:41 +0100 | [diff] [blame] | 80 | class SelectionDAG; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 81 | class StoreInst; |
Tim Northover | 3d7a057 | 2019-05-24 08:39:43 +0000 | [diff] [blame] | 82 | class SwiftErrorValueTracking; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 83 | class SwitchInst; |
Owen Anderson | bb15fec | 2011-12-08 22:15:21 +0000 | [diff] [blame] | 84 | class TargetLibraryInfo; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 85 | class TargetMachine; |
| 86 | class Type; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 87 | class VAArgInst; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 88 | class UnreachableInst; |
| 89 | class Use; |
| 90 | class User; |
| 91 | class Value; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 92 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 93 | //===----------------------------------------------------------------------===// |
Dan Gohman | 1a6c47f | 2009-11-23 18:04:58 +0000 | [diff] [blame] | 94 | /// SelectionDAGBuilder - This is the common target-independent lowering |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 95 | /// implementation that is parameterized by a TargetLowering object. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 96 | /// |
Benjamin Kramer | 079b96e | 2013-09-11 18:05:11 +0000 | [diff] [blame] | 97 | class SelectionDAGBuilder { |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 98 | /// The current instruction being visited. |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 99 | const Instruction *CurInst = nullptr; |
Dale Johannesen | db7c5f6 | 2009-01-31 02:22:37 +0000 | [diff] [blame] | 100 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 101 | DenseMap<const Value*, SDValue> NodeMap; |
Andrew Trick | d4d1d9c | 2013-10-31 17:18:07 +0000 | [diff] [blame] | 102 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 103 | /// Maps argument value for unused arguments. This is used |
Devang Patel | b0c7639 | 2010-06-01 19:59:01 +0000 | [diff] [blame] | 104 | /// to preserve debug information for incoming arguments. |
| 105 | DenseMap<const Value*, SDValue> UnusedArgNodeMap; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 106 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 107 | /// Helper type for DanglingDebugInfoMap. |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 108 | class DanglingDebugInfo { |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 109 | unsigned SDNodeOrder = 0; |
| 110 | |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 111 | public: |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 112 | DILocalVariable *Variable; |
| 113 | DIExpression *Expression; |
| 114 | DebugLoc dl; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 115 | DanglingDebugInfo() = default; |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 116 | DanglingDebugInfo(DILocalVariable *Var, DIExpression *Expr, DebugLoc DL, |
| 117 | unsigned SDNO) |
| 118 | : SDNodeOrder(SDNO), Variable(Var), Expression(Expr), |
| 119 | dl(std::move(DL)) {} |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 120 | |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 121 | DILocalVariable *getVariable() const { return Variable; } |
| 122 | DIExpression *getExpression() const { return Expression; } |
| 123 | DebugLoc getDebugLoc() const { return dl; } |
OCHyams | a1ac6ef | 2022-11-16 10:10:24 +0000 | [diff] [blame] | 124 | unsigned getSDNodeOrder() const { return SDNodeOrder; } |
| 125 | |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 126 | /// Helper for printing DanglingDebugInfo. This hoop-jumping is to |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 127 | /// store a Value pointer, so that we can print a whole DDI as one object. |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 128 | /// Call SelectionDAGBuilder::printDDI instead of using directly. |
| 129 | struct Print { |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 130 | Print(const Value *V, const DanglingDebugInfo &DDI) : V(V), DDI(DDI) {} |
| 131 | const Value *V; |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 132 | const DanglingDebugInfo &DDI; |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 133 | friend raw_ostream &operator<<(raw_ostream &OS, |
| 134 | const DanglingDebugInfo::Print &P) { |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 135 | OS << "DDI(var=" << *P.DDI.getVariable(); |
| 136 | if (P.V) |
| 137 | OS << ", val=" << *P.V; |
| 138 | else |
| 139 | OS << ", val=nullptr"; |
| 140 | |
| 141 | OS << ", expr=" << *P.DDI.getExpression() |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 142 | << ", order=" << P.DDI.getSDNodeOrder() |
| 143 | << ", loc=" << P.DDI.getDebugLoc() << ")"; |
| 144 | return OS; |
| 145 | } |
| 146 | }; |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 147 | }; |
| 148 | |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 149 | /// Returns an object that defines `raw_ostream &operator<<` for printing. |
| 150 | /// Usage example: |
| 151 | //// errs() << printDDI(MyDanglingInfo) << " is dangling\n"; |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 152 | DanglingDebugInfo::Print printDDI(const Value *V, |
| 153 | const DanglingDebugInfo &DDI) { |
| 154 | return DanglingDebugInfo::Print(V, DDI); |
OCHyams | 1d1de74 | 2022-12-09 15:43:56 +0000 | [diff] [blame] | 155 | } |
| 156 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 157 | /// Helper type for DanglingDebugInfoMap. |
Bjorn Pettersson | 5c25f88 | 2018-03-21 09:44:34 +0000 | [diff] [blame] | 158 | typedef std::vector<DanglingDebugInfo> DanglingDebugInfoVector; |
| 159 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 160 | /// Keeps track of dbg_values for which we have not yet seen the referent. |
| 161 | /// We defer handling these until we do see it. |
Yi Kong | 74b874a | 2019-03-26 12:18:08 +0000 | [diff] [blame] | 162 | MapVector<const Value*, DanglingDebugInfoVector> DanglingDebugInfoMap; |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 163 | |
OCHyams | 06f28f2 | 2023-03-29 12:51:59 +0100 | [diff] [blame] | 164 | /// Cache the module flag for whether we should use debug-info assignment |
| 165 | /// tracking. |
| 166 | bool AssignmentTrackingEnabled = false; |
| 167 | |
Chris Lattner | 1a32ede | 2009-12-24 00:37:38 +0000 | [diff] [blame] | 168 | public: |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 169 | /// Loads are not emitted to the program immediately. We bunch them up and |
| 170 | /// then emit token factor nodes when possible. This allows us to get simple |
| 171 | /// disambiguation between loads without worrying about alias analysis. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 172 | SmallVector<SDValue, 8> PendingLoads; |
Philip Reames | 1a1bdb2 | 2014-12-02 18:50:36 +0000 | [diff] [blame] | 173 | |
| 174 | /// State used while lowering a statepoint sequence (gc_statepoint, |
| 175 | /// gc_relocate, and gc_result). See StatepointLowering.hpp/cpp for details. |
| 176 | StatepointLoweringState StatepointLowering; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 177 | |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 178 | private: |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 179 | /// CopyToReg nodes that copy values to virtual registers for export to other |
| 180 | /// blocks need to be emitted before any terminator instruction, but they have |
| 181 | /// no other ordering requirements. We bunch them up and the emit a single |
| 182 | /// tokenfactor for them just before terminator instructions. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 183 | SmallVector<SDValue, 8> PendingExports; |
| 184 | |
Ulrich Weigand | 04a8696 | 2020-01-13 14:37:07 +0100 | [diff] [blame] | 185 | /// Similar to loads, nodes corresponding to constrained FP intrinsics are |
| 186 | /// bunched up and emitted when necessary. These can be moved across each |
| 187 | /// other and any (normal) memory operation (load or store), but not across |
| 188 | /// calls or instructions having unspecified side effects. As a special |
| 189 | /// case, constrained FP intrinsics using fpexcept.strict may not be deleted |
| 190 | /// even if otherwise unused, so they need to be chained before any |
| 191 | /// terminator instruction (like PendingExports). We track the latter |
| 192 | /// set of nodes in a separate list. |
| 193 | SmallVector<SDValue, 8> PendingConstrainedFP; |
| 194 | SmallVector<SDValue, 8> PendingConstrainedFPStrict; |
| 195 | |
Ulrich Weigand | 81ee484 | 2020-01-14 14:04:53 +0100 | [diff] [blame] | 196 | /// Update root to include all chains from the Pending list. |
| 197 | SDValue updateRoot(SmallVectorImpl<SDValue> &Pending); |
| 198 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 199 | /// A unique monotonically increasing number used to order the SDNodes we |
| 200 | /// create. |
Bill Wendling | 022d18f | 2009-12-18 23:32:53 +0000 | [diff] [blame] | 201 | unsigned SDNodeOrder; |
| 202 | |
Hans Wennborg | 0867b15 | 2015-04-23 16:45:24 +0000 | [diff] [blame] | 203 | /// Emit comparison and split W into two subtrees. |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 204 | void splitWorkItem(SwitchCG::SwitchWorkList &WorkList, |
| 205 | const SwitchCG::SwitchWorkListItem &W, Value *Cond, |
| 206 | MachineBasicBlock *SwitchMBB); |
Hans Wennborg | 0867b15 | 2015-04-23 16:45:24 +0000 | [diff] [blame] | 207 | |
| 208 | /// Lower W. |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 209 | void lowerWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond, |
Hans Wennborg | 0867b15 | 2015-04-23 16:45:24 +0000 | [diff] [blame] | 210 | MachineBasicBlock *SwitchMBB, |
| 211 | MachineBasicBlock *DefaultMBB); |
| 212 | |
Rong Xu | 3573d8d | 2017-11-14 21:44:09 +0000 | [diff] [blame] | 213 | /// Peel the top probability case if it exceeds the threshold |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 214 | MachineBasicBlock * |
| 215 | peelDominantCaseCluster(const SwitchInst &SI, |
| 216 | SwitchCG::CaseClusterVector &Clusters, |
| 217 | BranchProbability &PeeledCaseProb); |
Hans Wennborg | 0867b15 | 2015-04-23 16:45:24 +0000 | [diff] [blame] | 218 | |
Bill Wendling | a3cd350 | 2013-06-19 21:36:55 +0000 | [diff] [blame] | 219 | private: |
Dan Gohman | c334960 | 2010-04-19 19:05:59 +0000 | [diff] [blame] | 220 | const TargetMachine &TM; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 221 | |
Bill Wendling | a3cd350 | 2013-06-19 21:36:55 +0000 | [diff] [blame] | 222 | public: |
Nico Rieck | b5262d6 | 2014-01-12 14:09:17 +0000 | [diff] [blame] | 223 | /// Lowest valid SDNodeOrder. The special case 0 is reserved for scheduling |
| 224 | /// nodes without a corresponding SDNode. |
| 225 | static const unsigned LowestSDNodeOrder = 1; |
| 226 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 227 | SelectionDAG &DAG; |
Benjamin Maxwell | 7781381 | 2025-01-23 09:16:09 +0000 | [diff] [blame] | 228 | BatchAAResults *BatchAA = nullptr; |
Matt Arsenault | bcb931c | 2022-09-19 15:25:37 -0400 | [diff] [blame] | 229 | AssumptionCache *AC = nullptr; |
Akshay Khadse | 8bf7f86 | 2023-04-17 16:16:23 +0800 | [diff] [blame] | 230 | const TargetLibraryInfo *LibInfo = nullptr; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 231 | |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 232 | class SDAGSwitchLowering : public SwitchCG::SwitchLowering { |
| 233 | public: |
| 234 | SDAGSwitchLowering(SelectionDAGBuilder *sdb, FunctionLoweringInfo &funcinfo) |
| 235 | : SwitchCG::SwitchLowering(funcinfo), SDB(sdb) {} |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 236 | |
Kazu Hirata | 3f3930a | 2022-07-25 23:00:59 -0700 | [diff] [blame] | 237 | void addSuccessorWithProb( |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 238 | MachineBasicBlock *Src, MachineBasicBlock *Dst, |
| 239 | BranchProbability Prob = BranchProbability::getUnknown()) override { |
| 240 | SDB->addSuccessorWithProb(Src, Dst, Prob); |
| 241 | } |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 242 | |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 243 | private: |
Akshay Khadse | 8bf7f86 | 2023-04-17 16:16:23 +0800 | [diff] [blame] | 244 | SelectionDAGBuilder *SDB = nullptr; |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 245 | }; |
| 246 | |
James Y Knight | e953679 | 2020-05-12 00:22:54 -0400 | [diff] [blame] | 247 | // Data related to deferred switch lowerings. Used to construct additional |
| 248 | // Basic Blocks in SelectionDAGISel::FinishBasicBlock. |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 249 | std::unique_ptr<SDAGSwitchLowering> SL; |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 250 | |
Michael Gottesman | b27f0f1 | 2013-08-20 07:00:16 +0000 | [diff] [blame] | 251 | /// A StackProtectorDescriptor structure used to communicate stack protector |
| 252 | /// information in between SelectBasicBlock and FinishBasicBlock. |
| 253 | StackProtectorDescriptor SPDescriptor; |
Evan Cheng | 270d0f9 | 2009-09-18 21:02:19 +0000 | [diff] [blame] | 254 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 255 | // Emit PHI-node-operand constants only once even if used by multiple |
| 256 | // PHI nodes. |
Craig Topper | 7bd2be4 | 2025-03-02 16:16:19 -0800 | [diff] [blame] | 257 | DenseMap<const Constant *, Register> ConstantsOut; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 258 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 259 | /// Information about the function as a whole. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 260 | FunctionLoweringInfo &FuncInfo; |
Bill Wendling | 19e0a5b | 2009-02-19 21:12:54 +0000 | [diff] [blame] | 261 | |
Tim Northover | 3d7a057 | 2019-05-24 08:39:43 +0000 | [diff] [blame] | 262 | /// Information about the swifterror values used throughout the function. |
| 263 | SwiftErrorValueTracking &SwiftError; |
| 264 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 265 | /// Garbage collection metadata for the function. |
Akshay Khadse | 8bf7f86 | 2023-04-17 16:16:23 +0800 | [diff] [blame] | 266 | GCFunctionInfo *GFI = nullptr; |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 267 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 268 | /// Map a landing pad to the call site indexes. |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 269 | DenseMap<MachineBasicBlock *, SmallVector<unsigned, 4>> LPadToCallSiteMap; |
Bill Wendling | 3d11aa7 | 2011-10-04 22:00:35 +0000 | [diff] [blame] | 270 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 271 | /// This is set to true if a call in the current block has been translated as |
| 272 | /// a tail call. In this case, no subsequent DAG nodes should be created. |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 273 | bool HasTailCall = false; |
Dan Gohman | f9bbcd1 | 2009-08-05 01:29:28 +0000 | [diff] [blame] | 274 | |
Akshay Khadse | 8bf7f86 | 2023-04-17 16:16:23 +0800 | [diff] [blame] | 275 | LLVMContext *Context = nullptr; |
Owen Anderson | 53a5221 | 2009-07-13 04:09:18 +0000 | [diff] [blame] | 276 | |
Dan Gohman | c334960 | 2010-04-19 19:05:59 +0000 | [diff] [blame] | 277 | SelectionDAGBuilder(SelectionDAG &dag, FunctionLoweringInfo &funcinfo, |
Arthur Eubanks | 0a1aa6c | 2023-09-14 14:10:14 -0700 | [diff] [blame] | 278 | SwiftErrorValueTracking &swifterror, CodeGenOptLevel ol) |
Tim Northover | 3d7a057 | 2019-05-24 08:39:43 +0000 | [diff] [blame] | 279 | : SDNodeOrder(LowestSDNodeOrder), TM(dag.getTarget()), DAG(dag), |
Arthur Eubanks | 0a1aa6c | 2023-09-14 14:10:14 -0700 | [diff] [blame] | 280 | SL(std::make_unique<SDAGSwitchLowering>(this, funcinfo)), |
| 281 | FuncInfo(funcinfo), SwiftError(swifterror) {} |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 282 | |
Benjamin Maxwell | 7781381 | 2025-01-23 09:16:09 +0000 | [diff] [blame] | 283 | void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, |
Owen Anderson | bb15fec | 2011-12-08 22:15:21 +0000 | [diff] [blame] | 284 | const TargetLibraryInfo *li); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 285 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 286 | /// Clear out the current SelectionDAG and the associated state and prepare |
| 287 | /// this SelectionDAGBuilder object to be used for a new block. This doesn't |
| 288 | /// clear out information about additional blocks that are needed to complete |
| 289 | /// switch lowering or PHI node updating; that information is cleared out as |
| 290 | /// it is consumed. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 291 | void clear(); |
| 292 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 293 | /// Clear the dangling debug information map. This function is separated from |
| 294 | /// the clear so that debug information that is dangling in a basic block can |
| 295 | /// be properly resolved in a different basic block. This allows the |
| 296 | /// SelectionDAG to resolve dangling debug information attached to PHI nodes. |
Devang Patel | 79928838 | 2011-05-23 17:44:13 +0000 | [diff] [blame] | 297 | void clearDanglingDebugInfo(); |
| 298 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 299 | /// Return the current virtual root of the Selection DAG, flushing any |
| 300 | /// PendingLoad items. This must be done before emitting a store or any other |
Ulrich Weigand | 04a8696 | 2020-01-13 14:37:07 +0100 | [diff] [blame] | 301 | /// memory node that may need to be ordered after any prior load instructions. |
| 302 | SDValue getMemoryRoot(); |
| 303 | |
| 304 | /// Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) |
| 305 | /// items. This must be done before emitting any call other any other node |
| 306 | /// that may need to be ordered after FP instructions due to other side |
| 307 | /// effects. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 308 | SDValue getRoot(); |
| 309 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 310 | /// Similar to getRoot, but instead of flushing all the PendingLoad items, |
Ulrich Weigand | 04a8696 | 2020-01-13 14:37:07 +0100 | [diff] [blame] | 311 | /// flush all the PendingExports (and PendingConstrainedFPStrict) items. |
| 312 | /// It is necessary to do this before emitting a terminator instruction. |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 313 | SDValue getControlRoot(); |
| 314 | |
Andrew Trick | 175143b | 2013-05-25 02:20:36 +0000 | [diff] [blame] | 315 | SDLoc getCurSDLoc() const { |
Andrew Trick | 175143b | 2013-05-25 02:20:36 +0000 | [diff] [blame] | 316 | return SDLoc(CurInst, SDNodeOrder); |
| 317 | } |
| 318 | |
| 319 | DebugLoc getCurDebugLoc() const { |
| 320 | return CurInst ? CurInst->getDebugLoc() : DebugLoc(); |
| 321 | } |
Devang Patel | f3292b2 | 2011-02-21 23:21:26 +0000 | [diff] [blame] | 322 | |
Craig Topper | 7bd2be4 | 2025-03-02 16:16:19 -0800 | [diff] [blame] | 323 | void CopyValueToVirtualRegister(const Value *V, Register Reg, |
Craig Topper | 85eae45 | 2022-03-30 11:24:01 -0700 | [diff] [blame] | 324 | ISD::NodeType ExtendType = ISD::ANY_EXTEND); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 325 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 326 | void visit(const Instruction &I); |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 327 | void visitDbgInfo(const Instruction &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 328 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 329 | void visit(unsigned Opcode, const User &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 330 | |
Bjorn Pettersson | 350352c | 2019-02-06 17:36:18 +0000 | [diff] [blame] | 331 | /// If there was virtual register allocated for the value V emit CopyFromReg |
| 332 | /// of the specified type Ty. Return empty SDValue() otherwise. |
Igor Laevsky | 85f7f72 | 2015-03-10 16:26:48 +0000 | [diff] [blame] | 333 | SDValue getCopyFromRegs(const Value *V, Type *Ty); |
| 334 | |
gbtozers | 93b170e | 2020-09-29 15:43:21 +0100 | [diff] [blame] | 335 | /// Register a dbg_value which relies on a Value which we have not yet seen. |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 336 | void addDanglingDebugInfo(SmallVectorImpl<Value *> &Values, |
| 337 | DILocalVariable *Var, DIExpression *Expr, |
| 338 | bool IsVariadic, DebugLoc DL, unsigned Order); |
gbtozers | 93b170e | 2020-09-29 15:43:21 +0100 | [diff] [blame] | 339 | |
Bjorn Pettersson | a223f815 | 2018-03-12 18:02:39 +0000 | [diff] [blame] | 340 | /// If we have dangling debug info that describes \p Variable, or an |
| 341 | /// overlapping part of variable considering the \p Expr, then this method |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 342 | /// will drop that debug info as it isn't valid any longer. |
Bjorn Pettersson | a223f815 | 2018-03-12 18:02:39 +0000 | [diff] [blame] | 343 | void dropDanglingDebugInfo(const DILocalVariable *Variable, |
| 344 | const DIExpression *Expr); |
| 345 | |
Jeremy Morse | 291713a | 2019-02-13 16:33:05 +0000 | [diff] [blame] | 346 | /// If we saw an earlier dbg_value referring to V, generate the debug data |
| 347 | /// structures now that we've seen its definition. |
Dale Johannesen | bfd4fd7 | 2010-07-16 00:02:08 +0000 | [diff] [blame] | 348 | void resolveDanglingDebugInfo(const Value *V, SDValue Val); |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 349 | |
Jeremy Morse | 291713a | 2019-02-13 16:33:05 +0000 | [diff] [blame] | 350 | /// For the given dangling debuginfo record, perform last-ditch efforts to |
| 351 | /// resolve the debuginfo to something that is represented in this DAG. If |
| 352 | /// this cannot be done, produce an Undef debug value record. |
Jeremy Morse | 4495485 | 2023-11-21 17:10:25 +0000 | [diff] [blame] | 353 | void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI); |
Jeremy Morse | 291713a | 2019-02-13 16:33:05 +0000 | [diff] [blame] | 354 | |
gbtozers | 93b170e | 2020-09-29 15:43:21 +0100 | [diff] [blame] | 355 | /// For a given list of Values, attempt to create and record a SDDbgValue in |
| 356 | /// the SelectionDAG. |
| 357 | bool handleDebugValue(ArrayRef<const Value *> Values, DILocalVariable *Var, |
OCHyams | 9792744 | 2022-11-16 09:59:35 +0000 | [diff] [blame] | 358 | DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, |
| 359 | bool IsVariadic); |
Jeremy Morse | 6d3cd3b | 2019-02-13 15:53:10 +0000 | [diff] [blame] | 360 | |
OCHyams | 2b3c13b | 2023-04-25 17:19:44 +0100 | [diff] [blame] | 361 | /// Create a record for a kill location debug intrinsic. |
| 362 | void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, |
| 363 | DebugLoc DbgLoc, unsigned Order); |
| 364 | |
Orlando Cazalet-Hyams | 5ee0881 | 2023-12-12 11:32:19 +0000 | [diff] [blame] | 365 | void handleDebugDeclare(Value *Address, DILocalVariable *Variable, |
| 366 | DIExpression *Expression, DebugLoc DL); |
| 367 | |
Jeremy Morse | 291713a | 2019-02-13 16:33:05 +0000 | [diff] [blame] | 368 | /// Evict any dangling debug information, attempting to salvage it first. |
| 369 | void resolveOrClearDbgInfo(); |
| 370 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 371 | SDValue getValue(const Value *V); |
Elena Demikhovsky | 584ce37 | 2015-04-28 07:57:37 +0000 | [diff] [blame] | 372 | |
Dan Gohman | d432223 | 2010-07-01 01:59:43 +0000 | [diff] [blame] | 373 | SDValue getNonRegisterValue(const Value *V); |
| 374 | SDValue getValueImpl(const Value *V); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 375 | |
| 376 | void setValue(const Value *V, SDValue NewN) { |
| 377 | SDValue &N = NodeMap[V]; |
Craig Topper | ada0857 | 2014-04-16 04:21:27 +0000 | [diff] [blame] | 378 | assert(!N.getNode() && "Already set a value for this node!"); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 379 | N = NewN; |
| 380 | } |
Andrew Trick | d4d1d9c | 2013-10-31 17:18:07 +0000 | [diff] [blame] | 381 | |
Devang Patel | b0c7639 | 2010-06-01 19:59:01 +0000 | [diff] [blame] | 382 | void setUnusedArgValue(const Value *V, SDValue NewN) { |
| 383 | SDValue &N = UnusedArgNodeMap[V]; |
Craig Topper | ada0857 | 2014-04-16 04:21:27 +0000 | [diff] [blame] | 384 | assert(!N.getNode() && "Already set a value for this node!"); |
Devang Patel | b0c7639 | 2010-06-01 19:59:01 +0000 | [diff] [blame] | 385 | N = NewN; |
| 386 | } |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 387 | |
Noah Goldstein | a4951ec | 2023-09-29 14:27:36 -0500 | [diff] [blame] | 388 | bool shouldKeepJumpConditionsTogether( |
| 389 | const FunctionLoweringInfo &FuncInfo, const BranchInst &I, |
| 390 | Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, |
| 391 | TargetLoweringBase::CondMergingParams Params) const; |
| 392 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 393 | void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 394 | MachineBasicBlock *FBB, MachineBasicBlock *CurBB, |
Pete Cooper | 6923461 | 2015-07-15 01:31:26 +0000 | [diff] [blame] | 395 | MachineBasicBlock *SwitchBB, |
Fangrui Song | cb0bab8 | 2018-07-16 18:51:40 +0000 | [diff] [blame] | 396 | Instruction::BinaryOps Opc, BranchProbability TProb, |
| 397 | BranchProbability FProb, bool InvertCond); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 398 | void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, |
Dan Gohman | d01ddb5 | 2008-10-17 21:16:08 +0000 | [diff] [blame] | 399 | MachineBasicBlock *FBB, |
Dan Gohman | 7c0303a | 2010-04-19 22:41:47 +0000 | [diff] [blame] | 400 | MachineBasicBlock *CurBB, |
Manman Ren | 4ece745 | 2014-01-31 00:42:44 +0000 | [diff] [blame] | 401 | MachineBasicBlock *SwitchBB, |
Fangrui Song | cb0bab8 | 2018-07-16 18:51:40 +0000 | [diff] [blame] | 402 | BranchProbability TProb, BranchProbability FProb, |
Geoff Berry | 92a286a | 2017-01-24 16:36:07 +0000 | [diff] [blame] | 403 | bool InvertCond); |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 404 | bool ShouldEmitAsBranches(const std::vector<SwitchCG::CaseBlock> &Cases); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 405 | bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB); |
| 406 | void CopyToExportRegsIfNeeded(const Value *V); |
| 407 | void ExportFromCurrentBlock(const Value *V); |
Craig Topper | 113f37a | 2020-04-13 10:44:59 -0700 | [diff] [blame] | 408 | void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, |
Ahmed Bougacha | cc548ec | 2024-05-31 14:08:10 -0700 | [diff] [blame] | 409 | bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr, |
| 410 | const TargetLowering::PtrAuthInfo *PAI = nullptr); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 411 | |
Matt Arsenault | 2bba779 | 2016-02-08 16:28:19 +0000 | [diff] [blame] | 412 | // Lower range metadata from 0 to N to assert zext to an integer of nearest |
| 413 | // floor power of two. |
| 414 | SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, |
| 415 | SDValue Op); |
| 416 | |
Sanjoy Das | 19c6159 | 2016-03-16 20:49:31 +0000 | [diff] [blame] | 417 | void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, |
Chandler Carruth | 3160734 | 2019-02-11 07:42:30 +0000 | [diff] [blame] | 418 | const CallBase *Call, unsigned ArgIdx, |
Sanjoy Das | 19c6159 | 2016-03-16 20:49:31 +0000 | [diff] [blame] | 419 | unsigned NumArgs, SDValue Callee, |
Markus Böck | 0ad92c0 | 2023-10-14 18:38:18 +0200 | [diff] [blame] | 420 | Type *ReturnTy, AttributeSet RetAttrs, |
| 421 | bool IsPatchPoint); |
Sanjoy Das | 19c6159 | 2016-03-16 20:49:31 +0000 | [diff] [blame] | 422 | |
| 423 | std::pair<SDValue, SDValue> |
| 424 | lowerInvokable(TargetLowering::CallLoweringInfo &CLI, |
| 425 | const BasicBlock *EHPadBB = nullptr); |
Andrew Trick | 74f4c74 | 2013-10-31 17:18:24 +0000 | [diff] [blame] | 426 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 427 | /// When an MBB was split during scheduling, update the |
Alp Toker | 798060e | 2014-01-11 14:01:43 +0000 | [diff] [blame] | 428 | /// references that need to refer to the last resulting block. |
Jakob Stoklund Olesen | 665aa6e | 2010-09-30 19:44:31 +0000 | [diff] [blame] | 429 | void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); |
| 430 | |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 431 | /// Describes a gc.statepoint or a gc.statepoint like thing for the purposes |
Sanjoy Das | a5b2972 | 2016-03-23 02:28:35 +0000 | [diff] [blame] | 432 | /// of lowering into a STATEPOINT node. |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 433 | struct StatepointLoweringInfo { |
| 434 | /// Bases[i] is the base pointer for Ptrs[i]. Together they denote the set |
| 435 | /// of gc pointers this STATEPOINT has to relocate. |
Sanjoy Das | e58ca59 | 2016-03-23 02:24:07 +0000 | [diff] [blame] | 436 | SmallVector<const Value *, 16> Bases; |
| 437 | SmallVector<const Value *, 16> Ptrs; |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 438 | |
| 439 | /// The set of gc.relocate calls associated with this gc.statepoint. |
Sanjoy Das | e58ca59 | 2016-03-23 02:24:07 +0000 | [diff] [blame] | 440 | SmallVector<const GCRelocateInst *, 16> GCRelocates; |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 441 | |
Philip Reames | 824cffe | 2024-08-26 14:05:17 -0700 | [diff] [blame] | 442 | /// The full list of gc-live arguments to the gc.statepoint being lowered. |
| 443 | ArrayRef<const Use> GCLives; |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 444 | |
| 445 | /// The gc.statepoint instruction. |
| 446 | const Instruction *StatepointInstr = nullptr; |
| 447 | |
| 448 | /// The list of gc transition arguments present in the gc.statepoint being |
| 449 | /// lowered. |
| 450 | ArrayRef<const Use> GCTransitionArgs; |
| 451 | |
| 452 | /// The ID that the resulting STATEPOINT instruction has to report. |
Il-Capitano | 0ef7437 | 2024-04-02 15:28:19 +0200 | [diff] [blame] | 453 | uint64_t ID = -1; |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 454 | |
| 455 | /// Information regarding the underlying call instruction. |
| 456 | TargetLowering::CallLoweringInfo CLI; |
| 457 | |
| 458 | /// The deoptimization state associated with this gc.statepoint call, if |
| 459 | /// any. |
| 460 | ArrayRef<const Use> DeoptState; |
| 461 | |
| 462 | /// Flags associated with the meta arguments being lowered. |
| 463 | uint64_t StatepointFlags = -1; |
| 464 | |
| 465 | /// The number of patchable bytes the call needs to get lowered into. |
| 466 | unsigned NumPatchBytes = -1; |
| 467 | |
| 468 | /// The exception handling unwind destination, in case this represents an |
| 469 | /// invoke of gc.statepoint. |
| 470 | const BasicBlock *EHPadBB = nullptr; |
| 471 | |
| 472 | explicit StatepointLoweringInfo(SelectionDAG &DAG) : CLI(DAG) {} |
| 473 | }; |
| 474 | |
| 475 | /// Lower \p SLI into a STATEPOINT instruction. |
Fangrui Song | cb0bab8 | 2018-07-16 18:51:40 +0000 | [diff] [blame] | 476 | SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI); |
Sanjoy Das | 70697ff | 2016-03-16 23:08:00 +0000 | [diff] [blame] | 477 | |
Igor Laevsky | 7fc58a4 | 2015-02-20 15:28:35 +0000 | [diff] [blame] | 478 | // This function is responsible for the whole statepoint lowering process. |
Igor Laevsky | 85f7f72 | 2015-03-10 16:26:48 +0000 | [diff] [blame] | 479 | // It uniformly handles invoke and call statepoints. |
Philip Reames | 98a87c6 | 2020-05-27 18:56:50 -0700 | [diff] [blame] | 480 | void LowerStatepoint(const GCStatepointInst &I, |
Reid Kleckner | 51189f0a | 2015-09-08 23:28:38 +0000 | [diff] [blame] | 481 | const BasicBlock *EHPadBB = nullptr); |
Sanjoy Das | 38bfc22 | 2016-03-22 00:59:13 +0000 | [diff] [blame] | 482 | |
Chandler Carruth | 3160734 | 2019-02-11 07:42:30 +0000 | [diff] [blame] | 483 | void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, |
Sanjoy Das | 38bfc22 | 2016-03-22 00:59:13 +0000 | [diff] [blame] | 484 | const BasicBlock *EHPadBB); |
| 485 | |
Sanjoy Das | df9ae70 | 2016-03-24 20:23:29 +0000 | [diff] [blame] | 486 | void LowerDeoptimizeCall(const CallInst *CI); |
Sanjoy Das | 65a6067 | 2016-04-06 01:33:49 +0000 | [diff] [blame] | 487 | void LowerDeoptimizingReturn(); |
Sanjoy Das | df9ae70 | 2016-03-24 20:23:29 +0000 | [diff] [blame] | 488 | |
Chandler Carruth | 3160734 | 2019-02-11 07:42:30 +0000 | [diff] [blame] | 489 | void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, |
Sanjoy Das | fd3eaa8 | 2016-03-24 22:51:49 +0000 | [diff] [blame] | 490 | const BasicBlock *EHPadBB, |
Sanjoy Das | 65a6067 | 2016-04-06 01:33:49 +0000 | [diff] [blame] | 491 | bool VarArgDisallowed, |
| 492 | bool ForceVoidReturnTy); |
Sanjoy Das | fd3eaa8 | 2016-03-24 22:51:49 +0000 | [diff] [blame] | 493 | |
Ahmed Bougacha | cc548ec | 2024-05-31 14:08:10 -0700 | [diff] [blame] | 494 | void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, |
| 495 | const BasicBlock *EHPadBB); |
| 496 | |
Sanjoy Das | 40c32dd | 2017-04-27 17:17:16 +0000 | [diff] [blame] | 497 | /// Returns the type of FrameIndex and TargetFrameIndex nodes. |
| 498 | MVT getFrameIndexTy() { |
| 499 | return DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout()); |
| 500 | } |
| 501 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 502 | private: |
| 503 | // Terminator instructions. |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 504 | void visitRet(const ReturnInst &I); |
| 505 | void visitBr(const BranchInst &I); |
| 506 | void visitSwitch(const SwitchInst &I); |
| 507 | void visitIndirectBr(const IndirectBrInst &I); |
Yaron Keren | d7ba46b | 2014-04-19 13:47:43 +0000 | [diff] [blame] | 508 | void visitUnreachable(const UnreachableInst &I); |
David Majnemer | 654e130 | 2015-07-31 17:58:14 +0000 | [diff] [blame] | 509 | void visitCleanupRet(const CleanupReturnInst &I); |
David Majnemer | 8a1c45d | 2015-12-12 05:38:55 +0000 | [diff] [blame] | 510 | void visitCatchSwitch(const CatchSwitchInst &I); |
David Majnemer | 654e130 | 2015-07-31 17:58:14 +0000 | [diff] [blame] | 511 | void visitCatchRet(const CatchReturnInst &I); |
| 512 | void visitCatchPad(const CatchPadInst &I); |
David Majnemer | 654e130 | 2015-07-31 17:58:14 +0000 | [diff] [blame] | 513 | void visitCleanupPad(const CleanupPadInst &CPI); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 514 | |
Cong Hou | 1938f2e | 2015-11-24 08:51:23 +0000 | [diff] [blame] | 515 | BranchProbability getEdgeProbability(const MachineBasicBlock *Src, |
| 516 | const MachineBasicBlock *Dst) const; |
| 517 | void addSuccessorWithProb( |
| 518 | MachineBasicBlock *Src, MachineBasicBlock *Dst, |
| 519 | BranchProbability Prob = BranchProbability::getUnknown()); |
| 520 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 521 | public: |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 522 | void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB); |
Michael Gottesman | b27f0f1 | 2013-08-20 07:00:16 +0000 | [diff] [blame] | 523 | void visitSPDescriptorParent(StackProtectorDescriptor &SPD, |
| 524 | MachineBasicBlock *ParentBB); |
| 525 | void visitSPDescriptorFailure(StackProtectorDescriptor &SPD); |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 526 | void visitBitTestHeader(SwitchCG::BitTestBlock &B, |
| 527 | MachineBasicBlock *SwitchBB); |
| 528 | void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, |
Craig Topper | 292ee93 | 2024-09-18 09:43:21 -0700 | [diff] [blame] | 529 | BranchProbability BranchProbToNext, Register Reg, |
Amara Emerson | 829037a | 2019-06-08 00:05:17 +0000 | [diff] [blame] | 530 | SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB); |
| 531 | void visitJumpTable(SwitchCG::JumpTable &JT); |
| 532 | void visitJumpTableHeader(SwitchCG::JumpTable &JT, |
| 533 | SwitchCG::JumpTableHeader &JTH, |
Dan Gohman | 7c0303a | 2010-04-19 22:41:47 +0000 | [diff] [blame] | 534 | MachineBasicBlock *SwitchBB); |
Andrew Trick | d4d1d9c | 2013-10-31 17:18:07 +0000 | [diff] [blame] | 535 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 536 | private: |
| 537 | // These all get lowered before this pass. |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 538 | void visitInvoke(const InvokeInst &I); |
Craig Topper | 784929d | 2019-02-08 20:48:56 +0000 | [diff] [blame] | 539 | void visitCallBr(const CallBrInst &I); |
Nick Desaulniers | 5cc1016 | 2023-02-16 17:47:37 -0800 | [diff] [blame] | 540 | void visitCallBrLandingPad(const CallInst &I); |
Bill Wendling | f891bf8 | 2011-07-31 06:30:59 +0000 | [diff] [blame] | 541 | void visitResume(const ResumeInst &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 542 | |
Cameron McInally | cbde0d9 | 2018-11-13 18:15:47 +0000 | [diff] [blame] | 543 | void visitUnary(const User &I, unsigned Opcode); |
| 544 | void visitFNeg(const User &I) { visitUnary(I, ISD::FNEG); } |
| 545 | |
Sanjay Patel | c4e4c5b | 2018-05-11 22:45:22 +0000 | [diff] [blame] | 546 | void visitBinary(const User &I, unsigned Opcode); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 547 | void visitShift(const User &I, unsigned Opcode); |
| 548 | void visitAdd(const User &I) { visitBinary(I, ISD::ADD); } |
| 549 | void visitFAdd(const User &I) { visitBinary(I, ISD::FADD); } |
| 550 | void visitSub(const User &I) { visitBinary(I, ISD::SUB); } |
Cameron McInally | 31c7a2f | 2020-08-03 10:19:33 -0500 | [diff] [blame] | 551 | void visitFSub(const User &I) { visitBinary(I, ISD::FSUB); } |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 552 | void visitMul(const User &I) { visitBinary(I, ISD::MUL); } |
| 553 | void visitFMul(const User &I) { visitBinary(I, ISD::FMUL); } |
| 554 | void visitURem(const User &I) { visitBinary(I, ISD::UREM); } |
| 555 | void visitSRem(const User &I) { visitBinary(I, ISD::SREM); } |
| 556 | void visitFRem(const User &I) { visitBinary(I, ISD::FREM); } |
| 557 | void visitUDiv(const User &I) { visitBinary(I, ISD::UDIV); } |
Benjamin Kramer | 9960a25 | 2011-07-08 10:31:30 +0000 | [diff] [blame] | 558 | void visitSDiv(const User &I); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 559 | void visitFDiv(const User &I) { visitBinary(I, ISD::FDIV); } |
| 560 | void visitAnd (const User &I) { visitBinary(I, ISD::AND); } |
| 561 | void visitOr (const User &I) { visitBinary(I, ISD::OR); } |
| 562 | void visitXor (const User &I) { visitBinary(I, ISD::XOR); } |
| 563 | void visitShl (const User &I) { visitShift(I, ISD::SHL); } |
| 564 | void visitLShr(const User &I) { visitShift(I, ISD::SRL); } |
| 565 | void visitAShr(const User &I) { visitShift(I, ISD::SRA); } |
Nikita Popov | deab451 | 2024-06-04 08:31:03 +0200 | [diff] [blame] | 566 | void visitICmp(const ICmpInst &I); |
| 567 | void visitFCmp(const FCmpInst &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 568 | // Visit the conversion instructions |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 569 | void visitTrunc(const User &I); |
| 570 | void visitZExt(const User &I); |
| 571 | void visitSExt(const User &I); |
| 572 | void visitFPTrunc(const User &I); |
| 573 | void visitFPExt(const User &I); |
| 574 | void visitFPToUI(const User &I); |
| 575 | void visitFPToSI(const User &I); |
| 576 | void visitUIToFP(const User &I); |
| 577 | void visitSIToFP(const User &I); |
| 578 | void visitPtrToInt(const User &I); |
| 579 | void visitIntToPtr(const User &I); |
| 580 | void visitBitCast(const User &I); |
Matt Arsenault | b03bd4d | 2013-11-15 01:34:59 +0000 | [diff] [blame] | 581 | void visitAddrSpaceCast(const User &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 582 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 583 | void visitExtractElement(const User &I); |
| 584 | void visitInsertElement(const User &I); |
| 585 | void visitShuffleVector(const User &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 586 | |
Nikita Popov | 16033ffd | 2022-06-29 10:42:46 +0200 | [diff] [blame] | 587 | void visitExtractValue(const ExtractValueInst &I); |
Nikita Popov | 7283f48 | 2022-06-28 11:25:54 +0200 | [diff] [blame] | 588 | void visitInsertValue(const InsertValueInst &I); |
Fangrui Song | cb0bab8 | 2018-07-16 18:51:40 +0000 | [diff] [blame] | 589 | void visitLandingPad(const LandingPadInst &LP); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 590 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 591 | void visitGetElementPtr(const User &I); |
| 592 | void visitSelect(const User &I); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 593 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 594 | void visitAlloca(const AllocaInst &I); |
| 595 | void visitLoad(const LoadInst &I); |
| 596 | void visitStore(const StoreInst &I); |
Elena Demikhovsky | caaceef | 2016-11-03 03:23:55 +0000 | [diff] [blame] | 597 | void visitMaskedLoad(const CallInst &I, bool IsExpanding = false); |
| 598 | void visitMaskedStore(const CallInst &I, bool IsCompressing = false); |
Elena Demikhovsky | 584ce37 | 2015-04-28 07:57:37 +0000 | [diff] [blame] | 599 | void visitMaskedGather(const CallInst &I); |
| 600 | void visitMaskedScatter(const CallInst &I); |
Eli Friedman | c9a551e | 2011-07-28 21:48:00 +0000 | [diff] [blame] | 601 | void visitAtomicCmpXchg(const AtomicCmpXchgInst &I); |
| 602 | void visitAtomicRMW(const AtomicRMWInst &I); |
Eli Friedman | fee02c6 | 2011-07-25 23:16:38 +0000 | [diff] [blame] | 603 | void visitFence(const FenceInst &I); |
Dan Gohman | f41ad47 | 2010-04-20 15:00:41 +0000 | [diff] [blame] | 604 | void visitPHI(const PHINode &I); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 605 | void visitCall(const CallInst &I); |
Dávid Bolvanský | 179e15d | 2020-09-25 17:54:21 +0200 | [diff] [blame] | 606 | bool visitMemCmpBCmpCall(const CallInst &I); |
Andrew Kaylor | b99d1cc | 2016-07-29 18:23:18 +0000 | [diff] [blame] | 607 | bool visitMemPCpyCall(const CallInst &I); |
Richard Sandiford | 6f6d551 | 2013-08-20 09:38:48 +0000 | [diff] [blame] | 608 | bool visitMemChrCall(const CallInst &I); |
Richard Sandiford | bb83a50 | 2013-08-16 11:29:37 +0000 | [diff] [blame] | 609 | bool visitStrCpyCall(const CallInst &I, bool isStpcpy); |
Richard Sandiford | ca23271 | 2013-08-16 11:21:54 +0000 | [diff] [blame] | 610 | bool visitStrCmpCall(const CallInst &I); |
Richard Sandiford | 0dec06a | 2013-08-16 11:41:43 +0000 | [diff] [blame] | 611 | bool visitStrLenCall(const CallInst &I); |
| 612 | bool visitStrNLenCall(const CallInst &I); |
Bob Wilson | 874886c | 2012-08-03 23:29:17 +0000 | [diff] [blame] | 613 | bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode); |
Matt Arsenault | 7c93690 | 2014-10-21 23:01:01 +0000 | [diff] [blame] | 614 | bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode); |
Eli Friedman | 342e8df | 2011-08-24 20:50:09 +0000 | [diff] [blame] | 615 | void visitAtomicLoad(const LoadInst &I); |
| 616 | void visitAtomicStore(const StoreInst &I); |
Manman Ren | e221a87 | 2016-04-05 18:13:16 +0000 | [diff] [blame] | 617 | void visitLoadFromSwiftError(const LoadInst &I); |
| 618 | void visitStoreToSwiftError(const StoreInst &I); |
aqjune | e87d716 | 2019-11-07 01:17:49 +0900 | [diff] [blame] | 619 | void visitFreeze(const FreezeInst &I); |
Eli Friedman | 342e8df | 2011-08-24 20:50:09 +0000 | [diff] [blame] | 620 | |
cynecx | 8ec9fd4 | 2021-05-13 19:05:11 +0100 | [diff] [blame] | 621 | void visitInlineAsm(const CallBase &Call, |
| 622 | const BasicBlock *EHPadBB = nullptr); |
Felipe de Azevedo Piovezan | 35f4ef1 | 2023-08-23 14:14:32 -0400 | [diff] [blame] | 623 | |
Jeremy Morse | 8c1b7fb | 2024-01-22 15:39:35 +0000 | [diff] [blame] | 624 | bool visitEntryValueDbgValue(ArrayRef<const Value *> Values, |
| 625 | DILocalVariable *Variable, DIExpression *Expr, |
| 626 | DebugLoc DbgLoc); |
Guillaume Chatelet | e386a01 | 2019-05-20 11:01:30 +0000 | [diff] [blame] | 627 | void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 628 | void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); |
Andrew Kaylor | f466001 | 2017-05-25 21:31:00 +0000 | [diff] [blame] | 629 | void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); |
Sameer Sahasrabuddhe | 6082263 | 2024-03-06 12:17:37 +0530 | [diff] [blame] | 630 | void visitConvergenceControl(const CallInst &I, unsigned Intrinsic); |
Graham Hunter | fbb37e9 | 2024-05-13 11:35:28 +0100 | [diff] [blame] | 631 | void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID); |
Graham Hunter | ed5aadd | 2024-11-14 17:48:43 +0000 | [diff] [blame] | 632 | void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic); |
Philip Reames | 143f3bf | 2022-09-21 09:04:42 -0700 | [diff] [blame] | 633 | void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 634 | const SmallVectorImpl<SDValue> &OpValues); |
Philip Reames | 143f3bf | 2022-09-21 09:04:42 -0700 | [diff] [blame] | 635 | void visitVPStore(const VPIntrinsic &VPIntrin, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 636 | const SmallVectorImpl<SDValue> &OpValues); |
Philip Reames | 143f3bf | 2022-09-21 09:04:42 -0700 | [diff] [blame] | 637 | void visitVPGather(const VPIntrinsic &VPIntrin, EVT VT, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 638 | const SmallVectorImpl<SDValue> &OpValues); |
Philip Reames | 143f3bf | 2022-09-21 09:04:42 -0700 | [diff] [blame] | 639 | void visitVPScatter(const VPIntrinsic &VPIntrin, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 640 | const SmallVectorImpl<SDValue> &OpValues); |
Lorenzo Albano | 28cfa76 | 2022-03-10 16:39:12 +0100 | [diff] [blame] | 641 | void visitVPStridedLoad(const VPIntrinsic &VPIntrin, EVT VT, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 642 | const SmallVectorImpl<SDValue> &OpValues); |
Lorenzo Albano | 28cfa76 | 2022-03-10 16:39:12 +0100 | [diff] [blame] | 643 | void visitVPStridedStore(const VPIntrinsic &VPIntrin, |
Craig Topper | bb64fd57 | 2023-04-03 13:59:55 -0700 | [diff] [blame] | 644 | const SmallVectorImpl<SDValue> &OpValues); |
Fraser Cormack | 6be5e87 | 2022-03-30 17:01:30 +0100 | [diff] [blame] | 645 | void visitVPCmp(const VPCmpIntrinsic &VPIntrin); |
Simon Moll | 3ffbc79 | 2020-12-09 11:36:30 +0100 | [diff] [blame] | 646 | void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 647 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 648 | void visitVAStart(const CallInst &I); |
| 649 | void visitVAArg(const VAArgInst &I); |
| 650 | void visitVAEnd(const CallInst &I); |
| 651 | void visitVACopy(const CallInst &I); |
Andrew Trick | 74f4c74 | 2013-10-31 17:18:24 +0000 | [diff] [blame] | 652 | void visitStackmap(const CallInst &I); |
Craig Topper | 806763e | 2020-04-10 23:12:07 -0700 | [diff] [blame] | 653 | void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 654 | |
Sanjoy Das | 3a02019 | 2016-03-17 00:47:14 +0000 | [diff] [blame] | 655 | // These two are implemented in StatepointLowering.cpp |
Fangrui Song | cb0bab8 | 2018-07-16 18:51:40 +0000 | [diff] [blame] | 656 | void visitGCRelocate(const GCRelocateInst &Relocate); |
Philip Reames | 92d1f0c | 2016-04-12 18:05:10 +0000 | [diff] [blame] | 657 | void visitGCResult(const GCResultInst &I); |
Philip Reames | 1a1bdb2 | 2014-12-02 18:50:36 +0000 | [diff] [blame] | 658 | |
Amara Emerson | cf9daa3 | 2017-05-09 10:43:25 +0000 | [diff] [blame] | 659 | void visitVectorReduce(const CallInst &I, unsigned Intrinsic); |
Caroline Concatto | 2d728bb | 2021-01-15 16:46:42 +0000 | [diff] [blame] | 660 | void visitVectorReverse(const CallInst &I); |
Cullen Rhodes | 2750f3ed | 2021-01-08 14:06:13 +0000 | [diff] [blame] | 661 | void visitVectorSplice(const CallInst &I); |
Min-Yih Hsu | 5a1e16f | 2025-02-05 15:30:33 -0800 | [diff] [blame] | 662 | void visitVectorInterleave(const CallInst &I, unsigned Factor); |
| 663 | void visitVectorDeinterleave(const CallInst &I, unsigned Factor); |
David Sherwood | 748ae52 | 2021-02-08 15:46:24 +0000 | [diff] [blame] | 664 | void visitStepVector(const CallInst &I); |
Amara Emerson | cf9daa3 | 2017-05-09 10:43:25 +0000 | [diff] [blame] | 665 | |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 666 | void visitUserOp1(const Instruction &I) { |
Torok Edwin | fbcc663 | 2009-07-14 16:55:14 +0000 | [diff] [blame] | 667 | llvm_unreachable("UserOp1 should not exist at instruction selection time!"); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 668 | } |
Dan Gohman | bcaf681 | 2010-04-15 01:51:59 +0000 | [diff] [blame] | 669 | void visitUserOp2(const Instruction &I) { |
Torok Edwin | fbcc663 | 2009-07-14 16:55:14 +0000 | [diff] [blame] | 670 | llvm_unreachable("UserOp2 should not exist at instruction selection time!"); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 671 | } |
Dan Gohman | 5b43aa0 | 2010-04-22 20:55:53 +0000 | [diff] [blame] | 672 | |
Richard Sandiford | e382775 | 2013-08-16 10:55:47 +0000 | [diff] [blame] | 673 | void processIntegerCallValue(const Instruction &I, |
| 674 | SDValue Value, bool IsSigned); |
| 675 | |
Bill Wendling | 0c0c57f | 2020-09-24 14:35:15 -0700 | [diff] [blame] | 676 | void HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB); |
Evan Cheng | 6e82245 | 2010-04-28 23:08:54 +0000 | [diff] [blame] | 677 | |
Craig Topper | f49f6cf | 2020-04-10 18:29:02 -0700 | [diff] [blame] | 678 | void emitInlineAsmError(const CallBase &Call, const Twine &Message); |
Renato Golin | 38ed802 | 2016-05-17 19:52:01 +0000 | [diff] [blame] | 679 | |
Michael Gottesman | e24f534 | 2022-04-01 15:57:52 -0700 | [diff] [blame] | 680 | /// An enum that states to emit func argument dbg value the kind of intrinsic |
| 681 | /// it originally had. This controls the internal behavior of |
| 682 | /// EmitFuncArgumentDbgValue. |
| 683 | enum class FuncArgumentDbgValueKind { |
| 684 | Value, // This was originally a llvm.dbg.value. |
Michael Gottesman | e24f534 | 2022-04-01 15:57:52 -0700 | [diff] [blame] | 685 | Declare, // This was originally a llvm.dbg.declare. |
| 686 | }; |
| 687 | |
Adrian Prantl | a617576 | 2017-07-28 21:27:35 +0000 | [diff] [blame] | 688 | /// If V is an function argument then create corresponding DBG_VALUE machine |
| 689 | /// instruction for it now. At the end of instruction selection, they will be |
| 690 | /// inserted to the entry BB. |
Duncan P. N. Exon Smith | a9308c4 | 2015-04-29 16:38:44 +0000 | [diff] [blame] | 691 | bool EmitFuncArgumentDbgValue(const Value *V, DILocalVariable *Variable, |
| 692 | DIExpression *Expr, DILocation *DL, |
Michael Gottesman | e24f534 | 2022-04-01 15:57:52 -0700 | [diff] [blame] | 693 | FuncArgumentDbgValueKind Kind, |
| 694 | const SDValue &N); |
Hans Wennborg | b4db142 | 2015-03-19 20:41:48 +0000 | [diff] [blame] | 695 | |
| 696 | /// Return the next block after MBB, or nullptr if there is none. |
| 697 | MachineBasicBlock *NextBlock(MachineBasicBlock *MBB); |
Krzysztof Parzyszek | a46c36b | 2015-04-13 17:16:45 +0000 | [diff] [blame] | 698 | |
| 699 | /// Update the DAG and DAG builder with the relevant information after |
| 700 | /// a new root node has been created which could be a tail call. |
| 701 | void updateDAGForMaybeTailCall(SDValue MaybeTC); |
Wolfgang Pieb | dfad9b2 | 2016-08-15 18:18:26 +0000 | [diff] [blame] | 702 | |
| 703 | /// Return the appropriate SDDbgValue based on N. |
| 704 | SDDbgValue *getDbgValue(SDValue N, DILocalVariable *Variable, |
Adrian Prantl | a617576 | 2017-07-28 21:27:35 +0000 | [diff] [blame] | 705 | DIExpression *Expr, const DebugLoc &dl, |
| 706 | unsigned DbgSDNodeOrder); |
Guillaume Chatelet | e386a01 | 2019-05-20 11:01:30 +0000 | [diff] [blame] | 707 | |
cynecx | 8ec9fd4 | 2021-05-13 19:05:11 +0100 | [diff] [blame] | 708 | SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, |
| 709 | MCSymbol *&BeginLabel); |
| 710 | SDValue lowerEndEH(SDValue Chain, const InvokeInst *II, |
| 711 | const BasicBlock *EHPadBB, MCSymbol *BeginLabel); |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 712 | }; |
| 713 | |
Bjorn Pettersson | ecd0960 | 2019-02-12 22:11:20 +0000 | [diff] [blame] | 714 | /// This struct represents the registers (physical or virtual) |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 715 | /// that a particular set of values is assigned, and the type information about |
| 716 | /// the value. The most common situation is to represent one value at a time, |
| 717 | /// but struct or array values are handled element-wise as multiple values. The |
| 718 | /// splitting of aggregates is performed recursively, so that we never have |
| 719 | /// aggregate-typed registers. The values at this point do not necessarily have |
| 720 | /// legal types, so each value may require one or more registers of some legal |
| 721 | /// type. |
| 722 | /// |
| 723 | struct RegsForValue { |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 724 | /// The value types of the values, which may not be legal, and |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 725 | /// may need be promoted or synthesized from one or more registers. |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 726 | SmallVector<EVT, 4> ValueVTs; |
| 727 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 728 | /// The value types of the registers. This is the same size as ValueVTs and it |
| 729 | /// records, for each value, what the type of the assigned register or |
| 730 | /// registers are. (Individual values are never synthesized from more than one |
| 731 | /// type of register.) |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 732 | /// |
| 733 | /// With virtual registers, the contents of RegVTs is redundant with TLI's |
| 734 | /// getRegisterType member function, however when with physical registers |
| 735 | /// it is necessary to have a separate record of the types. |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 736 | SmallVector<MVT, 4> RegVTs; |
| 737 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 738 | /// This list holds the registers assigned to the values. |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 739 | /// Each legal or promoted value requires one register, and each |
| 740 | /// expanded value requires multiple registers. |
Craig Topper | 9d3ab1c | 2024-09-17 23:45:58 -0700 | [diff] [blame] | 741 | SmallVector<Register, 4> Regs; |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 742 | |
Simon Dardis | 212cccb | 2017-06-09 14:37:08 +0000 | [diff] [blame] | 743 | /// This list holds the number of registers for each value. |
| 744 | SmallVector<unsigned, 4> RegCount; |
| 745 | |
| 746 | /// Records if this value needs to be treated in an ABI dependant manner, |
| 747 | /// different to normal type legalization. |
Krzysztof Parzyszek | 864aaa2 | 2022-12-01 16:18:32 -0800 | [diff] [blame] | 748 | std::optional<CallingConv::ID> CallConv; |
Simon Dardis | 212cccb | 2017-06-09 14:37:08 +0000 | [diff] [blame] | 749 | |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 750 | RegsForValue() = default; |
Craig Topper | 9d3ab1c | 2024-09-17 23:45:58 -0700 | [diff] [blame] | 751 | RegsForValue(const SmallVector<Register, 4> ®s, MVT regvt, EVT valuevt, |
Krzysztof Parzyszek | 864aaa2 | 2022-12-01 16:18:32 -0800 | [diff] [blame] | 752 | std::optional<CallingConv::ID> CC = std::nullopt); |
Mehdi Amini | 56228da | 2015-07-09 01:57:34 +0000 | [diff] [blame] | 753 | RegsForValue(LLVMContext &Context, const TargetLowering &TLI, |
Craig Topper | 9d3ab1c | 2024-09-17 23:45:58 -0700 | [diff] [blame] | 754 | const DataLayout &DL, Register Reg, Type *Ty, |
Krzysztof Parzyszek | 864aaa2 | 2022-12-01 16:18:32 -0800 | [diff] [blame] | 755 | std::optional<CallingConv::ID> CC); |
Matt Arsenault | 81920b0 | 2018-07-28 13:25:19 +0000 | [diff] [blame] | 756 | |
Kazu Hirata | d66cbc5 | 2022-06-20 20:26:05 -0700 | [diff] [blame] | 757 | bool isABIMangled() const { return CallConv.has_value(); } |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 758 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 759 | /// Add the specified values to this one. |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 760 | void append(const RegsForValue &RHS) { |
| 761 | ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); |
| 762 | RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); |
| 763 | Regs.append(RHS.Regs.begin(), RHS.Regs.end()); |
Simon Dardis | 212cccb | 2017-06-09 14:37:08 +0000 | [diff] [blame] | 764 | RegCount.push_back(RHS.Regs.size()); |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 765 | } |
| 766 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 767 | /// Emit a series of CopyFromReg nodes that copies from this value and returns |
| 768 | /// the result as a ValueVTs value. This uses Chain/Flag as the input and |
| 769 | /// updates them for the output Chain/Flag. If the Flag pointer is NULL, no |
| 770 | /// flag is used. |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 771 | SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, |
Craig Topper | b5f207e | 2023-04-02 19:45:36 -0700 | [diff] [blame] | 772 | const SDLoc &dl, SDValue &Chain, SDValue *Glue, |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 773 | const Value *V = nullptr) const; |
| 774 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 775 | /// Emit a series of CopyToReg nodes that copies the specified value into the |
| 776 | /// registers specified by this object. This uses Chain/Flag as the input and |
| 777 | /// updates them for the output Chain/Flag. If the Flag pointer is nullptr, no |
| 778 | /// flag is used. If V is not nullptr, then it is used in printing better |
| 779 | /// diagnostic messages on error. |
Benjamin Kramer | bdc4956 | 2016-06-12 15:39:02 +0000 | [diff] [blame] | 780 | void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, |
Craig Topper | b5f207e | 2023-04-02 19:45:36 -0700 | [diff] [blame] | 781 | SDValue &Chain, SDValue *Glue, const Value *V = nullptr, |
Benjamin Kramer | bdc4956 | 2016-06-12 15:39:02 +0000 | [diff] [blame] | 782 | ISD::NodeType PreferredExtendType = ISD::ANY_EXTEND) const; |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 783 | |
Sanjay Patel | 209b0f9 | 2017-03-02 20:48:08 +0000 | [diff] [blame] | 784 | /// Add this value to the specified inlineasm node operand list. This adds the |
| 785 | /// code marker, matching input operand index (if applicable), and includes |
| 786 | /// the number of values added into it. |
Nick Desaulniers | 2fad6e6 | 2023-08-31 08:39:52 -0700 | [diff] [blame] | 787 | void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, |
Benjamin Kramer | bdc4956 | 2016-06-12 15:39:02 +0000 | [diff] [blame] | 788 | unsigned MatchingIdx, const SDLoc &dl, |
| 789 | SelectionDAG &DAG, std::vector<SDValue> &Ops) const; |
Bjorn Pettersson | 304877e | 2018-05-03 17:04:16 +0000 | [diff] [blame] | 790 | |
| 791 | /// Check if the total RegCount is greater than one. |
| 792 | bool occupiesMultipleRegs() const { |
| 793 | return std::accumulate(RegCount.begin(), RegCount.end(), 0) > 1; |
| 794 | } |
Bjorn Pettersson | 27a841f | 2018-05-04 08:50:48 +0000 | [diff] [blame] | 795 | |
| 796 | /// Return a list of registers and their sizes. |
Craig Topper | 9d3ab1c | 2024-09-17 23:45:58 -0700 | [diff] [blame] | 797 | SmallVector<std::pair<Register, TypeSize>, 4> getRegsAndSizes() const; |
Sanjoy Das | 3936a97 | 2015-05-05 23:06:54 +0000 | [diff] [blame] | 798 | }; |
| 799 | |
Dan Gohman | 575fad3 | 2008-09-03 16:12:24 +0000 | [diff] [blame] | 800 | } // end namespace llvm |
| 801 | |
Eugene Zelenko | fa57bd0 | 2017-09-27 23:26:01 +0000 | [diff] [blame] | 802 | #endif // LLVM_LIB_CODEGEN_SELECTIONDAG_SELECTIONDAGBUILDER_H |