Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 1 | //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 2 | // |
Chandler Carruth | 2946cd7 | 2019-01-19 08:50:56 +0000 | [diff] [blame] | 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 9 | /// \file This file implements the LegalizerHelper class to legalize |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 10 | /// individual instructions and the LegalizeMachineIR wrapper pass for the |
| 11 | /// primary legalization. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 15 | #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 16 | #include "llvm/CodeGen/GlobalISel/CallLowering.h" |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 17 | #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" |
Jessica Del | fc672b6 | 2023-02-21 09:40:07 +0100 | [diff] [blame] | 18 | #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" |
serge-sans-paille | ed98c1b | 2022-03-09 22:29:31 +0100 | [diff] [blame] | 19 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 20 | #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 21 | #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 22 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
serge-sans-paille | ed98c1b | 2022-03-09 22:29:31 +0100 | [diff] [blame] | 23 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
Amara Emerson | a35c2c7 | 2021-02-21 14:17:03 -0800 | [diff] [blame] | 24 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 25 | #include "llvm/CodeGen/MachineConstantPool.h" |
serge-sans-paille | ed98c1b | 2022-03-09 22:29:31 +0100 | [diff] [blame] | 26 | #include "llvm/CodeGen/MachineFrameInfo.h" |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 27 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
Joseph Huber | 615b7ee | 2024-07-20 07:29:04 -0500 | [diff] [blame] | 28 | #include "llvm/CodeGen/RuntimeLibcallUtil.h" |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 29 | #include "llvm/CodeGen/TargetFrameLowering.h" |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 30 | #include "llvm/CodeGen/TargetInstrInfo.h" |
David Blaikie | b3bde2e | 2017-11-17 01:07:10 +0000 | [diff] [blame] | 31 | #include "llvm/CodeGen/TargetLowering.h" |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 32 | #include "llvm/CodeGen/TargetOpcodes.h" |
David Blaikie | b3bde2e | 2017-11-17 01:07:10 +0000 | [diff] [blame] | 33 | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 34 | #include "llvm/IR/Instructions.h" |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 35 | #include "llvm/Support/Debug.h" |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 36 | #include "llvm/Support/MathExtras.h" |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 37 | #include "llvm/Support/raw_ostream.h" |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 38 | #include "llvm/Target/TargetMachine.h" |
Kazu Hirata | 267f21a | 2022-08-28 10:41:51 -0700 | [diff] [blame] | 39 | #include <numeric> |
Kazu Hirata | 3ccbfc3 | 2022-11-26 14:44:54 -0800 | [diff] [blame] | 40 | #include <optional> |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 41 | |
Daniel Sanders | 5377fb3 | 2017-04-20 15:46:12 +0000 | [diff] [blame] | 42 | #define DEBUG_TYPE "legalizer" |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 43 | |
| 44 | using namespace llvm; |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 45 | using namespace LegalizeActions; |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 46 | using namespace MIPatternMatch; |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 47 | |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 48 | /// Try to break down \p OrigTy into \p NarrowTy sized pieces. |
| 49 | /// |
| 50 | /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, |
| 51 | /// with any leftover piece as type \p LeftoverTy |
| 52 | /// |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 53 | /// Returns -1 in the first element of the pair if the breakdown is not |
| 54 | /// satisfiable. |
| 55 | static std::pair<int, int> |
| 56 | getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 57 | assert(!LeftoverTy.isValid() && "this is an out argument"); |
| 58 | |
| 59 | unsigned Size = OrigTy.getSizeInBits(); |
| 60 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
| 61 | unsigned NumParts = Size / NarrowSize; |
| 62 | unsigned LeftoverSize = Size - NumParts * NarrowSize; |
| 63 | assert(Size > NarrowSize); |
| 64 | |
| 65 | if (LeftoverSize == 0) |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 66 | return {NumParts, 0}; |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 67 | |
| 68 | if (NarrowTy.isVector()) { |
| 69 | unsigned EltSize = OrigTy.getScalarSizeInBits(); |
| 70 | if (LeftoverSize % EltSize != 0) |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 71 | return {-1, -1}; |
David Green | 34de215 | 2024-05-13 21:58:41 +0100 | [diff] [blame] | 72 | LeftoverTy = |
| 73 | LLT::scalarOrVector(ElementCount::getFixed(LeftoverSize / EltSize), |
| 74 | OrigTy.getElementType()); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 75 | } else { |
| 76 | LeftoverTy = LLT::scalar(LeftoverSize); |
| 77 | } |
| 78 | |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 79 | int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits(); |
| 80 | return std::make_pair(NumParts, NumLeftover); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 81 | } |
| 82 | |
Konstantin Schwarz | 76986bd | 2020-02-06 10:01:57 -0800 | [diff] [blame] | 83 | static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) { |
| 84 | |
| 85 | if (!Ty.isScalar()) |
| 86 | return nullptr; |
| 87 | |
| 88 | switch (Ty.getSizeInBits()) { |
| 89 | case 16: |
| 90 | return Type::getHalfTy(Ctx); |
| 91 | case 32: |
| 92 | return Type::getFloatTy(Ctx); |
| 93 | case 64: |
| 94 | return Type::getDoubleTy(Ctx); |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 95 | case 80: |
| 96 | return Type::getX86_FP80Ty(Ctx); |
Konstantin Schwarz | 76986bd | 2020-02-06 10:01:57 -0800 | [diff] [blame] | 97 | case 128: |
| 98 | return Type::getFP128Ty(Ctx); |
| 99 | default: |
| 100 | return nullptr; |
| 101 | } |
| 102 | } |
| 103 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 104 | LegalizerHelper::LegalizerHelper(MachineFunction &MF, |
Aditya Nandakumar | 500e3ea | 2019-01-16 00:40:37 +0000 | [diff] [blame] | 105 | GISelChangeObserver &Observer, |
| 106 | MachineIRBuilder &Builder) |
Matt Arsenault | 7f8b2e1 | 2020-06-09 17:02:12 -0400 | [diff] [blame] | 107 | : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()), |
Matt Arsenault | adbcc8e | 2020-07-31 11:41:05 -0400 | [diff] [blame] | 108 | LI(*MF.getSubtarget().getLegalizerInfo()), |
Jessica Del | fc672b6 | 2023-02-21 09:40:07 +0100 | [diff] [blame] | 109 | TLI(*MF.getSubtarget().getTargetLowering()), KB(nullptr) {} |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 110 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 111 | LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, |
Aditya Nandakumar | 500e3ea | 2019-01-16 00:40:37 +0000 | [diff] [blame] | 112 | GISelChangeObserver &Observer, |
Jessica Del | fc672b6 | 2023-02-21 09:40:07 +0100 | [diff] [blame] | 113 | MachineIRBuilder &B, GISelKnownBits *KB) |
| 114 | : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI), |
| 115 | TLI(*MF.getSubtarget().getTargetLowering()), KB(KB) {} |
Matt Arsenault | d55d592 | 2020-08-19 10:46:59 -0400 | [diff] [blame] | 116 | |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 117 | LegalizerHelper::LegalizeResult |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 118 | LegalizerHelper::legalizeInstrStep(MachineInstr &MI, |
| 119 | LostDebugLocObserver &LocObserver) { |
Matt Arsenault | c1d771d | 2020-06-07 21:56:42 -0400 | [diff] [blame] | 120 | LLVM_DEBUG(dbgs() << "Legalizing: " << MI); |
Daniel Sanders | 5377fb3 | 2017-04-20 15:46:12 +0000 | [diff] [blame] | 121 | |
Matt Arsenault | 3282309 | 2020-06-07 20:57:28 -0400 | [diff] [blame] | 122 | MIRBuilder.setInstrAndDebugLoc(MI); |
| 123 | |
Sameer Sahasrabuddhe | d9847cd | 2023-07-31 12:14:34 +0530 | [diff] [blame] | 124 | if (isa<GIntrinsic>(MI)) |
Matt Arsenault | 7f8b2e1 | 2020-06-09 17:02:12 -0400 | [diff] [blame] | 125 | return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize; |
Daniel Sanders | 262ed0e | 2018-01-24 17:17:46 +0000 | [diff] [blame] | 126 | auto Step = LI.getAction(MI, MRI); |
| 127 | switch (Step.Action) { |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 128 | case Legal: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 129 | LLVM_DEBUG(dbgs() << ".. Already legal\n"); |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 130 | return AlreadyLegal; |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 131 | case Libcall: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 132 | LLVM_DEBUG(dbgs() << ".. Convert to libcall\n"); |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 133 | return libcall(MI, LocObserver); |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 134 | case NarrowScalar: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 135 | LLVM_DEBUG(dbgs() << ".. Narrow scalar\n"); |
Daniel Sanders | 262ed0e | 2018-01-24 17:17:46 +0000 | [diff] [blame] | 136 | return narrowScalar(MI, Step.TypeIdx, Step.NewType); |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 137 | case WidenScalar: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 138 | LLVM_DEBUG(dbgs() << ".. Widen scalar\n"); |
Daniel Sanders | 262ed0e | 2018-01-24 17:17:46 +0000 | [diff] [blame] | 139 | return widenScalar(MI, Step.TypeIdx, Step.NewType); |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 140 | case Bitcast: |
| 141 | LLVM_DEBUG(dbgs() << ".. Bitcast type\n"); |
| 142 | return bitcast(MI, Step.TypeIdx, Step.NewType); |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 143 | case Lower: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 144 | LLVM_DEBUG(dbgs() << ".. Lower\n"); |
Daniel Sanders | 262ed0e | 2018-01-24 17:17:46 +0000 | [diff] [blame] | 145 | return lower(MI, Step.TypeIdx, Step.NewType); |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 146 | case FewerElements: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 147 | LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n"); |
Daniel Sanders | 262ed0e | 2018-01-24 17:17:46 +0000 | [diff] [blame] | 148 | return fewerElementsVector(MI, Step.TypeIdx, Step.NewType); |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 149 | case MoreElements: |
| 150 | LLVM_DEBUG(dbgs() << ".. Increase number of elements\n"); |
| 151 | return moreElementsVector(MI, Step.TypeIdx, Step.NewType); |
Daniel Sanders | 9ade559 | 2018-01-29 17:37:29 +0000 | [diff] [blame] | 152 | case Custom: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 153 | LLVM_DEBUG(dbgs() << ".. Custom legalization\n"); |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 154 | return LI.legalizeCustom(*this, MI, LocObserver) ? Legalized |
| 155 | : UnableToLegalize; |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 156 | default: |
Nicola Zaghen | d34e60c | 2018-05-14 12:53:11 +0000 | [diff] [blame] | 157 | LLVM_DEBUG(dbgs() << ".. Unable to legalize\n"); |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 158 | return UnableToLegalize; |
| 159 | } |
| 160 | } |
| 161 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 162 | void LegalizerHelper::insertParts(Register DstReg, |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 163 | LLT ResultTy, LLT PartTy, |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 164 | ArrayRef<Register> PartRegs, |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 165 | LLT LeftoverTy, |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 166 | ArrayRef<Register> LeftoverRegs) { |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 167 | if (!LeftoverTy.isValid()) { |
| 168 | assert(LeftoverRegs.empty()); |
| 169 | |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 170 | if (!ResultTy.isVector()) { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 171 | MIRBuilder.buildMergeLikeInstr(DstReg, PartRegs); |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 172 | return; |
| 173 | } |
| 174 | |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 175 | if (PartTy.isVector()) |
| 176 | MIRBuilder.buildConcatVectors(DstReg, PartRegs); |
| 177 | else |
| 178 | MIRBuilder.buildBuildVector(DstReg, PartRegs); |
| 179 | return; |
| 180 | } |
| 181 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 182 | // Merge sub-vectors with different number of elements and insert into DstReg. |
| 183 | if (ResultTy.isVector()) { |
| 184 | assert(LeftoverRegs.size() == 1 && "Expected one leftover register"); |
Craig Topper | e3284d8 | 2024-12-10 07:18:20 -0800 | [diff] [blame] | 185 | SmallVector<Register, 8> AllRegs(PartRegs.begin(), PartRegs.end()); |
| 186 | AllRegs.append(LeftoverRegs.begin(), LeftoverRegs.end()); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 187 | return mergeMixedSubvectors(DstReg, AllRegs); |
| 188 | } |
| 189 | |
Matt Arsenault | 31a9659 | 2021-06-07 18:57:03 -0400 | [diff] [blame] | 190 | SmallVector<Register> GCDRegs; |
Jessica Paquette | 47aeeff | 2021-07-08 16:45:45 -0700 | [diff] [blame] | 191 | LLT GCDTy = getGCDType(getGCDType(ResultTy, LeftoverTy), PartTy); |
| 192 | for (auto PartReg : concat<const Register>(PartRegs, LeftoverRegs)) |
| 193 | extractGCDType(GCDRegs, GCDTy, PartReg); |
Matt Arsenault | 31a9659 | 2021-06-07 18:57:03 -0400 | [diff] [blame] | 194 | LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs); |
| 195 | buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 196 | } |
| 197 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 198 | void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts, |
| 199 | Register Reg) { |
| 200 | LLT Ty = MRI.getType(Reg); |
| 201 | SmallVector<Register, 8> RegElts; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 202 | extractParts(Reg, Ty.getScalarType(), Ty.getNumElements(), RegElts, |
| 203 | MIRBuilder, MRI); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 204 | Elts.append(RegElts); |
| 205 | } |
| 206 | |
| 207 | /// Merge \p PartRegs with different types into \p DstReg. |
| 208 | void LegalizerHelper::mergeMixedSubvectors(Register DstReg, |
| 209 | ArrayRef<Register> PartRegs) { |
| 210 | SmallVector<Register, 8> AllElts; |
| 211 | for (unsigned i = 0; i < PartRegs.size() - 1; ++i) |
| 212 | appendVectorElts(AllElts, PartRegs[i]); |
| 213 | |
| 214 | Register Leftover = PartRegs[PartRegs.size() - 1]; |
David Green | 34de215 | 2024-05-13 21:58:41 +0100 | [diff] [blame] | 215 | if (!MRI.getType(Leftover).isVector()) |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 216 | AllElts.push_back(Leftover); |
| 217 | else |
| 218 | appendVectorElts(AllElts, Leftover); |
| 219 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 220 | MIRBuilder.buildMergeLikeInstr(DstReg, AllElts); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 221 | } |
| 222 | |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 223 | /// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs. |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 224 | static void getUnmergeResults(SmallVectorImpl<Register> &Regs, |
| 225 | const MachineInstr &MI) { |
| 226 | assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES); |
| 227 | |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 228 | const int StartIdx = Regs.size(); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 229 | const int NumResults = MI.getNumOperands() - 1; |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 230 | Regs.resize(Regs.size() + NumResults); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 231 | for (int I = 0; I != NumResults; ++I) |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 232 | Regs[StartIdx + I] = MI.getOperand(I).getReg(); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 233 | } |
| 234 | |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 235 | void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, |
| 236 | LLT GCDTy, Register SrcReg) { |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 237 | LLT SrcTy = MRI.getType(SrcReg); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 238 | if (SrcTy == GCDTy) { |
| 239 | // If the source already evenly divides the result type, we don't need to do |
| 240 | // anything. |
| 241 | Parts.push_back(SrcReg); |
| 242 | } else { |
| 243 | // Need to split into common type sized pieces. |
| 244 | auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); |
| 245 | getUnmergeResults(Parts, *Unmerge); |
| 246 | } |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 247 | } |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 248 | |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 249 | LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, |
| 250 | LLT NarrowTy, Register SrcReg) { |
| 251 | LLT SrcTy = MRI.getType(SrcReg); |
| 252 | LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy); |
| 253 | extractGCDType(Parts, GCDTy, SrcReg); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 254 | return GCDTy; |
| 255 | } |
| 256 | |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 257 | LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, |
| 258 | SmallVectorImpl<Register> &VRegs, |
| 259 | unsigned PadStrategy) { |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 260 | LLT LCMTy = getLCMType(DstTy, NarrowTy); |
| 261 | |
| 262 | int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits(); |
| 263 | int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits(); |
| 264 | int NumOrigSrc = VRegs.size(); |
| 265 | |
| 266 | Register PadReg; |
| 267 | |
| 268 | // Get a value we can use to pad the source value if the sources won't evenly |
| 269 | // cover the result type. |
| 270 | if (NumOrigSrc < NumParts * NumSubParts) { |
| 271 | if (PadStrategy == TargetOpcode::G_ZEXT) |
| 272 | PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0); |
| 273 | else if (PadStrategy == TargetOpcode::G_ANYEXT) |
| 274 | PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0); |
| 275 | else { |
| 276 | assert(PadStrategy == TargetOpcode::G_SEXT); |
| 277 | |
| 278 | // Shift the sign bit of the low register through the high register. |
| 279 | auto ShiftAmt = |
| 280 | MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1); |
| 281 | PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0); |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | // Registers for the final merge to be produced. |
Matt Arsenault | de8451f | 2020-02-04 10:34:22 -0500 | [diff] [blame] | 286 | SmallVector<Register, 4> Remerge(NumParts); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 287 | |
| 288 | // Registers needed for intermediate merges, which will be merged into a |
| 289 | // source for Remerge. |
Matt Arsenault | de8451f | 2020-02-04 10:34:22 -0500 | [diff] [blame] | 290 | SmallVector<Register, 4> SubMerge(NumSubParts); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 291 | |
| 292 | // Once we've fully read off the end of the original source bits, we can reuse |
| 293 | // the same high bits for remaining padding elements. |
| 294 | Register AllPadReg; |
| 295 | |
| 296 | // Build merges to the LCM type to cover the original result type. |
| 297 | for (int I = 0; I != NumParts; ++I) { |
| 298 | bool AllMergePartsArePadding = true; |
| 299 | |
| 300 | // Build the requested merges to the requested type. |
| 301 | for (int J = 0; J != NumSubParts; ++J) { |
| 302 | int Idx = I * NumSubParts + J; |
| 303 | if (Idx >= NumOrigSrc) { |
| 304 | SubMerge[J] = PadReg; |
| 305 | continue; |
| 306 | } |
| 307 | |
| 308 | SubMerge[J] = VRegs[Idx]; |
| 309 | |
| 310 | // There are meaningful bits here we can't reuse later. |
| 311 | AllMergePartsArePadding = false; |
| 312 | } |
| 313 | |
| 314 | // If we've filled up a complete piece with padding bits, we can directly |
| 315 | // emit the natural sized constant if applicable, rather than a merge of |
| 316 | // smaller constants. |
| 317 | if (AllMergePartsArePadding && !AllPadReg) { |
| 318 | if (PadStrategy == TargetOpcode::G_ANYEXT) |
| 319 | AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0); |
| 320 | else if (PadStrategy == TargetOpcode::G_ZEXT) |
| 321 | AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0); |
| 322 | |
| 323 | // If this is a sign extension, we can't materialize a trivial constant |
| 324 | // with the right type and have to produce a merge. |
| 325 | } |
| 326 | |
| 327 | if (AllPadReg) { |
| 328 | // Avoid creating additional instructions if we're just adding additional |
| 329 | // copies of padding bits. |
| 330 | Remerge[I] = AllPadReg; |
| 331 | continue; |
| 332 | } |
| 333 | |
| 334 | if (NumSubParts == 1) |
| 335 | Remerge[I] = SubMerge[0]; |
| 336 | else |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 337 | Remerge[I] = MIRBuilder.buildMergeLikeInstr(NarrowTy, SubMerge).getReg(0); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 338 | |
| 339 | // In the sign extend padding case, re-use the first all-signbit merge. |
| 340 | if (AllMergePartsArePadding && !AllPadReg) |
| 341 | AllPadReg = Remerge[I]; |
| 342 | } |
| 343 | |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 344 | VRegs = std::move(Remerge); |
| 345 | return LCMTy; |
| 346 | } |
| 347 | |
| 348 | void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, |
| 349 | ArrayRef<Register> RemergeRegs) { |
| 350 | LLT DstTy = MRI.getType(DstReg); |
| 351 | |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 352 | // Create the merge to the widened source, and extract the relevant bits into |
| 353 | // the result. |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 354 | |
| 355 | if (DstTy == LCMTy) { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 356 | MIRBuilder.buildMergeLikeInstr(DstReg, RemergeRegs); |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 357 | return; |
| 358 | } |
| 359 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 360 | auto Remerge = MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs); |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 361 | if (DstTy.isScalar() && LCMTy.isScalar()) { |
| 362 | MIRBuilder.buildTrunc(DstReg, Remerge); |
| 363 | return; |
| 364 | } |
| 365 | |
| 366 | if (LCMTy.isVector()) { |
Matt Arsenault | e75afc9 | 2020-07-28 10:15:42 -0400 | [diff] [blame] | 367 | unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits(); |
| 368 | SmallVector<Register, 8> UnmergeDefs(NumDefs); |
| 369 | UnmergeDefs[0] = DstReg; |
| 370 | for (unsigned I = 1; I != NumDefs; ++I) |
| 371 | UnmergeDefs[I] = MRI.createGenericVirtualRegister(DstTy); |
| 372 | |
| 373 | MIRBuilder.buildUnmerge(UnmergeDefs, |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 374 | MIRBuilder.buildMergeLikeInstr(LCMTy, RemergeRegs)); |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 375 | return; |
| 376 | } |
| 377 | |
| 378 | llvm_unreachable("unhandled case"); |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 379 | } |
| 380 | |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 381 | static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 382 | #define RTLIBCASE_INT(LibcallPrefix) \ |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 383 | do { \ |
| 384 | switch (Size) { \ |
| 385 | case 32: \ |
| 386 | return RTLIB::LibcallPrefix##32; \ |
| 387 | case 64: \ |
| 388 | return RTLIB::LibcallPrefix##64; \ |
| 389 | case 128: \ |
| 390 | return RTLIB::LibcallPrefix##128; \ |
| 391 | default: \ |
| 392 | llvm_unreachable("unexpected size"); \ |
| 393 | } \ |
| 394 | } while (0) |
| 395 | |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 396 | #define RTLIBCASE(LibcallPrefix) \ |
| 397 | do { \ |
| 398 | switch (Size) { \ |
| 399 | case 32: \ |
| 400 | return RTLIB::LibcallPrefix##32; \ |
| 401 | case 64: \ |
| 402 | return RTLIB::LibcallPrefix##64; \ |
| 403 | case 80: \ |
| 404 | return RTLIB::LibcallPrefix##80; \ |
| 405 | case 128: \ |
| 406 | return RTLIB::LibcallPrefix##128; \ |
| 407 | default: \ |
| 408 | llvm_unreachable("unexpected size"); \ |
| 409 | } \ |
| 410 | } while (0) |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 411 | |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 412 | switch (Opcode) { |
Kai Nacke | b383753 | 2022-08-02 13:12:38 -0400 | [diff] [blame] | 413 | case TargetOpcode::G_MUL: |
| 414 | RTLIBCASE_INT(MUL_I); |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 415 | case TargetOpcode::G_SDIV: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 416 | RTLIBCASE_INT(SDIV_I); |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 417 | case TargetOpcode::G_UDIV: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 418 | RTLIBCASE_INT(UDIV_I); |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 419 | case TargetOpcode::G_SREM: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 420 | RTLIBCASE_INT(SREM_I); |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 421 | case TargetOpcode::G_UREM: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 422 | RTLIBCASE_INT(UREM_I); |
Diana Picus | 0528e2c | 2018-11-26 11:07:02 +0000 | [diff] [blame] | 423 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 424 | RTLIBCASE_INT(CTLZ_I); |
Diana Picus | 1314a28 | 2017-04-11 10:52:34 +0000 | [diff] [blame] | 425 | case TargetOpcode::G_FADD: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 426 | RTLIBCASE(ADD_F); |
Javed Absar | 5cde1cc | 2017-10-30 13:51:56 +0000 | [diff] [blame] | 427 | case TargetOpcode::G_FSUB: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 428 | RTLIBCASE(SUB_F); |
Diana Picus | 9faa09b | 2017-11-23 12:44:20 +0000 | [diff] [blame] | 429 | case TargetOpcode::G_FMUL: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 430 | RTLIBCASE(MUL_F); |
Diana Picus | c01f7f1 | 2017-11-23 13:26:07 +0000 | [diff] [blame] | 431 | case TargetOpcode::G_FDIV: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 432 | RTLIBCASE(DIV_F); |
Jessica Paquette | 84bedac | 2019-01-30 23:46:15 +0000 | [diff] [blame] | 433 | case TargetOpcode::G_FEXP: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 434 | RTLIBCASE(EXP_F); |
Jessica Paquette | e794121 | 2019-04-03 16:58:32 +0000 | [diff] [blame] | 435 | case TargetOpcode::G_FEXP2: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 436 | RTLIBCASE(EXP2_F); |
Matt Arsenault | b14e83d | 2023-08-12 07:20:00 -0400 | [diff] [blame] | 437 | case TargetOpcode::G_FEXP10: |
| 438 | RTLIBCASE(EXP10_F); |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 439 | case TargetOpcode::G_FREM: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 440 | RTLIBCASE(REM_F); |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 441 | case TargetOpcode::G_FPOW: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 442 | RTLIBCASE(POW_F); |
David Green | 5550e9c | 2024-01-04 07:26:23 +0000 | [diff] [blame] | 443 | case TargetOpcode::G_FPOWI: |
| 444 | RTLIBCASE(POWI_F); |
Diana Picus | e74243d | 2018-01-12 11:30:45 +0000 | [diff] [blame] | 445 | case TargetOpcode::G_FMA: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 446 | RTLIBCASE(FMA_F); |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 447 | case TargetOpcode::G_FSIN: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 448 | RTLIBCASE(SIN_F); |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 449 | case TargetOpcode::G_FCOS: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 450 | RTLIBCASE(COS_F); |
Farzon Lotfi | 1d87433 | 2024-06-05 15:01:33 -0400 | [diff] [blame] | 451 | case TargetOpcode::G_FTAN: |
| 452 | RTLIBCASE(TAN_F); |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 453 | case TargetOpcode::G_FASIN: |
| 454 | RTLIBCASE(ASIN_F); |
| 455 | case TargetOpcode::G_FACOS: |
| 456 | RTLIBCASE(ACOS_F); |
| 457 | case TargetOpcode::G_FATAN: |
| 458 | RTLIBCASE(ATAN_F); |
Tex Riddell | c03d09c | 2024-10-24 17:53:12 -0700 | [diff] [blame] | 459 | case TargetOpcode::G_FATAN2: |
| 460 | RTLIBCASE(ATAN2_F); |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 461 | case TargetOpcode::G_FSINH: |
| 462 | RTLIBCASE(SINH_F); |
| 463 | case TargetOpcode::G_FCOSH: |
| 464 | RTLIBCASE(COSH_F); |
| 465 | case TargetOpcode::G_FTANH: |
| 466 | RTLIBCASE(TANH_F); |
Jessica Paquette | c49428a | 2019-01-28 19:53:14 +0000 | [diff] [blame] | 467 | case TargetOpcode::G_FLOG10: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 468 | RTLIBCASE(LOG10_F); |
Jessica Paquette | 2d73ecd | 2019-01-28 21:27:23 +0000 | [diff] [blame] | 469 | case TargetOpcode::G_FLOG: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 470 | RTLIBCASE(LOG_F); |
Jessica Paquette | 0154bd1 | 2019-01-30 21:16:04 +0000 | [diff] [blame] | 471 | case TargetOpcode::G_FLOG2: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 472 | RTLIBCASE(LOG2_F); |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 473 | case TargetOpcode::G_FLDEXP: |
| 474 | RTLIBCASE(LDEXP_F); |
Petar Avramovic | faaa2b5d | 2019-06-06 09:02:24 +0000 | [diff] [blame] | 475 | case TargetOpcode::G_FCEIL: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 476 | RTLIBCASE(CEIL_F); |
Petar Avramovic | faaa2b5d | 2019-06-06 09:02:24 +0000 | [diff] [blame] | 477 | case TargetOpcode::G_FFLOOR: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 478 | RTLIBCASE(FLOOR_F); |
| 479 | case TargetOpcode::G_FMINNUM: |
| 480 | RTLIBCASE(FMIN_F); |
| 481 | case TargetOpcode::G_FMAXNUM: |
| 482 | RTLIBCASE(FMAX_F); |
| 483 | case TargetOpcode::G_FSQRT: |
| 484 | RTLIBCASE(SQRT_F); |
| 485 | case TargetOpcode::G_FRINT: |
| 486 | RTLIBCASE(RINT_F); |
| 487 | case TargetOpcode::G_FNEARBYINT: |
| 488 | RTLIBCASE(NEARBYINT_F); |
Craig Topper | d5d1417 | 2024-09-18 12:07:44 -0700 | [diff] [blame] | 489 | case TargetOpcode::G_INTRINSIC_TRUNC: |
| 490 | RTLIBCASE(TRUNC_F); |
| 491 | case TargetOpcode::G_INTRINSIC_ROUND: |
| 492 | RTLIBCASE(ROUND_F); |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 493 | case TargetOpcode::G_INTRINSIC_ROUNDEVEN: |
| 494 | RTLIBCASE(ROUNDEVEN_F); |
David Green | 28d28d5 | 2024-04-15 09:41:08 +0100 | [diff] [blame] | 495 | case TargetOpcode::G_INTRINSIC_LRINT: |
| 496 | RTLIBCASE(LRINT_F); |
David Green | 8d49ce1 | 2024-04-17 18:38:24 +0100 | [diff] [blame] | 497 | case TargetOpcode::G_INTRINSIC_LLRINT: |
| 498 | RTLIBCASE(LLRINT_F); |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 499 | } |
| 500 | llvm_unreachable("Unknown libcall function"); |
Craig Topper | ebcaa57 | 2024-11-25 18:00:03 -0800 | [diff] [blame] | 501 | #undef RTLIBCASE_INT |
| 502 | #undef RTLIBCASE |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 503 | } |
| 504 | |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 505 | /// True if an instruction is in tail position in its caller. Intended for |
| 506 | /// legalizing libcalls as tail calls when possible. |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 507 | static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result, |
| 508 | MachineInstr &MI, |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 509 | const TargetInstrInfo &TII, |
| 510 | MachineRegisterInfo &MRI) { |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 511 | MachineBasicBlock &MBB = *MI.getParent(); |
| 512 | const Function &F = MBB.getParent()->getFunction(); |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 513 | |
| 514 | // Conservatively require the attributes of the call to match those of |
| 515 | // the return. Ignore NoAlias and NonNull because they don't affect the |
| 516 | // call sequence. |
| 517 | AttributeList CallerAttrs = F.getAttributes(); |
Nikita Popov | c63a317 | 2022-01-15 22:14:16 +0100 | [diff] [blame] | 518 | if (AttrBuilder(F.getContext(), CallerAttrs.getRetAttrs()) |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 519 | .removeAttribute(Attribute::NoAlias) |
| 520 | .removeAttribute(Attribute::NonNull) |
| 521 | .hasAttributes()) |
| 522 | return false; |
| 523 | |
| 524 | // It's not safe to eliminate the sign / zero extension of the return value. |
Arthur Eubanks | d7593eb | 2021-08-13 11:59:18 -0700 | [diff] [blame] | 525 | if (CallerAttrs.hasRetAttr(Attribute::ZExt) || |
| 526 | CallerAttrs.hasRetAttr(Attribute::SExt)) |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 527 | return false; |
| 528 | |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 529 | // Only tail call if the following instruction is a standard return or if we |
| 530 | // have a `thisreturn` callee, and a sequence like: |
| 531 | // |
| 532 | // G_MEMCPY %0, %1, %2 |
| 533 | // $x0 = COPY %0 |
| 534 | // RET_ReallyLR implicit $x0 |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 535 | auto Next = next_nodbg(MI.getIterator(), MBB.instr_end()); |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 536 | if (Next != MBB.instr_end() && Next->isCopy()) { |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 537 | if (MI.getOpcode() == TargetOpcode::G_BZERO) |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 538 | return false; |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 539 | |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 540 | // For MEMCPY/MOMMOVE/MEMSET these will be the first use (the dst), as the |
| 541 | // mempy/etc routines return the same parameter. For other it will be the |
| 542 | // returned value. |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 543 | Register VReg = MI.getOperand(0).getReg(); |
| 544 | if (!VReg.isVirtual() || VReg != Next->getOperand(1).getReg()) |
| 545 | return false; |
| 546 | |
| 547 | Register PReg = Next->getOperand(0).getReg(); |
| 548 | if (!PReg.isPhysical()) |
| 549 | return false; |
| 550 | |
| 551 | auto Ret = next_nodbg(Next, MBB.instr_end()); |
| 552 | if (Ret == MBB.instr_end() || !Ret->isReturn()) |
| 553 | return false; |
| 554 | |
| 555 | if (Ret->getNumImplicitOperands() != 1) |
| 556 | return false; |
| 557 | |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 558 | if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg()) |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 559 | return false; |
| 560 | |
| 561 | // Skip over the COPY that we just validated. |
| 562 | Next = Ret; |
| 563 | } |
| 564 | |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 565 | if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn()) |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 566 | return false; |
| 567 | |
| 568 | return true; |
| 569 | } |
| 570 | |
Diana Picus | fc1675e | 2017-07-05 12:57:24 +0000 | [diff] [blame] | 571 | LegalizerHelper::LegalizeResult |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 572 | llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, |
Diana Picus | fc1675e | 2017-07-05 12:57:24 +0000 | [diff] [blame] | 573 | const CallLowering::ArgInfo &Result, |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 574 | ArrayRef<CallLowering::ArgInfo> Args, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 575 | const CallingConv::ID CC, LostDebugLocObserver &LocObserver, |
| 576 | MachineInstr *MI) { |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 577 | auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); |
Diana Picus | d0104ea | 2017-07-06 09:09:33 +0000 | [diff] [blame] | 578 | |
Tim Northover | e1a5f66 | 2019-08-09 08:26:38 +0000 | [diff] [blame] | 579 | CallLowering::CallLoweringInfo Info; |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 580 | Info.CallConv = CC; |
Tim Northover | e1a5f66 | 2019-08-09 08:26:38 +0000 | [diff] [blame] | 581 | Info.Callee = MachineOperand::CreateES(Name); |
| 582 | Info.OrigRet = Result; |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 583 | if (MI) |
| 584 | Info.IsTailCall = |
| 585 | (Result.Ty->isVoidTy() || |
| 586 | Result.Ty == MIRBuilder.getMF().getFunction().getReturnType()) && |
| 587 | isLibCallInTailPosition(Result, *MI, MIRBuilder.getTII(), |
| 588 | *MIRBuilder.getMRI()); |
| 589 | |
Tim Northover | e1a5f66 | 2019-08-09 08:26:38 +0000 | [diff] [blame] | 590 | std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); |
| 591 | if (!CLI.lowerCall(MIRBuilder, Info)) |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 592 | return LegalizerHelper::UnableToLegalize; |
Diana Picus | d0104ea | 2017-07-06 09:09:33 +0000 | [diff] [blame] | 593 | |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 594 | if (MI && Info.LoweredTailCall) { |
| 595 | assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"); |
| 596 | |
| 597 | // Check debug locations before removing the return. |
| 598 | LocObserver.checkpoint(true); |
| 599 | |
| 600 | // We must have a return following the call (or debug insts) to get past |
| 601 | // isLibCallInTailPosition. |
| 602 | do { |
| 603 | MachineInstr *Next = MI->getNextNode(); |
| 604 | assert(Next && |
| 605 | (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) && |
| 606 | "Expected instr following MI to be return or debug inst?"); |
| 607 | // We lowered a tail call, so the call is now the return from the block. |
| 608 | // Delete the old return. |
| 609 | Next->eraseFromParent(); |
| 610 | } while (MI->getNextNode()); |
| 611 | |
| 612 | // We expect to lose the debug location from the return. |
| 613 | LocObserver.checkpoint(false); |
| 614 | } |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 615 | return LegalizerHelper::Legalized; |
| 616 | } |
| 617 | |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 618 | LegalizerHelper::LegalizeResult |
| 619 | llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, |
| 620 | const CallLowering::ArgInfo &Result, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 621 | ArrayRef<CallLowering::ArgInfo> Args, |
| 622 | LostDebugLocObserver &LocObserver, MachineInstr *MI) { |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 623 | auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); |
| 624 | const char *Name = TLI.getLibcallName(Libcall); |
David Green | 47c65cf | 2024-02-17 08:57:14 +0000 | [diff] [blame] | 625 | if (!Name) |
| 626 | return LegalizerHelper::UnableToLegalize; |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 627 | const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall); |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 628 | return createLibcall(MIRBuilder, Name, Result, Args, CC, LocObserver, MI); |
Dominik Montada | 9fedb69 | 2020-03-26 13:59:08 +0100 | [diff] [blame] | 629 | } |
| 630 | |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 631 | // Useful for libcalls where all operands have the same type. |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 632 | static LegalizerHelper::LegalizeResult |
| 633 | simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 634 | Type *OpType, LostDebugLocObserver &LocObserver) { |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 635 | auto Libcall = getRTLibDesc(MI.getOpcode(), Size); |
Diana Picus | e74243d | 2018-01-12 11:30:45 +0000 | [diff] [blame] | 636 | |
Matt Arsenault | 9b057f6 | 2021-07-08 11:26:30 -0400 | [diff] [blame] | 637 | // FIXME: What does the original arg index mean here? |
Diana Picus | e74243d | 2018-01-12 11:30:45 +0000 | [diff] [blame] | 638 | SmallVector<CallLowering::ArgInfo, 3> Args; |
Kazu Hirata | 259cd6f | 2021-11-25 22:17:10 -0800 | [diff] [blame] | 639 | for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) |
| 640 | Args.push_back({MO.getReg(), OpType, 0}); |
Matt Arsenault | 9b057f6 | 2021-07-08 11:26:30 -0400 | [diff] [blame] | 641 | return createLibcall(MIRBuilder, Libcall, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 642 | {MI.getOperand(0).getReg(), OpType, 0}, Args, |
| 643 | LocObserver, &MI); |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 644 | } |
| 645 | |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 646 | LegalizerHelper::LegalizeResult |
| 647 | llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 648 | MachineInstr &MI, LostDebugLocObserver &LocObserver) { |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 649 | auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
| 650 | |
| 651 | SmallVector<CallLowering::ArgInfo, 3> Args; |
Amara Emerson | 509a494 | 2019-09-28 05:33:21 +0000 | [diff] [blame] | 652 | // Add all the args, except for the last which is an imm denoting 'tail'. |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 653 | for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) { |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 654 | Register Reg = MI.getOperand(i).getReg(); |
| 655 | |
| 656 | // Need derive an IR type for call lowering. |
| 657 | LLT OpLLT = MRI.getType(Reg); |
| 658 | Type *OpTy = nullptr; |
| 659 | if (OpLLT.isPointer()) |
Bjorn Pettersson | a7ee80f | 2023-08-11 14:38:53 +0200 | [diff] [blame] | 660 | OpTy = PointerType::get(Ctx, OpLLT.getAddressSpace()); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 661 | else |
| 662 | OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits()); |
Matt Arsenault | 9b057f6 | 2021-07-08 11:26:30 -0400 | [diff] [blame] | 663 | Args.push_back({Reg, OpTy, 0}); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 664 | } |
| 665 | |
| 666 | auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); |
| 667 | auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 668 | RTLIB::Libcall RTLibcall; |
Jessica Paquette | 23f657c | 2021-03-24 23:45:36 -0700 | [diff] [blame] | 669 | unsigned Opc = MI.getOpcode(); |
| 670 | switch (Opc) { |
| 671 | case TargetOpcode::G_BZERO: |
| 672 | RTLibcall = RTLIB::BZERO; |
| 673 | break; |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 674 | case TargetOpcode::G_MEMCPY: |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 675 | RTLibcall = RTLIB::MEMCPY; |
Jon Roelofs | afaf928 | 2021-07-02 13:08:57 -0700 | [diff] [blame] | 676 | Args[0].Flags[0].setReturned(); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 677 | break; |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 678 | case TargetOpcode::G_MEMMOVE: |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 679 | RTLibcall = RTLIB::MEMMOVE; |
Jon Roelofs | afaf928 | 2021-07-02 13:08:57 -0700 | [diff] [blame] | 680 | Args[0].Flags[0].setReturned(); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 681 | break; |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 682 | case TargetOpcode::G_MEMSET: |
| 683 | RTLibcall = RTLIB::MEMSET; |
Jon Roelofs | afaf928 | 2021-07-02 13:08:57 -0700 | [diff] [blame] | 684 | Args[0].Flags[0].setReturned(); |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 685 | break; |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 686 | default: |
Jon Roelofs | afaf928 | 2021-07-02 13:08:57 -0700 | [diff] [blame] | 687 | llvm_unreachable("unsupported opcode"); |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 688 | } |
| 689 | const char *Name = TLI.getLibcallName(RTLibcall); |
| 690 | |
Jessica Paquette | 23f657c | 2021-03-24 23:45:36 -0700 | [diff] [blame] | 691 | // Unsupported libcall on the target. |
| 692 | if (!Name) { |
| 693 | LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for " |
| 694 | << MIRBuilder.getTII().getName(Opc) << "\n"); |
| 695 | return LegalizerHelper::UnableToLegalize; |
| 696 | } |
| 697 | |
Tim Northover | e1a5f66 | 2019-08-09 08:26:38 +0000 | [diff] [blame] | 698 | CallLowering::CallLoweringInfo Info; |
| 699 | Info.CallConv = TLI.getLibcallCallingConv(RTLibcall); |
| 700 | Info.Callee = MachineOperand::CreateES(Name); |
Matt Arsenault | 9b057f6 | 2021-07-08 11:26:30 -0400 | [diff] [blame] | 701 | Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0); |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 702 | Info.IsTailCall = |
| 703 | MI.getOperand(MI.getNumOperands() - 1).getImm() && |
| 704 | isLibCallInTailPosition(Info.OrigRet, MI, MIRBuilder.getTII(), MRI); |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 705 | |
Tim Northover | e1a5f66 | 2019-08-09 08:26:38 +0000 | [diff] [blame] | 706 | std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); |
| 707 | if (!CLI.lowerCall(MIRBuilder, Info)) |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 708 | return LegalizerHelper::UnableToLegalize; |
| 709 | |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 710 | if (Info.LoweredTailCall) { |
| 711 | assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?"); |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 712 | |
| 713 | // Check debug locations before removing the return. |
| 714 | LocObserver.checkpoint(true); |
| 715 | |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 716 | // We must have a return following the call (or debug insts) to get past |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 717 | // isLibCallInTailPosition. |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 718 | do { |
| 719 | MachineInstr *Next = MI.getNextNode(); |
Jon Roelofs | a14b4e3 | 2021-07-06 08:28:11 -0700 | [diff] [blame] | 720 | assert(Next && |
| 721 | (Next->isCopy() || Next->isReturn() || Next->isDebugInstr()) && |
Vedant Kumar | f1a71b5 | 2020-04-16 15:23:57 -0700 | [diff] [blame] | 722 | "Expected instr following MI to be return or debug inst?"); |
| 723 | // We lowered a tail call, so the call is now the return from the block. |
| 724 | // Delete the old return. |
| 725 | Next->eraseFromParent(); |
| 726 | } while (MI.getNextNode()); |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 727 | |
| 728 | // We expect to lose the debug location from the return. |
| 729 | LocObserver.checkpoint(false); |
Jessica Paquette | 727328a | 2019-09-13 20:25:58 +0000 | [diff] [blame] | 730 | } |
| 731 | |
Amara Emerson | cf12c78 | 2019-07-19 00:24:45 +0000 | [diff] [blame] | 732 | return LegalizerHelper::Legalized; |
| 733 | } |
| 734 | |
Thomas Preud'homme | ce61b0e | 2024-01-04 10:15:16 +0000 | [diff] [blame] | 735 | static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) { |
| 736 | unsigned Opc = MI.getOpcode(); |
| 737 | auto &AtomicMI = cast<GMemOperation>(MI); |
| 738 | auto &MMO = AtomicMI.getMMO(); |
| 739 | auto Ordering = MMO.getMergedOrdering(); |
| 740 | LLT MemType = MMO.getMemoryType(); |
| 741 | uint64_t MemSize = MemType.getSizeInBytes(); |
| 742 | if (MemType.isVector()) |
| 743 | return RTLIB::UNKNOWN_LIBCALL; |
| 744 | |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 745 | #define LCALLS(A, B) {A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL} |
Thomas Preud'homme | ce61b0e | 2024-01-04 10:15:16 +0000 | [diff] [blame] | 746 | #define LCALL5(A) \ |
| 747 | LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16) |
| 748 | switch (Opc) { |
| 749 | case TargetOpcode::G_ATOMIC_CMPXCHG: |
| 750 | case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { |
| 751 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)}; |
| 752 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 753 | } |
| 754 | case TargetOpcode::G_ATOMICRMW_XCHG: { |
| 755 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)}; |
| 756 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 757 | } |
| 758 | case TargetOpcode::G_ATOMICRMW_ADD: |
| 759 | case TargetOpcode::G_ATOMICRMW_SUB: { |
| 760 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)}; |
| 761 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 762 | } |
| 763 | case TargetOpcode::G_ATOMICRMW_AND: { |
| 764 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)}; |
| 765 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 766 | } |
| 767 | case TargetOpcode::G_ATOMICRMW_OR: { |
| 768 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)}; |
| 769 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 770 | } |
| 771 | case TargetOpcode::G_ATOMICRMW_XOR: { |
| 772 | const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)}; |
| 773 | return getOutlineAtomicHelper(LC, Ordering, MemSize); |
| 774 | } |
| 775 | default: |
| 776 | return RTLIB::UNKNOWN_LIBCALL; |
| 777 | } |
| 778 | #undef LCALLS |
| 779 | #undef LCALL5 |
| 780 | } |
| 781 | |
| 782 | static LegalizerHelper::LegalizeResult |
| 783 | createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) { |
| 784 | auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
| 785 | |
| 786 | Type *RetTy; |
| 787 | SmallVector<Register> RetRegs; |
| 788 | SmallVector<CallLowering::ArgInfo, 3> Args; |
| 789 | unsigned Opc = MI.getOpcode(); |
| 790 | switch (Opc) { |
| 791 | case TargetOpcode::G_ATOMIC_CMPXCHG: |
| 792 | case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { |
| 793 | Register Success; |
| 794 | LLT SuccessLLT; |
| 795 | auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] = |
| 796 | MI.getFirst4RegLLTs(); |
| 797 | RetRegs.push_back(Ret); |
| 798 | RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits()); |
| 799 | if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) { |
| 800 | std::tie(Ret, RetLLT, Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New, |
| 801 | NewLLT) = MI.getFirst5RegLLTs(); |
| 802 | RetRegs.push_back(Success); |
| 803 | RetTy = StructType::get( |
| 804 | Ctx, {RetTy, IntegerType::get(Ctx, SuccessLLT.getSizeInBits())}); |
| 805 | } |
| 806 | Args.push_back({Cmp, IntegerType::get(Ctx, CmpLLT.getSizeInBits()), 0}); |
| 807 | Args.push_back({New, IntegerType::get(Ctx, NewLLT.getSizeInBits()), 0}); |
| 808 | Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0}); |
| 809 | break; |
| 810 | } |
| 811 | case TargetOpcode::G_ATOMICRMW_XCHG: |
| 812 | case TargetOpcode::G_ATOMICRMW_ADD: |
| 813 | case TargetOpcode::G_ATOMICRMW_SUB: |
| 814 | case TargetOpcode::G_ATOMICRMW_AND: |
| 815 | case TargetOpcode::G_ATOMICRMW_OR: |
| 816 | case TargetOpcode::G_ATOMICRMW_XOR: { |
| 817 | auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs(); |
| 818 | RetRegs.push_back(Ret); |
| 819 | RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits()); |
| 820 | if (Opc == TargetOpcode::G_ATOMICRMW_AND) |
| 821 | Val = |
| 822 | MIRBuilder.buildXor(ValLLT, MIRBuilder.buildConstant(ValLLT, -1), Val) |
| 823 | .getReg(0); |
| 824 | else if (Opc == TargetOpcode::G_ATOMICRMW_SUB) |
| 825 | Val = |
| 826 | MIRBuilder.buildSub(ValLLT, MIRBuilder.buildConstant(ValLLT, 0), Val) |
| 827 | .getReg(0); |
| 828 | Args.push_back({Val, IntegerType::get(Ctx, ValLLT.getSizeInBits()), 0}); |
| 829 | Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0}); |
| 830 | break; |
| 831 | } |
| 832 | default: |
| 833 | llvm_unreachable("unsupported opcode"); |
| 834 | } |
| 835 | |
| 836 | auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); |
| 837 | auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); |
| 838 | RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI); |
| 839 | const char *Name = TLI.getLibcallName(RTLibcall); |
| 840 | |
| 841 | // Unsupported libcall on the target. |
| 842 | if (!Name) { |
| 843 | LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for " |
| 844 | << MIRBuilder.getTII().getName(Opc) << "\n"); |
| 845 | return LegalizerHelper::UnableToLegalize; |
| 846 | } |
| 847 | |
| 848 | CallLowering::CallLoweringInfo Info; |
| 849 | Info.CallConv = TLI.getLibcallCallingConv(RTLibcall); |
| 850 | Info.Callee = MachineOperand::CreateES(Name); |
| 851 | Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0); |
| 852 | |
| 853 | std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs)); |
| 854 | if (!CLI.lowerCall(MIRBuilder, Info)) |
| 855 | return LegalizerHelper::UnableToLegalize; |
| 856 | |
| 857 | return LegalizerHelper::Legalized; |
| 858 | } |
| 859 | |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 860 | static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, |
| 861 | Type *FromType) { |
| 862 | auto ToMVT = MVT::getVT(ToType); |
| 863 | auto FromMVT = MVT::getVT(FromType); |
| 864 | |
| 865 | switch (Opcode) { |
| 866 | case TargetOpcode::G_FPEXT: |
| 867 | return RTLIB::getFPEXT(FromMVT, ToMVT); |
| 868 | case TargetOpcode::G_FPTRUNC: |
| 869 | return RTLIB::getFPROUND(FromMVT, ToMVT); |
Diana Picus | 4ed0ee7 | 2018-01-30 07:54:52 +0000 | [diff] [blame] | 870 | case TargetOpcode::G_FPTOSI: |
| 871 | return RTLIB::getFPTOSINT(FromMVT, ToMVT); |
| 872 | case TargetOpcode::G_FPTOUI: |
| 873 | return RTLIB::getFPTOUINT(FromMVT, ToMVT); |
Diana Picus | 517531e | 2018-01-30 09:15:17 +0000 | [diff] [blame] | 874 | case TargetOpcode::G_SITOFP: |
| 875 | return RTLIB::getSINTTOFP(FromMVT, ToMVT); |
| 876 | case TargetOpcode::G_UITOFP: |
| 877 | return RTLIB::getUINTTOFP(FromMVT, ToMVT); |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 878 | } |
| 879 | llvm_unreachable("Unsupported libcall function"); |
| 880 | } |
| 881 | |
| 882 | static LegalizerHelper::LegalizeResult |
| 883 | conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, |
Craig Topper | 4cf2cf1 | 2024-12-04 10:42:49 -0800 | [diff] [blame] | 884 | Type *FromType, LostDebugLocObserver &LocObserver, |
| 885 | const TargetLowering &TLI, bool IsSigned = false) { |
| 886 | CallLowering::ArgInfo Arg = {MI.getOperand(1).getReg(), FromType, 0}; |
| 887 | if (FromType->isIntegerTy()) { |
| 888 | if (TLI.shouldSignExtendTypeInLibCall(FromType, IsSigned)) |
| 889 | Arg.Flags[0].setSExt(); |
| 890 | else |
| 891 | Arg.Flags[0].setZExt(); |
| 892 | } |
| 893 | |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 894 | RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType); |
Craig Topper | 4cf2cf1 | 2024-12-04 10:42:49 -0800 | [diff] [blame] | 895 | return createLibcall(MIRBuilder, Libcall, |
| 896 | {MI.getOperand(0).getReg(), ToType, 0}, Arg, LocObserver, |
| 897 | &MI); |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 898 | } |
| 899 | |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 900 | static RTLIB::Libcall |
| 901 | getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) { |
| 902 | RTLIB::Libcall RTLibcall; |
| 903 | switch (MI.getOpcode()) { |
Serge Pavlov | 7fc7ef1 | 2024-01-10 14:18:00 +0700 | [diff] [blame] | 904 | case TargetOpcode::G_GET_FPENV: |
| 905 | RTLibcall = RTLIB::FEGETENV; |
| 906 | break; |
| 907 | case TargetOpcode::G_SET_FPENV: |
| 908 | case TargetOpcode::G_RESET_FPENV: |
| 909 | RTLibcall = RTLIB::FESETENV; |
| 910 | break; |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 911 | case TargetOpcode::G_GET_FPMODE: |
| 912 | RTLibcall = RTLIB::FEGETMODE; |
| 913 | break; |
| 914 | case TargetOpcode::G_SET_FPMODE: |
| 915 | case TargetOpcode::G_RESET_FPMODE: |
| 916 | RTLibcall = RTLIB::FESETMODE; |
| 917 | break; |
| 918 | default: |
| 919 | llvm_unreachable("Unexpected opcode"); |
| 920 | } |
| 921 | return RTLibcall; |
| 922 | } |
| 923 | |
| 924 | // Some library functions that read FP state (fegetmode, fegetenv) write the |
| 925 | // state into a region in memory. IR intrinsics that do the same operations |
| 926 | // (get_fpmode, get_fpenv) return the state as integer value. To implement these |
| 927 | // intrinsics via the library functions, we need to use temporary variable, |
| 928 | // for example: |
| 929 | // |
| 930 | // %0:_(s32) = G_GET_FPMODE |
| 931 | // |
| 932 | // is transformed to: |
| 933 | // |
| 934 | // %1:_(p0) = G_FRAME_INDEX %stack.0 |
| 935 | // BL &fegetmode |
| 936 | // %0:_(s32) = G_LOAD % 1 |
| 937 | // |
| 938 | LegalizerHelper::LegalizeResult |
| 939 | LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 940 | MachineInstr &MI, |
| 941 | LostDebugLocObserver &LocObserver) { |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 942 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 943 | auto &MF = MIRBuilder.getMF(); |
| 944 | auto &MRI = *MIRBuilder.getMRI(); |
| 945 | auto &Ctx = MF.getFunction().getContext(); |
| 946 | |
| 947 | // Create temporary, where library function will put the read state. |
| 948 | Register Dst = MI.getOperand(0).getReg(); |
| 949 | LLT StateTy = MRI.getType(Dst); |
| 950 | TypeSize StateSize = StateTy.getSizeInBytes(); |
| 951 | Align TempAlign = getStackTemporaryAlignment(StateTy); |
| 952 | MachinePointerInfo TempPtrInfo; |
| 953 | auto Temp = createStackTemporary(StateSize, TempAlign, TempPtrInfo); |
| 954 | |
| 955 | // Create a call to library function, with the temporary as an argument. |
| 956 | unsigned TempAddrSpace = DL.getAllocaAddrSpace(); |
| 957 | Type *StatePtrTy = PointerType::get(Ctx, TempAddrSpace); |
| 958 | RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI); |
| 959 | auto Res = |
| 960 | createLibcall(MIRBuilder, RTLibcall, |
| 961 | CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0), |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 962 | CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}), |
| 963 | LocObserver, nullptr); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 964 | if (Res != LegalizerHelper::Legalized) |
| 965 | return Res; |
| 966 | |
| 967 | // Create a load from the temporary. |
| 968 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
| 969 | TempPtrInfo, MachineMemOperand::MOLoad, StateTy, TempAlign); |
| 970 | MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, Dst, Temp, *MMO); |
| 971 | |
| 972 | return LegalizerHelper::Legalized; |
| 973 | } |
| 974 | |
| 975 | // Similar to `createGetStateLibcall` the function calls a library function |
| 976 | // using transient space in stack. In this case the library function reads |
| 977 | // content of memory region. |
| 978 | LegalizerHelper::LegalizeResult |
| 979 | LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 980 | MachineInstr &MI, |
| 981 | LostDebugLocObserver &LocObserver) { |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 982 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 983 | auto &MF = MIRBuilder.getMF(); |
| 984 | auto &MRI = *MIRBuilder.getMRI(); |
| 985 | auto &Ctx = MF.getFunction().getContext(); |
| 986 | |
| 987 | // Create temporary, where library function will get the new state. |
| 988 | Register Src = MI.getOperand(0).getReg(); |
| 989 | LLT StateTy = MRI.getType(Src); |
| 990 | TypeSize StateSize = StateTy.getSizeInBytes(); |
| 991 | Align TempAlign = getStackTemporaryAlignment(StateTy); |
| 992 | MachinePointerInfo TempPtrInfo; |
| 993 | auto Temp = createStackTemporary(StateSize, TempAlign, TempPtrInfo); |
| 994 | |
| 995 | // Put the new state into the temporary. |
| 996 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
| 997 | TempPtrInfo, MachineMemOperand::MOStore, StateTy, TempAlign); |
| 998 | MIRBuilder.buildStore(Src, Temp, *MMO); |
| 999 | |
| 1000 | // Create a call to library function, with the temporary as an argument. |
| 1001 | unsigned TempAddrSpace = DL.getAllocaAddrSpace(); |
| 1002 | Type *StatePtrTy = PointerType::get(Ctx, TempAddrSpace); |
| 1003 | RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI); |
| 1004 | return createLibcall(MIRBuilder, RTLibcall, |
| 1005 | CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0), |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1006 | CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}), |
| 1007 | LocObserver, nullptr); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1008 | } |
| 1009 | |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1010 | /// Returns the corresponding libcall for the given Pred and |
| 1011 | /// the ICMP predicate that should be generated to compare with #0 |
| 1012 | /// after the libcall. |
| 1013 | static std::pair<RTLIB::Libcall, CmpInst::Predicate> |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1014 | getFCMPLibcallDesc(const CmpInst::Predicate Pred, unsigned Size) { |
| 1015 | #define RTLIBCASE_CMP(LibcallPrefix, ICmpPred) \ |
| 1016 | do { \ |
| 1017 | switch (Size) { \ |
| 1018 | case 32: \ |
| 1019 | return {RTLIB::LibcallPrefix##32, ICmpPred}; \ |
| 1020 | case 64: \ |
| 1021 | return {RTLIB::LibcallPrefix##64, ICmpPred}; \ |
| 1022 | case 128: \ |
| 1023 | return {RTLIB::LibcallPrefix##128, ICmpPred}; \ |
| 1024 | default: \ |
| 1025 | llvm_unreachable("unexpected size"); \ |
| 1026 | } \ |
| 1027 | } while (0) |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1028 | |
| 1029 | switch (Pred) { |
| 1030 | case CmpInst::FCMP_OEQ: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1031 | RTLIBCASE_CMP(OEQ_F, CmpInst::ICMP_EQ); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1032 | case CmpInst::FCMP_UNE: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1033 | RTLIBCASE_CMP(UNE_F, CmpInst::ICMP_NE); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1034 | case CmpInst::FCMP_OGE: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1035 | RTLIBCASE_CMP(OGE_F, CmpInst::ICMP_SGE); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1036 | case CmpInst::FCMP_OLT: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1037 | RTLIBCASE_CMP(OLT_F, CmpInst::ICMP_SLT); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1038 | case CmpInst::FCMP_OLE: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1039 | RTLIBCASE_CMP(OLE_F, CmpInst::ICMP_SLE); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1040 | case CmpInst::FCMP_OGT: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1041 | RTLIBCASE_CMP(OGT_F, CmpInst::ICMP_SGT); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1042 | case CmpInst::FCMP_UNO: |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1043 | RTLIBCASE_CMP(UO_F, CmpInst::ICMP_NE); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1044 | default: |
| 1045 | return {RTLIB::UNKNOWN_LIBCALL, CmpInst::BAD_ICMP_PREDICATE}; |
| 1046 | } |
| 1047 | } |
| 1048 | |
| 1049 | LegalizerHelper::LegalizeResult |
| 1050 | LegalizerHelper::createFCMPLibcall(MachineIRBuilder &MIRBuilder, |
| 1051 | MachineInstr &MI, |
| 1052 | LostDebugLocObserver &LocObserver) { |
| 1053 | auto &MF = MIRBuilder.getMF(); |
| 1054 | auto &Ctx = MF.getFunction().getContext(); |
| 1055 | const GFCmp *Cmp = cast<GFCmp>(&MI); |
| 1056 | |
| 1057 | LLT OpLLT = MRI.getType(Cmp->getLHSReg()); |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1058 | unsigned Size = OpLLT.getSizeInBits(); |
| 1059 | if ((Size != 32 && Size != 64 && Size != 128) || |
| 1060 | OpLLT != MRI.getType(Cmp->getRHSReg())) |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1061 | return UnableToLegalize; |
| 1062 | |
| 1063 | Type *OpType = getFloatTypeForLLT(Ctx, OpLLT); |
| 1064 | |
| 1065 | // DstReg type is s32 |
| 1066 | const Register DstReg = Cmp->getReg(0); |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1067 | LLT DstTy = MRI.getType(DstReg); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1068 | const auto Cond = Cmp->getCond(); |
| 1069 | |
| 1070 | // Reference: |
| 1071 | // https://gcc.gnu.org/onlinedocs/gccint/Soft-float-library-routines.html#Comparison-functions-1 |
| 1072 | // Generates a libcall followed by ICMP. |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1073 | const auto BuildLibcall = [&](const RTLIB::Libcall Libcall, |
| 1074 | const CmpInst::Predicate ICmpPred, |
| 1075 | const DstOp &Res) -> Register { |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1076 | // FCMP libcall always returns an i32, and needs an ICMP with #0. |
| 1077 | constexpr LLT TempLLT = LLT::scalar(32); |
| 1078 | Register Temp = MRI.createGenericVirtualRegister(TempLLT); |
| 1079 | // Generate libcall, holding result in Temp |
| 1080 | const auto Status = createLibcall( |
| 1081 | MIRBuilder, Libcall, {Temp, Type::getInt32Ty(Ctx), 0}, |
| 1082 | {{Cmp->getLHSReg(), OpType, 0}, {Cmp->getRHSReg(), OpType, 1}}, |
| 1083 | LocObserver, &MI); |
| 1084 | if (!Status) |
| 1085 | return {}; |
| 1086 | |
| 1087 | // Compare temp with #0 to get the final result. |
| 1088 | return MIRBuilder |
| 1089 | .buildICmp(ICmpPred, Res, Temp, MIRBuilder.buildConstant(TempLLT, 0)) |
| 1090 | .getReg(0); |
| 1091 | }; |
| 1092 | |
| 1093 | // Simple case if we have a direct mapping from predicate to libcall |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1094 | if (const auto [Libcall, ICmpPred] = getFCMPLibcallDesc(Cond, Size); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1095 | Libcall != RTLIB::UNKNOWN_LIBCALL && |
| 1096 | ICmpPred != CmpInst::BAD_ICMP_PREDICATE) { |
| 1097 | if (BuildLibcall(Libcall, ICmpPred, DstReg)) { |
| 1098 | return Legalized; |
| 1099 | } |
| 1100 | return UnableToLegalize; |
| 1101 | } |
| 1102 | |
| 1103 | // No direct mapping found, should be generated as combination of libcalls. |
| 1104 | |
| 1105 | switch (Cond) { |
| 1106 | case CmpInst::FCMP_UEQ: { |
| 1107 | // FCMP_UEQ: unordered or equal |
| 1108 | // Convert into (FCMP_OEQ || FCMP_UNO). |
| 1109 | |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1110 | const auto [OeqLibcall, OeqPred] = |
| 1111 | getFCMPLibcallDesc(CmpInst::FCMP_OEQ, Size); |
| 1112 | const auto Oeq = BuildLibcall(OeqLibcall, OeqPred, DstTy); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1113 | |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1114 | const auto [UnoLibcall, UnoPred] = |
| 1115 | getFCMPLibcallDesc(CmpInst::FCMP_UNO, Size); |
| 1116 | const auto Uno = BuildLibcall(UnoLibcall, UnoPred, DstTy); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1117 | if (Oeq && Uno) |
| 1118 | MIRBuilder.buildOr(DstReg, Oeq, Uno); |
| 1119 | else |
| 1120 | return UnableToLegalize; |
| 1121 | |
| 1122 | break; |
| 1123 | } |
| 1124 | case CmpInst::FCMP_ONE: { |
| 1125 | // FCMP_ONE: ordered and operands are unequal |
| 1126 | // Convert into (!FCMP_OEQ && !FCMP_UNO). |
| 1127 | |
| 1128 | // We inverse the predicate instead of generating a NOT |
| 1129 | // to save one instruction. |
| 1130 | // On AArch64 isel can even select two cmp into a single ccmp. |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1131 | const auto [OeqLibcall, OeqPred] = |
| 1132 | getFCMPLibcallDesc(CmpInst::FCMP_OEQ, Size); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1133 | const auto NotOeq = |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1134 | BuildLibcall(OeqLibcall, CmpInst::getInversePredicate(OeqPred), DstTy); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1135 | |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1136 | const auto [UnoLibcall, UnoPred] = |
| 1137 | getFCMPLibcallDesc(CmpInst::FCMP_UNO, Size); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1138 | const auto NotUno = |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1139 | BuildLibcall(UnoLibcall, CmpInst::getInversePredicate(UnoPred), DstTy); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1140 | |
| 1141 | if (NotOeq && NotUno) |
| 1142 | MIRBuilder.buildAnd(DstReg, NotOeq, NotUno); |
| 1143 | else |
| 1144 | return UnableToLegalize; |
| 1145 | |
| 1146 | break; |
| 1147 | } |
| 1148 | case CmpInst::FCMP_ULT: |
| 1149 | case CmpInst::FCMP_UGE: |
| 1150 | case CmpInst::FCMP_UGT: |
| 1151 | case CmpInst::FCMP_ULE: |
| 1152 | case CmpInst::FCMP_ORD: { |
| 1153 | // Convert into: !(inverse(Pred)) |
| 1154 | // E.g. FCMP_ULT becomes !FCMP_OGE |
| 1155 | // This is equivalent to the following, but saves some instructions. |
| 1156 | // MIRBuilder.buildNot( |
| 1157 | // PredTy, |
| 1158 | // MIRBuilder.buildFCmp(CmpInst::getInversePredicate(Pred), PredTy, |
| 1159 | // Op1, Op2)); |
| 1160 | const auto [InversedLibcall, InversedPred] = |
Craig Topper | 43b6b78 | 2024-11-26 15:48:49 -0800 | [diff] [blame] | 1161 | getFCMPLibcallDesc(CmpInst::getInversePredicate(Cond), Size); |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1162 | if (!BuildLibcall(InversedLibcall, |
| 1163 | CmpInst::getInversePredicate(InversedPred), DstReg)) |
| 1164 | return UnableToLegalize; |
| 1165 | break; |
| 1166 | } |
| 1167 | default: |
| 1168 | return UnableToLegalize; |
| 1169 | } |
| 1170 | |
| 1171 | return Legalized; |
| 1172 | } |
| 1173 | |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1174 | // The function is used to legalize operations that set default environment |
| 1175 | // state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that. |
| 1176 | // On most targets supported in glibc FE_DFL_MODE is defined as |
| 1177 | // `((const femode_t *) -1)`. Such assumption is used here. If for some target |
| 1178 | // it is not true, the target must provide custom lowering. |
| 1179 | LegalizerHelper::LegalizeResult |
| 1180 | LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder, |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1181 | MachineInstr &MI, |
| 1182 | LostDebugLocObserver &LocObserver) { |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1183 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 1184 | auto &MF = MIRBuilder.getMF(); |
| 1185 | auto &Ctx = MF.getFunction().getContext(); |
| 1186 | |
| 1187 | // Create an argument for the library function. |
| 1188 | unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace(); |
| 1189 | Type *StatePtrTy = PointerType::get(Ctx, AddrSpace); |
| 1190 | unsigned PtrSize = DL.getPointerSizeInBits(AddrSpace); |
| 1191 | LLT MemTy = LLT::pointer(AddrSpace, PtrSize); |
| 1192 | auto DefValue = MIRBuilder.buildConstant(LLT::scalar(PtrSize), -1LL); |
| 1193 | DstOp Dest(MRI.createGenericVirtualRegister(MemTy)); |
| 1194 | MIRBuilder.buildIntToPtr(Dest, DefValue); |
| 1195 | |
| 1196 | RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI); |
| 1197 | return createLibcall(MIRBuilder, RTLibcall, |
| 1198 | CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0), |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1199 | CallLowering::ArgInfo({Dest.getReg(), StatePtrTy, 0}), |
| 1200 | LocObserver, &MI); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1201 | } |
| 1202 | |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 1203 | LegalizerHelper::LegalizeResult |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 1204 | LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { |
Matthias Braun | f1caa28 | 2017-12-15 22:22:58 +0000 | [diff] [blame] | 1205 | auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 1206 | |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 1207 | switch (MI.getOpcode()) { |
| 1208 | default: |
| 1209 | return UnableToLegalize; |
Kai Nacke | b383753 | 2022-08-02 13:12:38 -0400 | [diff] [blame] | 1210 | case TargetOpcode::G_MUL: |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 1211 | case TargetOpcode::G_SDIV: |
Diana Picus | 02e1101 | 2017-06-15 10:53:31 +0000 | [diff] [blame] | 1212 | case TargetOpcode::G_UDIV: |
| 1213 | case TargetOpcode::G_SREM: |
Diana Picus | 0528e2c | 2018-11-26 11:07:02 +0000 | [diff] [blame] | 1214 | case TargetOpcode::G_UREM: |
| 1215 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: { |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1216 | LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); |
| 1217 | unsigned Size = LLTy.getSizeInBits(); |
Petar Avramovic | 0a5e4eb | 2018-12-18 15:59:51 +0000 | [diff] [blame] | 1218 | Type *HLTy = IntegerType::get(Ctx, Size); |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1219 | auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver); |
Diana Picus | fc1675e | 2017-07-05 12:57:24 +0000 | [diff] [blame] | 1220 | if (Status != Legalized) |
| 1221 | return Status; |
| 1222 | break; |
Diana Picus | e97822e | 2017-04-24 07:22:31 +0000 | [diff] [blame] | 1223 | } |
Diana Picus | 1314a28 | 2017-04-11 10:52:34 +0000 | [diff] [blame] | 1224 | case TargetOpcode::G_FADD: |
Javed Absar | 5cde1cc | 2017-10-30 13:51:56 +0000 | [diff] [blame] | 1225 | case TargetOpcode::G_FSUB: |
Diana Picus | 9faa09b | 2017-11-23 12:44:20 +0000 | [diff] [blame] | 1226 | case TargetOpcode::G_FMUL: |
Diana Picus | c01f7f1 | 2017-11-23 13:26:07 +0000 | [diff] [blame] | 1227 | case TargetOpcode::G_FDIV: |
Diana Picus | e74243d | 2018-01-12 11:30:45 +0000 | [diff] [blame] | 1228 | case TargetOpcode::G_FMA: |
Tim Northover | e041841 | 2017-02-08 23:23:39 +0000 | [diff] [blame] | 1229 | case TargetOpcode::G_FPOW: |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 1230 | case TargetOpcode::G_FREM: |
| 1231 | case TargetOpcode::G_FCOS: |
Jessica Paquette | c49428a | 2019-01-28 19:53:14 +0000 | [diff] [blame] | 1232 | case TargetOpcode::G_FSIN: |
Farzon Lotfi | 1d87433 | 2024-06-05 15:01:33 -0400 | [diff] [blame] | 1233 | case TargetOpcode::G_FTAN: |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 1234 | case TargetOpcode::G_FACOS: |
| 1235 | case TargetOpcode::G_FASIN: |
| 1236 | case TargetOpcode::G_FATAN: |
Tex Riddell | c03d09c | 2024-10-24 17:53:12 -0700 | [diff] [blame] | 1237 | case TargetOpcode::G_FATAN2: |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 1238 | case TargetOpcode::G_FCOSH: |
| 1239 | case TargetOpcode::G_FSINH: |
| 1240 | case TargetOpcode::G_FTANH: |
Jessica Paquette | 2d73ecd | 2019-01-28 21:27:23 +0000 | [diff] [blame] | 1241 | case TargetOpcode::G_FLOG10: |
Jessica Paquette | 0154bd1 | 2019-01-30 21:16:04 +0000 | [diff] [blame] | 1242 | case TargetOpcode::G_FLOG: |
Jessica Paquette | 84bedac | 2019-01-30 23:46:15 +0000 | [diff] [blame] | 1243 | case TargetOpcode::G_FLOG2: |
Jessica Paquette | e794121 | 2019-04-03 16:58:32 +0000 | [diff] [blame] | 1244 | case TargetOpcode::G_FEXP: |
Petar Avramovic | faaa2b5d | 2019-06-06 09:02:24 +0000 | [diff] [blame] | 1245 | case TargetOpcode::G_FEXP2: |
Matt Arsenault | b14e83d | 2023-08-12 07:20:00 -0400 | [diff] [blame] | 1246 | case TargetOpcode::G_FEXP10: |
Petar Avramovic | faaa2b5d | 2019-06-06 09:02:24 +0000 | [diff] [blame] | 1247 | case TargetOpcode::G_FCEIL: |
Dominik Montada | feb20a1 | 2020-03-02 16:28:17 +0100 | [diff] [blame] | 1248 | case TargetOpcode::G_FFLOOR: |
| 1249 | case TargetOpcode::G_FMINNUM: |
| 1250 | case TargetOpcode::G_FMAXNUM: |
| 1251 | case TargetOpcode::G_FSQRT: |
| 1252 | case TargetOpcode::G_FRINT: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 1253 | case TargetOpcode::G_FNEARBYINT: |
Craig Topper | d5d1417 | 2024-09-18 12:07:44 -0700 | [diff] [blame] | 1254 | case TargetOpcode::G_INTRINSIC_TRUNC: |
| 1255 | case TargetOpcode::G_INTRINSIC_ROUND: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 1256 | case TargetOpcode::G_INTRINSIC_ROUNDEVEN: { |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1257 | LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); |
| 1258 | unsigned Size = LLTy.getSizeInBits(); |
Konstantin Schwarz | 76986bd | 2020-02-06 10:01:57 -0800 | [diff] [blame] | 1259 | Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 1260 | if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) { |
| 1261 | LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n"); |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 1262 | return UnableToLegalize; |
| 1263 | } |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1264 | auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy, LocObserver); |
Diana Picus | fc1675e | 2017-07-05 12:57:24 +0000 | [diff] [blame] | 1265 | if (Status != Legalized) |
| 1266 | return Status; |
| 1267 | break; |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 1268 | } |
David Green | 8d49ce1 | 2024-04-17 18:38:24 +0100 | [diff] [blame] | 1269 | case TargetOpcode::G_INTRINSIC_LRINT: |
| 1270 | case TargetOpcode::G_INTRINSIC_LLRINT: { |
David Green | 28d28d5 | 2024-04-15 09:41:08 +0100 | [diff] [blame] | 1271 | LLT LLTy = MRI.getType(MI.getOperand(1).getReg()); |
| 1272 | unsigned Size = LLTy.getSizeInBits(); |
| 1273 | Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); |
| 1274 | Type *ITy = IntegerType::get( |
| 1275 | Ctx, MRI.getType(MI.getOperand(0).getReg()).getSizeInBits()); |
| 1276 | if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) { |
| 1277 | LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n"); |
| 1278 | return UnableToLegalize; |
| 1279 | } |
| 1280 | auto Libcall = getRTLibDesc(MI.getOpcode(), Size); |
| 1281 | LegalizeResult Status = |
| 1282 | createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ITy, 0}, |
| 1283 | {{MI.getOperand(1).getReg(), HLTy, 0}}, LocObserver, &MI); |
| 1284 | if (Status != Legalized) |
| 1285 | return Status; |
| 1286 | MI.eraseFromParent(); |
| 1287 | return Legalized; |
| 1288 | } |
Craig Topper | a15400d | 2024-12-02 13:30:46 -0800 | [diff] [blame] | 1289 | case TargetOpcode::G_FPOWI: |
| 1290 | case TargetOpcode::G_FLDEXP: { |
David Green | 5550e9c | 2024-01-04 07:26:23 +0000 | [diff] [blame] | 1291 | LLT LLTy = MRI.getType(MI.getOperand(0).getReg()); |
| 1292 | unsigned Size = LLTy.getSizeInBits(); |
| 1293 | Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); |
| 1294 | Type *ITy = IntegerType::get( |
| 1295 | Ctx, MRI.getType(MI.getOperand(2).getReg()).getSizeInBits()); |
| 1296 | if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) { |
| 1297 | LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n"); |
| 1298 | return UnableToLegalize; |
| 1299 | } |
| 1300 | auto Libcall = getRTLibDesc(MI.getOpcode(), Size); |
Craig Topper | bee33b5 | 2024-12-02 09:06:38 -0800 | [diff] [blame] | 1301 | SmallVector<CallLowering::ArgInfo, 2> Args = { |
David Green | 5550e9c | 2024-01-04 07:26:23 +0000 | [diff] [blame] | 1302 | {MI.getOperand(1).getReg(), HLTy, 0}, |
| 1303 | {MI.getOperand(2).getReg(), ITy, 1}}; |
Craig Topper | bee33b5 | 2024-12-02 09:06:38 -0800 | [diff] [blame] | 1304 | Args[1].Flags[0].setSExt(); |
David Green | 5550e9c | 2024-01-04 07:26:23 +0000 | [diff] [blame] | 1305 | LegalizeResult Status = |
| 1306 | createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), HLTy, 0}, |
| 1307 | Args, LocObserver, &MI); |
| 1308 | if (Status != Legalized) |
| 1309 | return Status; |
| 1310 | break; |
| 1311 | } |
Konstantin Schwarz | 76986bd | 2020-02-06 10:01:57 -0800 | [diff] [blame] | 1312 | case TargetOpcode::G_FPEXT: |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 1313 | case TargetOpcode::G_FPTRUNC: { |
Konstantin Schwarz | 76986bd | 2020-02-06 10:01:57 -0800 | [diff] [blame] | 1314 | Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg())); |
| 1315 | Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg())); |
| 1316 | if (!FromTy || !ToTy) |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 1317 | return UnableToLegalize; |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1318 | LegalizeResult Status = |
Craig Topper | 4cf2cf1 | 2024-12-04 10:42:49 -0800 | [diff] [blame] | 1319 | conversionLibcall(MI, MIRBuilder, ToTy, FromTy, LocObserver, TLI); |
Diana Picus | 65ed364 | 2018-01-17 13:34:10 +0000 | [diff] [blame] | 1320 | if (Status != Legalized) |
| 1321 | return Status; |
| 1322 | break; |
| 1323 | } |
Him188 | ba461f8 | 2024-07-25 11:07:31 +0100 | [diff] [blame] | 1324 | case TargetOpcode::G_FCMP: { |
| 1325 | LegalizeResult Status = createFCMPLibcall(MIRBuilder, MI, LocObserver); |
| 1326 | if (Status != Legalized) |
| 1327 | return Status; |
| 1328 | MI.eraseFromParent(); |
| 1329 | return Status; |
| 1330 | } |
Diana Picus | 4ed0ee7 | 2018-01-30 07:54:52 +0000 | [diff] [blame] | 1331 | case TargetOpcode::G_FPTOSI: |
| 1332 | case TargetOpcode::G_FPTOUI: { |
| 1333 | // FIXME: Support other types |
David Green | e887624 | 2024-06-21 10:24:57 +0100 | [diff] [blame] | 1334 | Type *FromTy = |
| 1335 | getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg())); |
Diana Picus | 4ed0ee7 | 2018-01-30 07:54:52 +0000 | [diff] [blame] | 1336 | unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); |
David Green | e887624 | 2024-06-21 10:24:57 +0100 | [diff] [blame] | 1337 | if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy) |
Diana Picus | 4ed0ee7 | 2018-01-30 07:54:52 +0000 | [diff] [blame] | 1338 | return UnableToLegalize; |
| 1339 | LegalizeResult Status = conversionLibcall( |
Craig Topper | 4cf2cf1 | 2024-12-04 10:42:49 -0800 | [diff] [blame] | 1340 | MI, MIRBuilder, Type::getIntNTy(Ctx, ToSize), FromTy, LocObserver, TLI); |
Diana Picus | 4ed0ee7 | 2018-01-30 07:54:52 +0000 | [diff] [blame] | 1341 | if (Status != Legalized) |
| 1342 | return Status; |
| 1343 | break; |
| 1344 | } |
Diana Picus | 517531e | 2018-01-30 09:15:17 +0000 | [diff] [blame] | 1345 | case TargetOpcode::G_SITOFP: |
| 1346 | case TargetOpcode::G_UITOFP: { |
Diana Picus | 517531e | 2018-01-30 09:15:17 +0000 | [diff] [blame] | 1347 | unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); |
Him188 | 365f5b4 | 2024-07-15 16:24:24 +0100 | [diff] [blame] | 1348 | Type *ToTy = |
| 1349 | getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg())); |
| 1350 | if ((FromSize != 32 && FromSize != 64 && FromSize != 128) || !ToTy) |
Diana Picus | 517531e | 2018-01-30 09:15:17 +0000 | [diff] [blame] | 1351 | return UnableToLegalize; |
Craig Topper | 4cf2cf1 | 2024-12-04 10:42:49 -0800 | [diff] [blame] | 1352 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_SITOFP; |
| 1353 | LegalizeResult Status = |
| 1354 | conversionLibcall(MI, MIRBuilder, ToTy, Type::getIntNTy(Ctx, FromSize), |
| 1355 | LocObserver, TLI, IsSigned); |
Diana Picus | 517531e | 2018-01-30 09:15:17 +0000 | [diff] [blame] | 1356 | if (Status != Legalized) |
| 1357 | return Status; |
| 1358 | break; |
| 1359 | } |
Thomas Preud'homme | ce61b0e | 2024-01-04 10:15:16 +0000 | [diff] [blame] | 1360 | case TargetOpcode::G_ATOMICRMW_XCHG: |
| 1361 | case TargetOpcode::G_ATOMICRMW_ADD: |
| 1362 | case TargetOpcode::G_ATOMICRMW_SUB: |
| 1363 | case TargetOpcode::G_ATOMICRMW_AND: |
| 1364 | case TargetOpcode::G_ATOMICRMW_OR: |
| 1365 | case TargetOpcode::G_ATOMICRMW_XOR: |
| 1366 | case TargetOpcode::G_ATOMIC_CMPXCHG: |
| 1367 | case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { |
| 1368 | auto Status = createAtomicLibcall(MIRBuilder, MI); |
| 1369 | if (Status != Legalized) |
| 1370 | return Status; |
| 1371 | break; |
| 1372 | } |
Jessica Paquette | 23f657c | 2021-03-24 23:45:36 -0700 | [diff] [blame] | 1373 | case TargetOpcode::G_BZERO: |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 1374 | case TargetOpcode::G_MEMCPY: |
| 1375 | case TargetOpcode::G_MEMMOVE: |
| 1376 | case TargetOpcode::G_MEMSET: { |
Jessica Paquette | 23f657c | 2021-03-24 23:45:36 -0700 | [diff] [blame] | 1377 | LegalizeResult Result = |
Jessica Paquette | 324af79 | 2021-05-25 16:54:20 -0700 | [diff] [blame] | 1378 | createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI, LocObserver); |
Jessica Paquette | 23f657c | 2021-03-24 23:45:36 -0700 | [diff] [blame] | 1379 | if (Result != Legalized) |
| 1380 | return Result; |
Matt Arsenault | 0b7f6cc | 2020-08-03 09:00:24 -0400 | [diff] [blame] | 1381 | MI.eraseFromParent(); |
| 1382 | return Result; |
| 1383 | } |
Serge Pavlov | 7fc7ef1 | 2024-01-10 14:18:00 +0700 | [diff] [blame] | 1384 | case TargetOpcode::G_GET_FPENV: |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1385 | case TargetOpcode::G_GET_FPMODE: { |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1386 | LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI, LocObserver); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1387 | if (Result != Legalized) |
| 1388 | return Result; |
| 1389 | break; |
| 1390 | } |
Serge Pavlov | 7fc7ef1 | 2024-01-10 14:18:00 +0700 | [diff] [blame] | 1391 | case TargetOpcode::G_SET_FPENV: |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1392 | case TargetOpcode::G_SET_FPMODE: { |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1393 | LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI, LocObserver); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1394 | if (Result != Legalized) |
| 1395 | return Result; |
| 1396 | break; |
| 1397 | } |
Serge Pavlov | 7fc7ef1 | 2024-01-10 14:18:00 +0700 | [diff] [blame] | 1398 | case TargetOpcode::G_RESET_FPENV: |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1399 | case TargetOpcode::G_RESET_FPMODE: { |
David Green | d659bd1 | 2024-01-03 07:59:36 +0000 | [diff] [blame] | 1400 | LegalizeResult Result = |
| 1401 | createResetStateLibcall(MIRBuilder, MI, LocObserver); |
Serge Pavlov | 462d583 | 2023-10-09 21:13:39 +0700 | [diff] [blame] | 1402 | if (Result != Legalized) |
| 1403 | return Result; |
| 1404 | break; |
| 1405 | } |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 1406 | } |
Diana Picus | fc1675e | 2017-07-05 12:57:24 +0000 | [diff] [blame] | 1407 | |
| 1408 | MI.eraseFromParent(); |
| 1409 | return Legalized; |
Tim Northover | edb3c8c | 2016-08-29 19:07:16 +0000 | [diff] [blame] | 1410 | } |
| 1411 | |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 1412 | LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, |
| 1413 | unsigned TypeIdx, |
| 1414 | LLT NarrowTy) { |
Daniel Sanders | 27fe8a5 | 2018-04-27 19:48:53 +0000 | [diff] [blame] | 1415 | uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); |
| 1416 | uint64_t NarrowSize = NarrowTy.getSizeInBits(); |
Kristof Beyls | af9814a | 2017-11-07 10:34:34 +0000 | [diff] [blame] | 1417 | |
Tim Northover | 9656f14 | 2016-08-04 20:54:13 +0000 | [diff] [blame] | 1418 | switch (MI.getOpcode()) { |
| 1419 | default: |
| 1420 | return UnableToLegalize; |
Tim Northover | ff5e7e1 | 2017-06-30 20:27:36 +0000 | [diff] [blame] | 1421 | case TargetOpcode::G_IMPLICIT_DEF: { |
Dominik Montada | 35950fe | 2020-03-23 12:30:55 +0100 | [diff] [blame] | 1422 | Register DstReg = MI.getOperand(0).getReg(); |
| 1423 | LLT DstTy = MRI.getType(DstReg); |
| 1424 | |
| 1425 | // If SizeOp0 is not an exact multiple of NarrowSize, emit |
| 1426 | // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed. |
| 1427 | // FIXME: Although this would also be legal for the general case, it causes |
| 1428 | // a lot of regressions in the emitted code (superfluous COPYs, artifact |
| 1429 | // combines not being hit). This seems to be a problem related to the |
| 1430 | // artifact combiner. |
| 1431 | if (SizeOp0 % NarrowSize != 0) { |
| 1432 | LLT ImplicitTy = NarrowTy; |
| 1433 | if (DstTy.isVector()) |
Sander de Smalen | d5e14ba | 2021-06-24 09:58:21 +0100 | [diff] [blame] | 1434 | ImplicitTy = LLT::vector(DstTy.getElementCount(), ImplicitTy); |
Dominik Montada | 35950fe | 2020-03-23 12:30:55 +0100 | [diff] [blame] | 1435 | |
| 1436 | Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0); |
| 1437 | MIRBuilder.buildAnyExt(DstReg, ImplicitReg); |
| 1438 | |
| 1439 | MI.eraseFromParent(); |
| 1440 | return Legalized; |
| 1441 | } |
| 1442 | |
Kristof Beyls | af9814a | 2017-11-07 10:34:34 +0000 | [diff] [blame] | 1443 | int NumParts = SizeOp0 / NarrowSize; |
Tim Northover | ff5e7e1 | 2017-06-30 20:27:36 +0000 | [diff] [blame] | 1444 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 1445 | SmallVector<Register, 2> DstRegs; |
Volkan Keles | 02bb174 | 2018-02-14 19:58:36 +0000 | [diff] [blame] | 1446 | for (int i = 0; i < NumParts; ++i) |
Dominik Montada | 35950fe | 2020-03-23 12:30:55 +0100 | [diff] [blame] | 1447 | DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0)); |
Amara Emerson | 5ec14604 | 2018-12-10 18:44:58 +0000 | [diff] [blame] | 1448 | |
Dominik Montada | 35950fe | 2020-03-23 12:30:55 +0100 | [diff] [blame] | 1449 | if (DstTy.isVector()) |
Amara Emerson | 5ec14604 | 2018-12-10 18:44:58 +0000 | [diff] [blame] | 1450 | MIRBuilder.buildBuildVector(DstReg, DstRegs); |
| 1451 | else |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 1452 | MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); |
Tim Northover | ff5e7e1 | 2017-06-30 20:27:36 +0000 | [diff] [blame] | 1453 | MI.eraseFromParent(); |
| 1454 | return Legalized; |
| 1455 | } |
Matt Arsenault | 7187272 | 2019-04-10 17:27:53 +0000 | [diff] [blame] | 1456 | case TargetOpcode::G_CONSTANT: { |
| 1457 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
| 1458 | const APInt &Val = MI.getOperand(1).getCImm()->getValue(); |
| 1459 | unsigned TotalSize = Ty.getSizeInBits(); |
| 1460 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
| 1461 | int NumParts = TotalSize / NarrowSize; |
| 1462 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 1463 | SmallVector<Register, 4> PartRegs; |
Matt Arsenault | 7187272 | 2019-04-10 17:27:53 +0000 | [diff] [blame] | 1464 | for (int I = 0; I != NumParts; ++I) { |
| 1465 | unsigned Offset = I * NarrowSize; |
| 1466 | auto K = MIRBuilder.buildConstant(NarrowTy, |
| 1467 | Val.lshr(Offset).trunc(NarrowSize)); |
| 1468 | PartRegs.push_back(K.getReg(0)); |
| 1469 | } |
| 1470 | |
| 1471 | LLT LeftoverTy; |
| 1472 | unsigned LeftoverBits = TotalSize - NumParts * NarrowSize; |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 1473 | SmallVector<Register, 1> LeftoverRegs; |
Matt Arsenault | 7187272 | 2019-04-10 17:27:53 +0000 | [diff] [blame] | 1474 | if (LeftoverBits != 0) { |
| 1475 | LeftoverTy = LLT::scalar(LeftoverBits); |
| 1476 | auto K = MIRBuilder.buildConstant( |
| 1477 | LeftoverTy, |
| 1478 | Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits)); |
| 1479 | LeftoverRegs.push_back(K.getReg(0)); |
| 1480 | } |
| 1481 | |
| 1482 | insertParts(MI.getOperand(0).getReg(), |
| 1483 | Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); |
| 1484 | |
| 1485 | MI.eraseFromParent(); |
| 1486 | return Legalized; |
| 1487 | } |
Matt Arsenault | 25e9938 | 2020-01-10 10:07:24 -0500 | [diff] [blame] | 1488 | case TargetOpcode::G_SEXT: |
Matt Arsenault | 91715617 | 2020-01-10 09:47:17 -0500 | [diff] [blame] | 1489 | case TargetOpcode::G_ZEXT: |
Matt Arsenault | be31a7b | 2020-01-10 11:02:18 -0500 | [diff] [blame] | 1490 | case TargetOpcode::G_ANYEXT: |
| 1491 | return narrowScalarExt(MI, TypeIdx, NarrowTy); |
Petar Avramovic | 5b4c5c2 | 2019-08-21 09:26:39 +0000 | [diff] [blame] | 1492 | case TargetOpcode::G_TRUNC: { |
| 1493 | if (TypeIdx != 1) |
| 1494 | return UnableToLegalize; |
| 1495 | |
| 1496 | uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); |
| 1497 | if (NarrowTy.getSizeInBits() * 2 != SizeOp1) { |
| 1498 | LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n"); |
| 1499 | return UnableToLegalize; |
| 1500 | } |
| 1501 | |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 1502 | auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); |
| 1503 | MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0)); |
Petar Avramovic | 5b4c5c2 | 2019-08-21 09:26:39 +0000 | [diff] [blame] | 1504 | MI.eraseFromParent(); |
| 1505 | return Legalized; |
| 1506 | } |
Yingwei Zheng | 821bcba | 2024-05-22 23:35:37 +0800 | [diff] [blame] | 1507 | case TargetOpcode::G_CONSTANT_FOLD_BARRIER: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 1508 | case TargetOpcode::G_FREEZE: { |
| 1509 | if (TypeIdx != 0) |
| 1510 | return UnableToLegalize; |
| 1511 | |
| 1512 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
| 1513 | // Should widen scalar first |
| 1514 | if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0) |
| 1515 | return UnableToLegalize; |
| 1516 | |
| 1517 | auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg()); |
| 1518 | SmallVector<Register, 8> Parts; |
| 1519 | for (unsigned i = 0; i < Unmerge->getNumDefs(); ++i) { |
| 1520 | Parts.push_back( |
Yingwei Zheng | 821bcba | 2024-05-22 23:35:37 +0800 | [diff] [blame] | 1521 | MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, {Unmerge.getReg(i)}) |
| 1522 | .getReg(0)); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 1523 | } |
| 1524 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 1525 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(0).getReg(), Parts); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 1526 | MI.eraseFromParent(); |
| 1527 | return Legalized; |
| 1528 | } |
Justin Bogner | 62ce4b0 | 2021-02-02 17:02:52 -0800 | [diff] [blame] | 1529 | case TargetOpcode::G_ADD: |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 1530 | case TargetOpcode::G_SUB: |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 1531 | case TargetOpcode::G_SADDO: |
| 1532 | case TargetOpcode::G_SSUBO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 1533 | case TargetOpcode::G_SADDE: |
| 1534 | case TargetOpcode::G_SSUBE: |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 1535 | case TargetOpcode::G_UADDO: |
| 1536 | case TargetOpcode::G_USUBO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 1537 | case TargetOpcode::G_UADDE: |
| 1538 | case TargetOpcode::G_USUBE: |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 1539 | return narrowScalarAddSub(MI, TypeIdx, NarrowTy); |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 1540 | case TargetOpcode::G_MUL: |
Petar Avramovic | 5229f47 | 2019-03-11 10:08:44 +0000 | [diff] [blame] | 1541 | case TargetOpcode::G_UMULH: |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 1542 | return narrowScalarMul(MI, NarrowTy); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 1543 | case TargetOpcode::G_EXTRACT: |
| 1544 | return narrowScalarExtract(MI, TypeIdx, NarrowTy); |
| 1545 | case TargetOpcode::G_INSERT: |
| 1546 | return narrowScalarInsert(MI, TypeIdx, NarrowTy); |
Justin Bogner | d09c3ce | 2017-01-19 01:05:48 +0000 | [diff] [blame] | 1547 | case TargetOpcode::G_LOAD: { |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1548 | auto &LoadMI = cast<GLoad>(MI); |
| 1549 | Register DstReg = LoadMI.getDstReg(); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1550 | LLT DstTy = MRI.getType(DstReg); |
Matt Arsenault | 7f09fd6 | 2019-02-05 00:26:12 +0000 | [diff] [blame] | 1551 | if (DstTy.isVector()) |
Matt Arsenault | 045bc9a | 2019-01-30 02:35:38 +0000 | [diff] [blame] | 1552 | return UnableToLegalize; |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1553 | |
David Green | 601e102 | 2024-03-17 18:15:56 +0000 | [diff] [blame] | 1554 | if (8 * LoadMI.getMemSize().getValue() != DstTy.getSizeInBits()) { |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 1555 | Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1556 | MIRBuilder.buildLoad(TmpReg, LoadMI.getPointerReg(), LoadMI.getMMO()); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1557 | MIRBuilder.buildAnyExt(DstReg, TmpReg); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1558 | LoadMI.eraseFromParent(); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1559 | return Legalized; |
| 1560 | } |
| 1561 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1562 | return reduceLoadStoreWidth(LoadMI, TypeIdx, NarrowTy); |
Justin Bogner | d09c3ce | 2017-01-19 01:05:48 +0000 | [diff] [blame] | 1563 | } |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1564 | case TargetOpcode::G_ZEXTLOAD: |
| 1565 | case TargetOpcode::G_SEXTLOAD: { |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1566 | auto &LoadMI = cast<GExtLoad>(MI); |
| 1567 | Register DstReg = LoadMI.getDstReg(); |
| 1568 | Register PtrReg = LoadMI.getPointerReg(); |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1569 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 1570 | Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1571 | auto &MMO = LoadMI.getMMO(); |
David Green | 601e102 | 2024-03-17 18:15:56 +0000 | [diff] [blame] | 1572 | unsigned MemSize = MMO.getSizeInBits().getValue(); |
Matt Arsenault | 2cbbc6e | 2021-01-05 23:25:18 -0500 | [diff] [blame] | 1573 | |
| 1574 | if (MemSize == NarrowSize) { |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1575 | MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); |
Matt Arsenault | 2cbbc6e | 2021-01-05 23:25:18 -0500 | [diff] [blame] | 1576 | } else if (MemSize < NarrowSize) { |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1577 | MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), TmpReg, PtrReg, MMO); |
Matt Arsenault | 2cbbc6e | 2021-01-05 23:25:18 -0500 | [diff] [blame] | 1578 | } else if (MemSize > NarrowSize) { |
| 1579 | // FIXME: Need to split the load. |
| 1580 | return UnableToLegalize; |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1581 | } |
| 1582 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1583 | if (isa<GZExtLoad>(LoadMI)) |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1584 | MIRBuilder.buildZExt(DstReg, TmpReg); |
| 1585 | else |
| 1586 | MIRBuilder.buildSExt(DstReg, TmpReg); |
| 1587 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1588 | LoadMI.eraseFromParent(); |
Matt Arsenault | 6614f85 | 2019-01-22 19:02:10 +0000 | [diff] [blame] | 1589 | return Legalized; |
| 1590 | } |
Justin Bogner | fde0104 | 2017-01-18 17:29:54 +0000 | [diff] [blame] | 1591 | case TargetOpcode::G_STORE: { |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1592 | auto &StoreMI = cast<GStore>(MI); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1593 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1594 | Register SrcReg = StoreMI.getValueReg(); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1595 | LLT SrcTy = MRI.getType(SrcReg); |
Matt Arsenault | 7f09fd6 | 2019-02-05 00:26:12 +0000 | [diff] [blame] | 1596 | if (SrcTy.isVector()) |
| 1597 | return UnableToLegalize; |
| 1598 | |
| 1599 | int NumParts = SizeOp0 / NarrowSize; |
| 1600 | unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); |
| 1601 | unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize; |
| 1602 | if (SrcTy.isVector() && LeftoverBits != 0) |
| 1603 | return UnableToLegalize; |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1604 | |
David Green | 601e102 | 2024-03-17 18:15:56 +0000 | [diff] [blame] | 1605 | if (8 * StoreMI.getMemSize().getValue() != SrcTy.getSizeInBits()) { |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 1606 | Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1607 | MIRBuilder.buildTrunc(TmpReg, SrcReg); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1608 | MIRBuilder.buildStore(TmpReg, StoreMI.getPointerReg(), StoreMI.getMMO()); |
| 1609 | StoreMI.eraseFromParent(); |
Matt Arsenault | 18619af | 2019-01-29 18:13:02 +0000 | [diff] [blame] | 1610 | return Legalized; |
| 1611 | } |
| 1612 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 1613 | return reduceLoadStoreWidth(StoreMI, 0, NarrowTy); |
Justin Bogner | fde0104 | 2017-01-18 17:29:54 +0000 | [diff] [blame] | 1614 | } |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 1615 | case TargetOpcode::G_SELECT: |
| 1616 | return narrowScalarSelect(MI, TypeIdx, NarrowTy); |
Petar Avramovic | 150fd43 | 2018-12-18 11:36:14 +0000 | [diff] [blame] | 1617 | case TargetOpcode::G_AND: |
| 1618 | case TargetOpcode::G_OR: |
| 1619 | case TargetOpcode::G_XOR: { |
Quentin Colombet | c2f3cea | 2017-10-03 04:53:56 +0000 | [diff] [blame] | 1620 | // Legalize bitwise operation: |
| 1621 | // A = BinOp<Ty> B, C |
| 1622 | // into: |
| 1623 | // B1, ..., BN = G_UNMERGE_VALUES B |
| 1624 | // C1, ..., CN = G_UNMERGE_VALUES C |
| 1625 | // A1 = BinOp<Ty/N> B1, C2 |
| 1626 | // ... |
| 1627 | // AN = BinOp<Ty/N> BN, CN |
| 1628 | // A = G_MERGE_VALUES A1, ..., AN |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 1629 | return narrowScalarBasic(MI, TypeIdx, NarrowTy); |
Quentin Colombet | c2f3cea | 2017-10-03 04:53:56 +0000 | [diff] [blame] | 1630 | } |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 1631 | case TargetOpcode::G_SHL: |
| 1632 | case TargetOpcode::G_LSHR: |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 1633 | case TargetOpcode::G_ASHR: |
| 1634 | return narrowScalarShift(MI, TypeIdx, NarrowTy); |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 1635 | case TargetOpcode::G_CTLZ: |
| 1636 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
| 1637 | case TargetOpcode::G_CTTZ: |
| 1638 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: |
| 1639 | case TargetOpcode::G_CTPOP: |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 1640 | if (TypeIdx == 1) |
| 1641 | switch (MI.getOpcode()) { |
| 1642 | case TargetOpcode::G_CTLZ: |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 1643 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 1644 | return narrowScalarCTLZ(MI, TypeIdx, NarrowTy); |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 1645 | case TargetOpcode::G_CTTZ: |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 1646 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 1647 | return narrowScalarCTTZ(MI, TypeIdx, NarrowTy); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 1648 | case TargetOpcode::G_CTPOP: |
| 1649 | return narrowScalarCTPOP(MI, TypeIdx, NarrowTy); |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 1650 | default: |
| 1651 | return UnableToLegalize; |
| 1652 | } |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 1653 | |
| 1654 | Observer.changingInstr(MI); |
| 1655 | narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); |
| 1656 | Observer.changedInstr(MI); |
| 1657 | return Legalized; |
Matt Arsenault | cbaada6 | 2019-02-02 23:29:55 +0000 | [diff] [blame] | 1658 | case TargetOpcode::G_INTTOPTR: |
| 1659 | if (TypeIdx != 1) |
| 1660 | return UnableToLegalize; |
| 1661 | |
| 1662 | Observer.changingInstr(MI); |
| 1663 | narrowScalarSrc(MI, NarrowTy, 1); |
| 1664 | Observer.changedInstr(MI); |
| 1665 | return Legalized; |
| 1666 | case TargetOpcode::G_PTRTOINT: |
| 1667 | if (TypeIdx != 0) |
| 1668 | return UnableToLegalize; |
| 1669 | |
| 1670 | Observer.changingInstr(MI); |
| 1671 | narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); |
| 1672 | Observer.changedInstr(MI); |
| 1673 | return Legalized; |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1674 | case TargetOpcode::G_PHI: { |
Nikita Popov | c35761d | 2021-03-01 21:37:26 +0100 | [diff] [blame] | 1675 | // FIXME: add support for when SizeOp0 isn't an exact multiple of |
| 1676 | // NarrowSize. |
| 1677 | if (SizeOp0 % NarrowSize != 0) |
| 1678 | return UnableToLegalize; |
| 1679 | |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1680 | unsigned NumParts = SizeOp0 / NarrowSize; |
Matt Arsenault | de8451f | 2020-02-04 10:34:22 -0500 | [diff] [blame] | 1681 | SmallVector<Register, 2> DstRegs(NumParts); |
| 1682 | SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2); |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1683 | Observer.changingInstr(MI); |
| 1684 | for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { |
| 1685 | MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); |
Amara Emerson | 53445f5 | 2022-11-13 01:43:04 -0800 | [diff] [blame] | 1686 | MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1687 | extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 1688 | SrcRegs[i / 2], MIRBuilder, MRI); |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1689 | } |
| 1690 | MachineBasicBlock &MBB = *MI.getParent(); |
| 1691 | MIRBuilder.setInsertPt(MBB, MI); |
| 1692 | for (unsigned i = 0; i < NumParts; ++i) { |
| 1693 | DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); |
| 1694 | MachineInstrBuilder MIB = |
| 1695 | MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); |
| 1696 | for (unsigned j = 1; j < MI.getNumOperands(); j += 2) |
| 1697 | MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1)); |
| 1698 | } |
Amara Emerson | 02bcc86 | 2019-09-13 21:49:24 +0000 | [diff] [blame] | 1699 | MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 1700 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs); |
Petar Avramovic | be20e36 | 2019-07-09 14:36:17 +0000 | [diff] [blame] | 1701 | Observer.changedInstr(MI); |
| 1702 | MI.eraseFromParent(); |
| 1703 | return Legalized; |
| 1704 | } |
Matt Arsenault | 434d664 | 2019-07-15 19:37:34 +0000 | [diff] [blame] | 1705 | case TargetOpcode::G_EXTRACT_VECTOR_ELT: |
| 1706 | case TargetOpcode::G_INSERT_VECTOR_ELT: { |
| 1707 | if (TypeIdx != 2) |
| 1708 | return UnableToLegalize; |
| 1709 | |
| 1710 | int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; |
| 1711 | Observer.changingInstr(MI); |
| 1712 | narrowScalarSrc(MI, NarrowTy, OpIdx); |
| 1713 | Observer.changedInstr(MI); |
| 1714 | return Legalized; |
| 1715 | } |
Petar Avramovic | 1e62635 | 2019-07-17 12:08:01 +0000 | [diff] [blame] | 1716 | case TargetOpcode::G_ICMP: { |
Jessica Paquette | 47d0780 | 2021-06-29 17:01:28 -0700 | [diff] [blame] | 1717 | Register LHS = MI.getOperand(2).getReg(); |
| 1718 | LLT SrcTy = MRI.getType(LHS); |
Petar Avramovic | 1e62635 | 2019-07-17 12:08:01 +0000 | [diff] [blame] | 1719 | CmpInst::Predicate Pred = |
| 1720 | static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); |
| 1721 | |
Jessica Paquette | 47d0780 | 2021-06-29 17:01:28 -0700 | [diff] [blame] | 1722 | LLT LeftoverTy; // Example: s88 -> s64 (NarrowTy) + s24 (leftover) |
| 1723 | SmallVector<Register, 4> LHSPartRegs, LHSLeftoverRegs; |
| 1724 | if (!extractParts(LHS, SrcTy, NarrowTy, LeftoverTy, LHSPartRegs, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 1725 | LHSLeftoverRegs, MIRBuilder, MRI)) |
Jessica Paquette | 47d0780 | 2021-06-29 17:01:28 -0700 | [diff] [blame] | 1726 | return UnableToLegalize; |
| 1727 | |
| 1728 | LLT Unused; // Matches LeftoverTy; G_ICMP LHS and RHS are the same type. |
| 1729 | SmallVector<Register, 4> RHSPartRegs, RHSLeftoverRegs; |
| 1730 | if (!extractParts(MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 1731 | RHSPartRegs, RHSLeftoverRegs, MIRBuilder, MRI)) |
Jessica Paquette | 47d0780 | 2021-06-29 17:01:28 -0700 | [diff] [blame] | 1732 | return UnableToLegalize; |
| 1733 | |
| 1734 | // We now have the LHS and RHS of the compare split into narrow-type |
| 1735 | // registers, plus potentially some leftover type. |
| 1736 | Register Dst = MI.getOperand(0).getReg(); |
| 1737 | LLT ResTy = MRI.getType(Dst); |
| 1738 | if (ICmpInst::isEquality(Pred)) { |
| 1739 | // For each part on the LHS and RHS, keep track of the result of XOR-ing |
| 1740 | // them together. For each equal part, the result should be all 0s. For |
| 1741 | // each non-equal part, we'll get at least one 1. |
| 1742 | auto Zero = MIRBuilder.buildConstant(NarrowTy, 0); |
| 1743 | SmallVector<Register, 4> Xors; |
| 1744 | for (auto LHSAndRHS : zip(LHSPartRegs, RHSPartRegs)) { |
| 1745 | auto LHS = std::get<0>(LHSAndRHS); |
| 1746 | auto RHS = std::get<1>(LHSAndRHS); |
| 1747 | auto Xor = MIRBuilder.buildXor(NarrowTy, LHS, RHS).getReg(0); |
| 1748 | Xors.push_back(Xor); |
| 1749 | } |
| 1750 | |
| 1751 | // Build a G_XOR for each leftover register. Each G_XOR must be widened |
| 1752 | // to the desired narrow type so that we can OR them together later. |
| 1753 | SmallVector<Register, 4> WidenedXors; |
| 1754 | for (auto LHSAndRHS : zip(LHSLeftoverRegs, RHSLeftoverRegs)) { |
| 1755 | auto LHS = std::get<0>(LHSAndRHS); |
| 1756 | auto RHS = std::get<1>(LHSAndRHS); |
| 1757 | auto Xor = MIRBuilder.buildXor(LeftoverTy, LHS, RHS).getReg(0); |
| 1758 | LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy, Xor); |
| 1759 | buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors, |
| 1760 | /* PadStrategy = */ TargetOpcode::G_ZEXT); |
| 1761 | Xors.insert(Xors.end(), WidenedXors.begin(), WidenedXors.end()); |
| 1762 | } |
| 1763 | |
| 1764 | // Now, for each part we broke up, we know if they are equal/not equal |
| 1765 | // based off the G_XOR. We can OR these all together and compare against |
| 1766 | // 0 to get the result. |
| 1767 | assert(Xors.size() >= 2 && "Should have gotten at least two Xors?"); |
| 1768 | auto Or = MIRBuilder.buildOr(NarrowTy, Xors[0], Xors[1]); |
| 1769 | for (unsigned I = 2, E = Xors.size(); I < E; ++I) |
| 1770 | Or = MIRBuilder.buildOr(NarrowTy, Or, Xors[I]); |
| 1771 | MIRBuilder.buildICmp(Pred, Dst, Or, Zero); |
Petar Avramovic | 1e62635 | 2019-07-17 12:08:01 +0000 | [diff] [blame] | 1772 | } else { |
Craig Topper | 7ece560 | 2024-12-12 09:50:26 -0800 | [diff] [blame] | 1773 | Register CmpIn; |
| 1774 | for (unsigned I = 0, E = LHSPartRegs.size(); I != E; ++I) { |
| 1775 | Register CmpOut; |
| 1776 | CmpInst::Predicate PartPred; |
| 1777 | |
| 1778 | if (I == E - 1 && LHSLeftoverRegs.empty()) { |
| 1779 | PartPred = Pred; |
| 1780 | CmpOut = Dst; |
| 1781 | } else { |
| 1782 | PartPred = ICmpInst::getUnsignedPredicate(Pred); |
| 1783 | CmpOut = MRI.createGenericVirtualRegister(ResTy); |
| 1784 | } |
| 1785 | |
| 1786 | if (!CmpIn) { |
| 1787 | MIRBuilder.buildICmp(PartPred, CmpOut, LHSPartRegs[I], |
| 1788 | RHSPartRegs[I]); |
| 1789 | } else { |
| 1790 | auto Cmp = MIRBuilder.buildICmp(PartPred, ResTy, LHSPartRegs[I], |
| 1791 | RHSPartRegs[I]); |
| 1792 | auto CmpEq = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, |
| 1793 | LHSPartRegs[I], RHSPartRegs[I]); |
| 1794 | MIRBuilder.buildSelect(CmpOut, CmpEq, CmpIn, Cmp); |
| 1795 | } |
| 1796 | |
| 1797 | CmpIn = CmpOut; |
| 1798 | } |
| 1799 | |
| 1800 | for (unsigned I = 0, E = LHSLeftoverRegs.size(); I != E; ++I) { |
| 1801 | Register CmpOut; |
| 1802 | CmpInst::Predicate PartPred; |
| 1803 | |
| 1804 | if (I == E - 1 && LHSLeftoverRegs.empty()) { |
| 1805 | PartPred = Pred; |
| 1806 | CmpOut = Dst; |
| 1807 | } else { |
| 1808 | PartPred = ICmpInst::getUnsignedPredicate(Pred); |
| 1809 | CmpOut = MRI.createGenericVirtualRegister(ResTy); |
| 1810 | } |
| 1811 | |
| 1812 | if (!CmpIn) { |
| 1813 | MIRBuilder.buildICmp(PartPred, CmpOut, LHSLeftoverRegs[I], |
| 1814 | RHSLeftoverRegs[I]); |
| 1815 | } else { |
| 1816 | auto Cmp = MIRBuilder.buildICmp(PartPred, ResTy, LHSLeftoverRegs[I], |
| 1817 | RHSLeftoverRegs[I]); |
| 1818 | auto CmpEq = |
| 1819 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, |
| 1820 | LHSLeftoverRegs[I], RHSLeftoverRegs[I]); |
| 1821 | MIRBuilder.buildSelect(CmpOut, CmpEq, CmpIn, Cmp); |
| 1822 | } |
| 1823 | |
| 1824 | CmpIn = CmpOut; |
| 1825 | } |
Petar Avramovic | 1e62635 | 2019-07-17 12:08:01 +0000 | [diff] [blame] | 1826 | } |
Petar Avramovic | 1e62635 | 2019-07-17 12:08:01 +0000 | [diff] [blame] | 1827 | MI.eraseFromParent(); |
| 1828 | return Legalized; |
| 1829 | } |
David Green | f297d0b | 2024-01-28 15:42:36 +0000 | [diff] [blame] | 1830 | case TargetOpcode::G_FCMP: |
| 1831 | if (TypeIdx != 0) |
| 1832 | return UnableToLegalize; |
| 1833 | |
| 1834 | Observer.changingInstr(MI); |
| 1835 | narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); |
| 1836 | Observer.changedInstr(MI); |
| 1837 | return Legalized; |
| 1838 | |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1839 | case TargetOpcode::G_SEXT_INREG: { |
| 1840 | if (TypeIdx != 0) |
| 1841 | return UnableToLegalize; |
| 1842 | |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1843 | int64_t SizeInBits = MI.getOperand(2).getImm(); |
| 1844 | |
| 1845 | // So long as the new type has more bits than the bits we're extending we |
| 1846 | // don't need to break it apart. |
Craig Topper | 5d501b1 | 2023-11-24 08:39:38 -0800 | [diff] [blame] | 1847 | if (NarrowTy.getScalarSizeInBits() > SizeInBits) { |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1848 | Observer.changingInstr(MI); |
| 1849 | // We don't lose any non-extension bits by truncating the src and |
| 1850 | // sign-extending the dst. |
| 1851 | MachineOperand &MO1 = MI.getOperand(1); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 1852 | auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 1853 | MO1.setReg(TruncMIB.getReg(0)); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1854 | |
| 1855 | MachineOperand &MO2 = MI.getOperand(0); |
| 1856 | Register DstExt = MRI.createGenericVirtualRegister(NarrowTy); |
| 1857 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 1858 | MIRBuilder.buildSExt(MO2, DstExt); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1859 | MO2.setReg(DstExt); |
| 1860 | Observer.changedInstr(MI); |
| 1861 | return Legalized; |
| 1862 | } |
| 1863 | |
| 1864 | // Break it apart. Components below the extension point are unmodified. The |
| 1865 | // component containing the extension point becomes a narrower SEXT_INREG. |
| 1866 | // Components above it are ashr'd from the component containing the |
| 1867 | // extension point. |
| 1868 | if (SizeOp0 % NarrowSize != 0) |
| 1869 | return UnableToLegalize; |
| 1870 | int NumParts = SizeOp0 / NarrowSize; |
| 1871 | |
| 1872 | // List the registers where the destination will be scattered. |
| 1873 | SmallVector<Register, 2> DstRegs; |
| 1874 | // List the registers where the source will be split. |
| 1875 | SmallVector<Register, 2> SrcRegs; |
| 1876 | |
| 1877 | // Create all the temporary registers. |
| 1878 | for (int i = 0; i < NumParts; ++i) { |
| 1879 | Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy); |
| 1880 | |
| 1881 | SrcRegs.push_back(SrcReg); |
| 1882 | } |
| 1883 | |
| 1884 | // Explode the big arguments into smaller chunks. |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 1885 | MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1)); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1886 | |
| 1887 | Register AshrCstReg = |
| 1888 | MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1) |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 1889 | .getReg(0); |
Craig Topper | 5d501b1 | 2023-11-24 08:39:38 -0800 | [diff] [blame] | 1890 | Register FullExtensionReg; |
| 1891 | Register PartialExtensionReg; |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1892 | |
| 1893 | // Do the operation on each small part. |
| 1894 | for (int i = 0; i < NumParts; ++i) { |
Craig Topper | 5d501b1 | 2023-11-24 08:39:38 -0800 | [diff] [blame] | 1895 | if ((i + 1) * NarrowTy.getScalarSizeInBits() <= SizeInBits) { |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1896 | DstRegs.push_back(SrcRegs[i]); |
Craig Topper | 5d501b1 | 2023-11-24 08:39:38 -0800 | [diff] [blame] | 1897 | PartialExtensionReg = DstRegs.back(); |
| 1898 | } else if (i * NarrowTy.getScalarSizeInBits() >= SizeInBits) { |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1899 | assert(PartialExtensionReg && |
| 1900 | "Expected to visit partial extension before full"); |
| 1901 | if (FullExtensionReg) { |
| 1902 | DstRegs.push_back(FullExtensionReg); |
| 1903 | continue; |
| 1904 | } |
Jay Foad | 28bb43b | 2020-01-16 12:09:48 +0000 | [diff] [blame] | 1905 | DstRegs.push_back( |
| 1906 | MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg) |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 1907 | .getReg(0)); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1908 | FullExtensionReg = DstRegs.back(); |
| 1909 | } else { |
| 1910 | DstRegs.push_back( |
| 1911 | MIRBuilder |
| 1912 | .buildInstr( |
| 1913 | TargetOpcode::G_SEXT_INREG, {NarrowTy}, |
| 1914 | {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()}) |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 1915 | .getReg(0)); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1916 | PartialExtensionReg = DstRegs.back(); |
| 1917 | } |
| 1918 | } |
| 1919 | |
| 1920 | // Gather the destination registers into the final destination. |
| 1921 | Register DstReg = MI.getOperand(0).getReg(); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 1922 | MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 1923 | MI.eraseFromParent(); |
| 1924 | return Legalized; |
| 1925 | } |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 1926 | case TargetOpcode::G_BSWAP: |
| 1927 | case TargetOpcode::G_BITREVERSE: { |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 1928 | if (SizeOp0 % NarrowSize != 0) |
| 1929 | return UnableToLegalize; |
| 1930 | |
| 1931 | Observer.changingInstr(MI); |
| 1932 | SmallVector<Register, 2> SrcRegs, DstRegs; |
| 1933 | unsigned NumParts = SizeOp0 / NarrowSize; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 1934 | extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs, |
| 1935 | MIRBuilder, MRI); |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 1936 | |
| 1937 | for (unsigned i = 0; i < NumParts; ++i) { |
| 1938 | auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, |
| 1939 | {SrcRegs[NumParts - 1 - i]}); |
| 1940 | DstRegs.push_back(DstPart.getReg(0)); |
| 1941 | } |
| 1942 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 1943 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), DstRegs); |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 1944 | |
| 1945 | Observer.changedInstr(MI); |
| 1946 | MI.eraseFromParent(); |
| 1947 | return Legalized; |
| 1948 | } |
Matt Arsenault | f6176f8 | 2020-07-25 11:00:35 -0400 | [diff] [blame] | 1949 | case TargetOpcode::G_PTR_ADD: |
Matt Arsenault | ef3e8312 | 2020-05-23 18:10:34 -0400 | [diff] [blame] | 1950 | case TargetOpcode::G_PTRMASK: { |
| 1951 | if (TypeIdx != 1) |
| 1952 | return UnableToLegalize; |
| 1953 | Observer.changingInstr(MI); |
| 1954 | narrowScalarSrc(MI, NarrowTy, 2); |
| 1955 | Observer.changedInstr(MI); |
| 1956 | return Legalized; |
| 1957 | } |
Matt Arsenault | 83a25a1 | 2021-03-26 17:29:36 -0400 | [diff] [blame] | 1958 | case TargetOpcode::G_FPTOUI: |
| 1959 | case TargetOpcode::G_FPTOSI: |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 1960 | case TargetOpcode::G_FPTOUI_SAT: |
| 1961 | case TargetOpcode::G_FPTOSI_SAT: |
Matt Arsenault | 83a25a1 | 2021-03-26 17:29:36 -0400 | [diff] [blame] | 1962 | return narrowScalarFPTOI(MI, TypeIdx, NarrowTy); |
Petar Avramovic | 6a1030a | 2020-07-20 16:12:19 +0200 | [diff] [blame] | 1963 | case TargetOpcode::G_FPEXT: |
| 1964 | if (TypeIdx != 0) |
| 1965 | return UnableToLegalize; |
| 1966 | Observer.changingInstr(MI); |
| 1967 | narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT); |
| 1968 | Observer.changedInstr(MI); |
| 1969 | return Legalized; |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 1970 | case TargetOpcode::G_FLDEXP: |
| 1971 | case TargetOpcode::G_STRICT_FLDEXP: |
| 1972 | return narrowScalarFLDEXP(MI, TypeIdx, NarrowTy); |
Michael Maitland | 54a9f0e | 2024-03-26 20:17:22 -0400 | [diff] [blame] | 1973 | case TargetOpcode::G_VSCALE: { |
| 1974 | Register Dst = MI.getOperand(0).getReg(); |
| 1975 | LLT Ty = MRI.getType(Dst); |
| 1976 | |
| 1977 | // Assume VSCALE(1) fits into a legal integer |
| 1978 | const APInt One(NarrowTy.getSizeInBits(), 1); |
| 1979 | auto VScaleBase = MIRBuilder.buildVScale(NarrowTy, One); |
| 1980 | auto ZExt = MIRBuilder.buildZExt(Ty, VScaleBase); |
| 1981 | auto C = MIRBuilder.buildConstant(Ty, *MI.getOperand(1).getCImm()); |
| 1982 | MIRBuilder.buildMul(Dst, ZExt, C); |
| 1983 | |
| 1984 | MI.eraseFromParent(); |
| 1985 | return Legalized; |
| 1986 | } |
Tim Northover | 9656f14 | 2016-08-04 20:54:13 +0000 | [diff] [blame] | 1987 | } |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 1988 | } |
| 1989 | |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 1990 | Register LegalizerHelper::coerceToScalar(Register Val) { |
| 1991 | LLT Ty = MRI.getType(Val); |
| 1992 | if (Ty.isScalar()) |
| 1993 | return Val; |
| 1994 | |
| 1995 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 1996 | LLT NewTy = LLT::scalar(Ty.getSizeInBits()); |
| 1997 | if (Ty.isPointer()) { |
| 1998 | if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace())) |
| 1999 | return Register(); |
| 2000 | return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0); |
| 2001 | } |
| 2002 | |
| 2003 | Register NewVal = Val; |
| 2004 | |
| 2005 | assert(Ty.isVector()); |
Jay Foad | d57515bd | 2024-02-13 08:21:35 +0000 | [diff] [blame] | 2006 | if (Ty.isPointerVector()) |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 2007 | NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0); |
| 2008 | return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0); |
| 2009 | } |
| 2010 | |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2011 | void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, |
| 2012 | unsigned OpIdx, unsigned ExtOpcode) { |
| 2013 | MachineOperand &MO = MI.getOperand(OpIdx); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 2014 | auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO}); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 2015 | MO.setReg(ExtB.getReg(0)); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2016 | } |
| 2017 | |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2018 | void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, |
| 2019 | unsigned OpIdx) { |
| 2020 | MachineOperand &MO = MI.getOperand(OpIdx); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 2021 | auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 2022 | MO.setReg(ExtB.getReg(0)); |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2023 | } |
| 2024 | |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2025 | void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, |
| 2026 | unsigned OpIdx, unsigned TruncOpcode) { |
| 2027 | MachineOperand &MO = MI.getOperand(OpIdx); |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 2028 | Register DstExt = MRI.createGenericVirtualRegister(WideTy); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2029 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 2030 | MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt}); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2031 | MO.setReg(DstExt); |
| 2032 | } |
| 2033 | |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2034 | void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, |
| 2035 | unsigned OpIdx, unsigned ExtOpcode) { |
| 2036 | MachineOperand &MO = MI.getOperand(OpIdx); |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 2037 | Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2038 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 2039 | MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc}); |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2040 | MO.setReg(DstTrunc); |
| 2041 | } |
| 2042 | |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 2043 | void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, |
| 2044 | unsigned OpIdx) { |
| 2045 | MachineOperand &MO = MI.getOperand(OpIdx); |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 2046 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 2047 | Register Dst = MO.getReg(); |
| 2048 | Register DstExt = MRI.createGenericVirtualRegister(WideTy); |
| 2049 | MO.setReg(DstExt); |
| 2050 | MIRBuilder.buildDeleteTrailingVectorElements(Dst, DstExt); |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 2051 | } |
| 2052 | |
Matt Arsenault | 26b7e85 | 2019-02-19 16:30:19 +0000 | [diff] [blame] | 2053 | void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, |
| 2054 | unsigned OpIdx) { |
| 2055 | MachineOperand &MO = MI.getOperand(OpIdx); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 2056 | SmallVector<Register, 8> Regs; |
| 2057 | MO.setReg(MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO).getReg(0)); |
Matt Arsenault | 26b7e85 | 2019-02-19 16:30:19 +0000 | [diff] [blame] | 2058 | } |
| 2059 | |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 2060 | void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { |
| 2061 | MachineOperand &Op = MI.getOperand(OpIdx); |
| 2062 | Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0)); |
| 2063 | } |
| 2064 | |
| 2065 | void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { |
| 2066 | MachineOperand &MO = MI.getOperand(OpIdx); |
| 2067 | Register CastDst = MRI.createGenericVirtualRegister(CastTy); |
| 2068 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
| 2069 | MIRBuilder.buildBitcast(MO, CastDst); |
| 2070 | MO.setReg(CastDst); |
| 2071 | } |
| 2072 | |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 2073 | LegalizerHelper::LegalizeResult |
Mitch Phillips | ae70b21 | 2021-07-26 19:32:49 -0700 | [diff] [blame] | 2074 | LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, |
| 2075 | LLT WideTy) { |
| 2076 | if (TypeIdx != 1) |
| 2077 | return UnableToLegalize; |
| 2078 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 2079 | auto [DstReg, DstTy, Src1Reg, Src1Ty] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 43cbca5 | 2019-07-03 23:08:06 +0000 | [diff] [blame] | 2080 | if (DstTy.isVector()) |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2081 | return UnableToLegalize; |
| 2082 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 2083 | LLT SrcTy = MRI.getType(Src1Reg); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2084 | const int DstSize = DstTy.getSizeInBits(); |
| 2085 | const int SrcSize = SrcTy.getSizeInBits(); |
| 2086 | const int WideSize = WideTy.getSizeInBits(); |
| 2087 | const int NumMerge = (DstSize + WideSize - 1) / WideSize; |
Matt Arsenault | c9f14f2 | 2019-07-01 19:36:10 +0000 | [diff] [blame] | 2088 | |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2089 | unsigned NumOps = MI.getNumOperands(); |
| 2090 | unsigned NumSrc = MI.getNumOperands() - 1; |
| 2091 | unsigned PartSize = DstTy.getSizeInBits() / NumSrc; |
| 2092 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2093 | if (WideSize >= DstSize) { |
| 2094 | // Directly pack the bits in the target type. |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 2095 | Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1Reg).getReg(0); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2096 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2097 | for (unsigned I = 2; I != NumOps; ++I) { |
| 2098 | const unsigned Offset = (I - 1) * PartSize; |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2099 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2100 | Register SrcReg = MI.getOperand(I).getReg(); |
| 2101 | assert(MRI.getType(SrcReg) == LLT::scalar(PartSize)); |
| 2102 | |
| 2103 | auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); |
| 2104 | |
Matt Arsenault | 5faa533 | 2019-08-01 18:13:16 +0000 | [diff] [blame] | 2105 | Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2106 | MRI.createGenericVirtualRegister(WideTy); |
| 2107 | |
| 2108 | auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); |
| 2109 | auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); |
| 2110 | MIRBuilder.buildOr(NextResult, ResultReg, Shl); |
| 2111 | ResultReg = NextResult; |
| 2112 | } |
| 2113 | |
| 2114 | if (WideSize > DstSize) |
| 2115 | MIRBuilder.buildTrunc(DstReg, ResultReg); |
Matt Arsenault | 5faa533 | 2019-08-01 18:13:16 +0000 | [diff] [blame] | 2116 | else if (DstTy.isPointer()) |
| 2117 | MIRBuilder.buildIntToPtr(DstReg, ResultReg); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2118 | |
| 2119 | MI.eraseFromParent(); |
| 2120 | return Legalized; |
| 2121 | } |
| 2122 | |
| 2123 | // Unmerge the original values to the GCD type, and recombine to the next |
| 2124 | // multiple greater than the original type. |
| 2125 | // |
| 2126 | // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6 |
| 2127 | // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0 |
| 2128 | // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1 |
| 2129 | // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2 |
| 2130 | // %10:_(s6) = G_MERGE_VALUES %4, %5, %6 |
| 2131 | // %11:_(s6) = G_MERGE_VALUES %7, %8, %9 |
| 2132 | // %12:_(s12) = G_MERGE_VALUES %10, %11 |
| 2133 | // |
| 2134 | // Padding with undef if necessary: |
| 2135 | // |
| 2136 | // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6 |
| 2137 | // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0 |
| 2138 | // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1 |
| 2139 | // %7:_(s2) = G_IMPLICIT_DEF |
| 2140 | // %8:_(s6) = G_MERGE_VALUES %3, %4, %5 |
| 2141 | // %9:_(s6) = G_MERGE_VALUES %6, %7, %7 |
| 2142 | // %10:_(s12) = G_MERGE_VALUES %8, %9 |
| 2143 | |
Kazu Hirata | 267f21a | 2022-08-28 10:41:51 -0700 | [diff] [blame] | 2144 | const int GCD = std::gcd(SrcSize, WideSize); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2145 | LLT GCDTy = LLT::scalar(GCD); |
| 2146 | |
| 2147 | SmallVector<Register, 8> Parts; |
| 2148 | SmallVector<Register, 8> NewMergeRegs; |
| 2149 | SmallVector<Register, 8> Unmerges; |
| 2150 | LLT WideDstTy = LLT::scalar(NumMerge * WideSize); |
| 2151 | |
| 2152 | // Decompose the original operands if they don't evenly divide. |
Kazu Hirata | 259cd6f | 2021-11-25 22:17:10 -0800 | [diff] [blame] | 2153 | for (const MachineOperand &MO : llvm::drop_begin(MI.operands())) { |
| 2154 | Register SrcReg = MO.getReg(); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2155 | if (GCD == SrcSize) { |
| 2156 | Unmerges.push_back(SrcReg); |
| 2157 | } else { |
| 2158 | auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); |
| 2159 | for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J) |
| 2160 | Unmerges.push_back(Unmerge.getReg(J)); |
| 2161 | } |
| 2162 | } |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2163 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2164 | // Pad with undef to the next size that is a multiple of the requested size. |
| 2165 | if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) { |
| 2166 | Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); |
| 2167 | for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I) |
| 2168 | Unmerges.push_back(UndefReg); |
| 2169 | } |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2170 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2171 | const int PartsPerGCD = WideSize / GCD; |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2172 | |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2173 | // Build merges of each piece. |
| 2174 | ArrayRef<Register> Slicer(Unmerges); |
| 2175 | for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 2176 | auto Merge = |
| 2177 | MIRBuilder.buildMergeLikeInstr(WideTy, Slicer.take_front(PartsPerGCD)); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2178 | NewMergeRegs.push_back(Merge.getReg(0)); |
| 2179 | } |
| 2180 | |
| 2181 | // A truncate may be necessary if the requested type doesn't evenly divide the |
| 2182 | // original result type. |
| 2183 | if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 2184 | MIRBuilder.buildMergeLikeInstr(DstReg, NewMergeRegs); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2185 | } else { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 2186 | auto FinalMerge = MIRBuilder.buildMergeLikeInstr(WideDstTy, NewMergeRegs); |
Matt Arsenault | 0966dd0 | 2019-07-17 20:22:44 +0000 | [diff] [blame] | 2187 | MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2188 | } |
| 2189 | |
| 2190 | MI.eraseFromParent(); |
| 2191 | return Legalized; |
| 2192 | } |
| 2193 | |
| 2194 | LegalizerHelper::LegalizeResult |
| 2195 | LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, |
| 2196 | LLT WideTy) { |
| 2197 | if (TypeIdx != 0) |
| 2198 | return UnableToLegalize; |
| 2199 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2200 | int NumDst = MI.getNumOperands() - 1; |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 2201 | Register SrcReg = MI.getOperand(NumDst).getReg(); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2202 | LLT SrcTy = MRI.getType(SrcReg); |
Matt Arsenault | bc101ff | 2020-01-21 11:12:36 -0500 | [diff] [blame] | 2203 | if (SrcTy.isVector()) |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2204 | return UnableToLegalize; |
| 2205 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 2206 | Register Dst0Reg = MI.getOperand(0).getReg(); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2207 | LLT DstTy = MRI.getType(Dst0Reg); |
| 2208 | if (!DstTy.isScalar()) |
| 2209 | return UnableToLegalize; |
| 2210 | |
Dominik Montada | ccf49b9 | 2020-03-20 14:46:01 +0100 | [diff] [blame] | 2211 | if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) { |
Matt Arsenault | bc101ff | 2020-01-21 11:12:36 -0500 | [diff] [blame] | 2212 | if (SrcTy.isPointer()) { |
| 2213 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 2214 | if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) { |
Dominik Montada | ccf49b9 | 2020-03-20 14:46:01 +0100 | [diff] [blame] | 2215 | LLVM_DEBUG( |
| 2216 | dbgs() << "Not casting non-integral address space integer\n"); |
Matt Arsenault | bc101ff | 2020-01-21 11:12:36 -0500 | [diff] [blame] | 2217 | return UnableToLegalize; |
| 2218 | } |
| 2219 | |
| 2220 | SrcTy = LLT::scalar(SrcTy.getSizeInBits()); |
| 2221 | SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0); |
| 2222 | } |
| 2223 | |
Dominik Montada | ccf49b9 | 2020-03-20 14:46:01 +0100 | [diff] [blame] | 2224 | // Widen SrcTy to WideTy. This does not affect the result, but since the |
| 2225 | // user requested this size, it is probably better handled than SrcTy and |
Daniel Thornburgh | 2e2999c | 2022-01-18 18:03:26 -0800 | [diff] [blame] | 2226 | // should reduce the total number of legalization artifacts. |
Dominik Montada | ccf49b9 | 2020-03-20 14:46:01 +0100 | [diff] [blame] | 2227 | if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { |
| 2228 | SrcTy = WideTy; |
| 2229 | SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); |
| 2230 | } |
| 2231 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2232 | // Theres no unmerge type to target. Directly extract the bits from the |
| 2233 | // source type |
| 2234 | unsigned DstSize = DstTy.getSizeInBits(); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2235 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2236 | MIRBuilder.buildTrunc(Dst0Reg, SrcReg); |
| 2237 | for (int I = 1; I != NumDst; ++I) { |
| 2238 | auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I); |
| 2239 | auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt); |
| 2240 | MIRBuilder.buildTrunc(MI.getOperand(I), Shr); |
| 2241 | } |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2242 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2243 | MI.eraseFromParent(); |
| 2244 | return Legalized; |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2245 | } |
| 2246 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2247 | // Extend the source to a wider type. |
| 2248 | LLT LCMTy = getLCMType(SrcTy, WideTy); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2249 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2250 | Register WideSrc = SrcReg; |
Matt Arsenault | bc101ff | 2020-01-21 11:12:36 -0500 | [diff] [blame] | 2251 | if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) { |
| 2252 | // TODO: If this is an integral address space, cast to integer and anyext. |
| 2253 | if (SrcTy.isPointer()) { |
| 2254 | LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n"); |
| 2255 | return UnableToLegalize; |
| 2256 | } |
| 2257 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2258 | WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0); |
Matt Arsenault | bc101ff | 2020-01-21 11:12:36 -0500 | [diff] [blame] | 2259 | } |
| 2260 | |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2261 | auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2262 | |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2263 | // Create a sequence of unmerges and merges to the original results. Since we |
| 2264 | // may have widened the source, we will need to pad the results with dead defs |
| 2265 | // to cover the source register. |
| 2266 | // e.g. widen s48 to s64: |
| 2267 | // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96) |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2268 | // |
| 2269 | // => |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2270 | // %4:_(s192) = G_ANYEXT %0:_(s96) |
| 2271 | // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge |
| 2272 | // ; unpack to GCD type, with extra dead defs |
| 2273 | // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64) |
| 2274 | // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64) |
| 2275 | // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64) |
| 2276 | // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination |
| 2277 | // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination |
| 2278 | const LLT GCDTy = getGCDType(WideTy, DstTy); |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2279 | const int NumUnmerge = Unmerge->getNumOperands() - 1; |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2280 | const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits(); |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2281 | |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2282 | // Directly unmerge to the destination without going through a GCD type |
| 2283 | // if possible |
| 2284 | if (PartsPerRemerge == 1) { |
| 2285 | const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits(); |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2286 | |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2287 | for (int I = 0; I != NumUnmerge; ++I) { |
| 2288 | auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); |
| 2289 | |
| 2290 | for (int J = 0; J != PartsPerUnmerge; ++J) { |
| 2291 | int Idx = I * PartsPerUnmerge + J; |
| 2292 | if (Idx < NumDst) |
| 2293 | MIB.addDef(MI.getOperand(Idx).getReg()); |
| 2294 | else { |
| 2295 | // Create dead def for excess components. |
| 2296 | MIB.addDef(MRI.createGenericVirtualRegister(DstTy)); |
| 2297 | } |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2298 | } |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2299 | |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2300 | MIB.addUse(Unmerge.getReg(I)); |
| 2301 | } |
| 2302 | } else { |
| 2303 | SmallVector<Register, 16> Parts; |
| 2304 | for (int J = 0; J != NumUnmerge; ++J) |
| 2305 | extractGCDType(Parts, GCDTy, Unmerge.getReg(J)); |
| 2306 | |
| 2307 | SmallVector<Register, 8> RemergeParts; |
| 2308 | for (int I = 0; I != NumDst; ++I) { |
| 2309 | for (int J = 0; J < PartsPerRemerge; ++J) { |
| 2310 | const int Idx = I * PartsPerRemerge + J; |
| 2311 | RemergeParts.emplace_back(Parts[Idx]); |
| 2312 | } |
| 2313 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 2314 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(I).getReg(), RemergeParts); |
Dominik Montada | 113114a | 2020-09-28 16:38:35 +0200 | [diff] [blame] | 2315 | RemergeParts.clear(); |
| 2316 | } |
Matt Arsenault | 2a160ba | 2020-01-21 09:02:42 -0500 | [diff] [blame] | 2317 | } |
| 2318 | |
| 2319 | MI.eraseFromParent(); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2320 | return Legalized; |
| 2321 | } |
| 2322 | |
| 2323 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2324 | LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, |
| 2325 | LLT WideTy) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 2326 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | fbe92a5 | 2019-02-18 22:39:27 +0000 | [diff] [blame] | 2327 | unsigned Offset = MI.getOperand(2).getImm(); |
| 2328 | |
| 2329 | if (TypeIdx == 0) { |
| 2330 | if (SrcTy.isVector() || DstTy.isVector()) |
| 2331 | return UnableToLegalize; |
| 2332 | |
| 2333 | SrcOp Src(SrcReg); |
| 2334 | if (SrcTy.isPointer()) { |
| 2335 | // Extracts from pointers can be handled only if they are really just |
| 2336 | // simple integers. |
| 2337 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 2338 | if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) |
| 2339 | return UnableToLegalize; |
| 2340 | |
| 2341 | LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits()); |
| 2342 | Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); |
| 2343 | SrcTy = SrcAsIntTy; |
| 2344 | } |
| 2345 | |
| 2346 | if (DstTy.isPointer()) |
| 2347 | return UnableToLegalize; |
| 2348 | |
| 2349 | if (Offset == 0) { |
| 2350 | // Avoid a shift in the degenerate case. |
| 2351 | MIRBuilder.buildTrunc(DstReg, |
| 2352 | MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); |
| 2353 | MI.eraseFromParent(); |
| 2354 | return Legalized; |
| 2355 | } |
| 2356 | |
| 2357 | // Do a shift in the source type. |
| 2358 | LLT ShiftTy = SrcTy; |
| 2359 | if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { |
| 2360 | Src = MIRBuilder.buildAnyExt(WideTy, Src); |
| 2361 | ShiftTy = WideTy; |
Matt Arsenault | 90b76da | 2020-07-29 13:31:59 -0400 | [diff] [blame] | 2362 | } |
Matt Arsenault | fbe92a5 | 2019-02-18 22:39:27 +0000 | [diff] [blame] | 2363 | |
| 2364 | auto LShr = MIRBuilder.buildLShr( |
| 2365 | ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); |
| 2366 | MIRBuilder.buildTrunc(DstReg, LShr); |
| 2367 | MI.eraseFromParent(); |
| 2368 | return Legalized; |
| 2369 | } |
| 2370 | |
Matt Arsenault | 8f624ab | 2019-04-22 15:10:42 +0000 | [diff] [blame] | 2371 | if (SrcTy.isScalar()) { |
| 2372 | Observer.changingInstr(MI); |
| 2373 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2374 | Observer.changedInstr(MI); |
| 2375 | return Legalized; |
| 2376 | } |
| 2377 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2378 | if (!SrcTy.isVector()) |
| 2379 | return UnableToLegalize; |
| 2380 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2381 | if (DstTy != SrcTy.getElementType()) |
| 2382 | return UnableToLegalize; |
| 2383 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2384 | if (Offset % SrcTy.getScalarSizeInBits() != 0) |
| 2385 | return UnableToLegalize; |
| 2386 | |
| 2387 | Observer.changingInstr(MI); |
| 2388 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2389 | |
| 2390 | MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) * |
| 2391 | Offset); |
| 2392 | widenScalarDst(MI, WideTy.getScalarType(), 0); |
| 2393 | Observer.changedInstr(MI); |
| 2394 | return Legalized; |
| 2395 | } |
| 2396 | |
| 2397 | LegalizerHelper::LegalizeResult |
| 2398 | LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, |
| 2399 | LLT WideTy) { |
Matt Arsenault | 5cbd4e4 | 2020-07-18 12:27:16 -0400 | [diff] [blame] | 2400 | if (TypeIdx != 0 || WideTy.isVector()) |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2401 | return UnableToLegalize; |
| 2402 | Observer.changingInstr(MI); |
| 2403 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2404 | widenScalarDst(MI, WideTy); |
| 2405 | Observer.changedInstr(MI); |
| 2406 | return Legalized; |
| 2407 | } |
| 2408 | |
| 2409 | LegalizerHelper::LegalizeResult |
Cassie Jones | f22f455 | 2021-01-28 13:20:35 -0500 | [diff] [blame] | 2410 | LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, |
| 2411 | LLT WideTy) { |
Cassie Jones | f22f455 | 2021-01-28 13:20:35 -0500 | [diff] [blame] | 2412 | unsigned Opcode; |
| 2413 | unsigned ExtOpcode; |
Kazu Hirata | 3ccbfc3 | 2022-11-26 14:44:54 -0800 | [diff] [blame] | 2414 | std::optional<Register> CarryIn; |
Cassie Jones | f22f455 | 2021-01-28 13:20:35 -0500 | [diff] [blame] | 2415 | switch (MI.getOpcode()) { |
| 2416 | default: |
| 2417 | llvm_unreachable("Unexpected opcode!"); |
| 2418 | case TargetOpcode::G_SADDO: |
| 2419 | Opcode = TargetOpcode::G_ADD; |
| 2420 | ExtOpcode = TargetOpcode::G_SEXT; |
| 2421 | break; |
| 2422 | case TargetOpcode::G_SSUBO: |
| 2423 | Opcode = TargetOpcode::G_SUB; |
| 2424 | ExtOpcode = TargetOpcode::G_SEXT; |
| 2425 | break; |
| 2426 | case TargetOpcode::G_UADDO: |
| 2427 | Opcode = TargetOpcode::G_ADD; |
| 2428 | ExtOpcode = TargetOpcode::G_ZEXT; |
| 2429 | break; |
| 2430 | case TargetOpcode::G_USUBO: |
| 2431 | Opcode = TargetOpcode::G_SUB; |
| 2432 | ExtOpcode = TargetOpcode::G_ZEXT; |
| 2433 | break; |
| 2434 | case TargetOpcode::G_SADDE: |
| 2435 | Opcode = TargetOpcode::G_UADDE; |
| 2436 | ExtOpcode = TargetOpcode::G_SEXT; |
| 2437 | CarryIn = MI.getOperand(4).getReg(); |
| 2438 | break; |
| 2439 | case TargetOpcode::G_SSUBE: |
| 2440 | Opcode = TargetOpcode::G_USUBE; |
| 2441 | ExtOpcode = TargetOpcode::G_SEXT; |
| 2442 | CarryIn = MI.getOperand(4).getReg(); |
| 2443 | break; |
| 2444 | case TargetOpcode::G_UADDE: |
| 2445 | Opcode = TargetOpcode::G_UADDE; |
| 2446 | ExtOpcode = TargetOpcode::G_ZEXT; |
| 2447 | CarryIn = MI.getOperand(4).getReg(); |
| 2448 | break; |
| 2449 | case TargetOpcode::G_USUBE: |
| 2450 | Opcode = TargetOpcode::G_USUBE; |
| 2451 | ExtOpcode = TargetOpcode::G_ZEXT; |
| 2452 | CarryIn = MI.getOperand(4).getReg(); |
| 2453 | break; |
| 2454 | } |
| 2455 | |
Matt Arsenault | 0e48992 | 2022-04-12 11:49:22 -0400 | [diff] [blame] | 2456 | if (TypeIdx == 1) { |
| 2457 | unsigned BoolExtOp = MIRBuilder.getBoolExtOp(WideTy.isVector(), false); |
| 2458 | |
| 2459 | Observer.changingInstr(MI); |
Matt Arsenault | 0e48992 | 2022-04-12 11:49:22 -0400 | [diff] [blame] | 2460 | if (CarryIn) |
| 2461 | widenScalarSrc(MI, WideTy, 4, BoolExtOp); |
Tomas Matheson | 9a390d6 | 2022-08-23 17:01:53 +0100 | [diff] [blame] | 2462 | widenScalarDst(MI, WideTy, 1); |
Matt Arsenault | 0e48992 | 2022-04-12 11:49:22 -0400 | [diff] [blame] | 2463 | |
| 2464 | Observer.changedInstr(MI); |
| 2465 | return Legalized; |
| 2466 | } |
| 2467 | |
Mitch Phillips | c9466ed | 2021-01-22 14:25:31 -0800 | [diff] [blame] | 2468 | auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)}); |
| 2469 | auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)}); |
| 2470 | // Do the arithmetic in the larger type. |
Cassie Jones | f22f455 | 2021-01-28 13:20:35 -0500 | [diff] [blame] | 2471 | Register NewOp; |
| 2472 | if (CarryIn) { |
| 2473 | LLT CarryOutTy = MRI.getType(MI.getOperand(1).getReg()); |
| 2474 | NewOp = MIRBuilder |
| 2475 | .buildInstr(Opcode, {WideTy, CarryOutTy}, |
| 2476 | {LHSExt, RHSExt, *CarryIn}) |
| 2477 | .getReg(0); |
| 2478 | } else { |
| 2479 | NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSExt, RHSExt}).getReg(0); |
| 2480 | } |
Mitch Phillips | c9466ed | 2021-01-22 14:25:31 -0800 | [diff] [blame] | 2481 | LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); |
| 2482 | auto TruncOp = MIRBuilder.buildTrunc(OrigTy, NewOp); |
| 2483 | auto ExtOp = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {TruncOp}); |
| 2484 | // There is no overflow if the ExtOp is the same as NewOp. |
| 2485 | MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, ExtOp); |
| 2486 | // Now trunc the NewOp to the original result. |
| 2487 | MIRBuilder.buildTrunc(MI.getOperand(0), NewOp); |
| 2488 | MI.eraseFromParent(); |
| 2489 | return Legalized; |
| 2490 | } |
| 2491 | |
| 2492 | LegalizerHelper::LegalizeResult |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2493 | LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, |
| 2494 | LLT WideTy) { |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2495 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT || |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2496 | MI.getOpcode() == TargetOpcode::G_SSUBSAT || |
| 2497 | MI.getOpcode() == TargetOpcode::G_SSHLSAT; |
| 2498 | bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT || |
| 2499 | MI.getOpcode() == TargetOpcode::G_USHLSAT; |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2500 | // We can convert this to: |
| 2501 | // 1. Any extend iN to iM |
| 2502 | // 2. SHL by M-N |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2503 | // 3. [US][ADD|SUB|SHL]SAT |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2504 | // 4. L/ASHR by M-N |
| 2505 | // |
| 2506 | // It may be more efficient to lower this to a min and a max operation in |
| 2507 | // the higher precision arithmetic if the promoted operation isn't legal, |
| 2508 | // but this decision is up to the target's lowering request. |
| 2509 | Register DstReg = MI.getOperand(0).getReg(); |
| 2510 | |
| 2511 | unsigned NewBits = WideTy.getScalarSizeInBits(); |
| 2512 | unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits(); |
| 2513 | |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2514 | // Shifts must zero-extend the RHS to preserve the unsigned quantity, and |
| 2515 | // must not left shift the RHS to preserve the shift amount. |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2516 | auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1)); |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2517 | auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2)) |
| 2518 | : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2)); |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2519 | auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount); |
| 2520 | auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK); |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2521 | auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK); |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2522 | |
| 2523 | auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, |
| 2524 | {ShiftL, ShiftR}, MI.getFlags()); |
| 2525 | |
| 2526 | // Use a shift that will preserve the number of sign bits when the trunc is |
| 2527 | // folded away. |
| 2528 | auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK) |
| 2529 | : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK); |
| 2530 | |
| 2531 | MIRBuilder.buildTrunc(DstReg, Result); |
| 2532 | MI.eraseFromParent(); |
| 2533 | return Legalized; |
| 2534 | } |
| 2535 | |
| 2536 | LegalizerHelper::LegalizeResult |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2537 | LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, |
| 2538 | LLT WideTy) { |
Matt Arsenault | 95c2bcb | 2022-04-12 12:03:04 -0400 | [diff] [blame] | 2539 | if (TypeIdx == 1) { |
| 2540 | Observer.changingInstr(MI); |
| 2541 | widenScalarDst(MI, WideTy, 1); |
| 2542 | Observer.changedInstr(MI); |
| 2543 | return Legalized; |
| 2544 | } |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2545 | |
| 2546 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO; |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 2547 | auto [Result, OriginalOverflow, LHS, RHS] = MI.getFirst4Regs(); |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2548 | LLT SrcTy = MRI.getType(LHS); |
| 2549 | LLT OverflowTy = MRI.getType(OriginalOverflow); |
| 2550 | unsigned SrcBitWidth = SrcTy.getScalarSizeInBits(); |
| 2551 | |
| 2552 | // To determine if the result overflowed in the larger type, we extend the |
| 2553 | // input to the larger type, do the multiply (checking if it overflows), |
| 2554 | // then also check the high bits of the result to see if overflow happened |
| 2555 | // there. |
| 2556 | unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; |
| 2557 | auto LeftOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {LHS}); |
| 2558 | auto RightOperand = MIRBuilder.buildInstr(ExtOp, {WideTy}, {RHS}); |
| 2559 | |
Craig Topper | 3750558 | 2023-10-13 20:34:45 -0700 | [diff] [blame] | 2560 | // Multiplication cannot overflow if the WideTy is >= 2 * original width, |
| 2561 | // so we don't need to check the overflow result of larger type Mulo. |
| 2562 | bool WideMulCanOverflow = WideTy.getScalarSizeInBits() < 2 * SrcBitWidth; |
| 2563 | |
| 2564 | unsigned MulOpc = |
| 2565 | WideMulCanOverflow ? MI.getOpcode() : (unsigned)TargetOpcode::G_MUL; |
| 2566 | |
| 2567 | MachineInstrBuilder Mulo; |
| 2568 | if (WideMulCanOverflow) |
| 2569 | Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy, OverflowTy}, |
| 2570 | {LeftOperand, RightOperand}); |
| 2571 | else |
| 2572 | Mulo = MIRBuilder.buildInstr(MulOpc, {WideTy}, {LeftOperand, RightOperand}); |
| 2573 | |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2574 | auto Mul = Mulo->getOperand(0); |
| 2575 | MIRBuilder.buildTrunc(Result, Mul); |
| 2576 | |
| 2577 | MachineInstrBuilder ExtResult; |
| 2578 | // Overflow occurred if it occurred in the larger type, or if the high part |
| 2579 | // of the result does not zero/sign-extend the low part. Check this second |
| 2580 | // possibility first. |
| 2581 | if (IsSigned) { |
| 2582 | // For signed, overflow occurred when the high part does not sign-extend |
| 2583 | // the low part. |
| 2584 | ExtResult = MIRBuilder.buildSExtInReg(WideTy, Mul, SrcBitWidth); |
| 2585 | } else { |
| 2586 | // Unsigned overflow occurred when the high part does not zero-extend the |
| 2587 | // low part. |
| 2588 | ExtResult = MIRBuilder.buildZExtInReg(WideTy, Mul, SrcBitWidth); |
| 2589 | } |
| 2590 | |
Craig Topper | 3750558 | 2023-10-13 20:34:45 -0700 | [diff] [blame] | 2591 | if (WideMulCanOverflow) { |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2592 | auto Overflow = |
| 2593 | MIRBuilder.buildICmp(CmpInst::ICMP_NE, OverflowTy, Mul, ExtResult); |
| 2594 | // Finally check if the multiplication in the larger type itself overflowed. |
| 2595 | MIRBuilder.buildOr(OriginalOverflow, Mulo->getOperand(1), Overflow); |
| 2596 | } else { |
| 2597 | MIRBuilder.buildICmp(CmpInst::ICMP_NE, OriginalOverflow, Mul, ExtResult); |
| 2598 | } |
| 2599 | MI.eraseFromParent(); |
| 2600 | return Legalized; |
| 2601 | } |
| 2602 | |
| 2603 | LegalizerHelper::LegalizeResult |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 2604 | LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2605 | unsigned Opcode = MI.getOpcode(); |
| 2606 | switch (Opcode) { |
Tim Northover | 3233581 | 2016-08-04 18:35:11 +0000 | [diff] [blame] | 2607 | default: |
| 2608 | return UnableToLegalize; |
Tim Northover | 291e0da | 2021-07-21 09:05:56 +0100 | [diff] [blame] | 2609 | case TargetOpcode::G_ATOMICRMW_XCHG: |
| 2610 | case TargetOpcode::G_ATOMICRMW_ADD: |
| 2611 | case TargetOpcode::G_ATOMICRMW_SUB: |
| 2612 | case TargetOpcode::G_ATOMICRMW_AND: |
| 2613 | case TargetOpcode::G_ATOMICRMW_OR: |
| 2614 | case TargetOpcode::G_ATOMICRMW_XOR: |
| 2615 | case TargetOpcode::G_ATOMICRMW_MIN: |
| 2616 | case TargetOpcode::G_ATOMICRMW_MAX: |
| 2617 | case TargetOpcode::G_ATOMICRMW_UMIN: |
| 2618 | case TargetOpcode::G_ATOMICRMW_UMAX: |
| 2619 | assert(TypeIdx == 0 && "atomicrmw with second scalar type"); |
| 2620 | Observer.changingInstr(MI); |
| 2621 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); |
| 2622 | widenScalarDst(MI, WideTy, 0); |
| 2623 | Observer.changedInstr(MI); |
| 2624 | return Legalized; |
| 2625 | case TargetOpcode::G_ATOMIC_CMPXCHG: |
| 2626 | assert(TypeIdx == 0 && "G_ATOMIC_CMPXCHG with second scalar type"); |
| 2627 | Observer.changingInstr(MI); |
| 2628 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); |
| 2629 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); |
| 2630 | widenScalarDst(MI, WideTy, 0); |
| 2631 | Observer.changedInstr(MI); |
| 2632 | return Legalized; |
| 2633 | case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: |
| 2634 | if (TypeIdx == 0) { |
| 2635 | Observer.changingInstr(MI); |
| 2636 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); |
| 2637 | widenScalarSrc(MI, WideTy, 4, TargetOpcode::G_ANYEXT); |
| 2638 | widenScalarDst(MI, WideTy, 0); |
| 2639 | Observer.changedInstr(MI); |
| 2640 | return Legalized; |
| 2641 | } |
| 2642 | assert(TypeIdx == 1 && |
| 2643 | "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type"); |
| 2644 | Observer.changingInstr(MI); |
| 2645 | widenScalarDst(MI, WideTy, 1); |
| 2646 | Observer.changedInstr(MI); |
| 2647 | return Legalized; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2648 | case TargetOpcode::G_EXTRACT: |
| 2649 | return widenScalarExtract(MI, TypeIdx, WideTy); |
| 2650 | case TargetOpcode::G_INSERT: |
| 2651 | return widenScalarInsert(MI, TypeIdx, WideTy); |
Matt Arsenault | 888aa5d | 2019-02-03 00:07:33 +0000 | [diff] [blame] | 2652 | case TargetOpcode::G_MERGE_VALUES: |
| 2653 | return widenScalarMergeValues(MI, TypeIdx, WideTy); |
| 2654 | case TargetOpcode::G_UNMERGE_VALUES: |
| 2655 | return widenScalarUnmergeValues(MI, TypeIdx, WideTy); |
Cassie Jones | aa8f367 | 2021-01-25 16:57:20 -0500 | [diff] [blame] | 2656 | case TargetOpcode::G_SADDO: |
Mitch Phillips | c9466ed | 2021-01-22 14:25:31 -0800 | [diff] [blame] | 2657 | case TargetOpcode::G_SSUBO: |
Aditya Nandakumar | 6d47a41 | 2018-08-29 03:17:08 +0000 | [diff] [blame] | 2658 | case TargetOpcode::G_UADDO: |
Mitch Phillips | c9466ed | 2021-01-22 14:25:31 -0800 | [diff] [blame] | 2659 | case TargetOpcode::G_USUBO: |
Cassie Jones | f22f455 | 2021-01-28 13:20:35 -0500 | [diff] [blame] | 2660 | case TargetOpcode::G_SADDE: |
| 2661 | case TargetOpcode::G_SSUBE: |
| 2662 | case TargetOpcode::G_UADDE: |
| 2663 | case TargetOpcode::G_USUBE: |
| 2664 | return widenScalarAddSubOverflow(MI, TypeIdx, WideTy); |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 2665 | case TargetOpcode::G_UMULO: |
| 2666 | case TargetOpcode::G_SMULO: |
| 2667 | return widenScalarMulo(MI, TypeIdx, WideTy); |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2668 | case TargetOpcode::G_SADDSAT: |
| 2669 | case TargetOpcode::G_SSUBSAT: |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2670 | case TargetOpcode::G_SSHLSAT: |
Matt Arsenault | 6a8c11a | 2020-07-12 13:58:53 -0400 | [diff] [blame] | 2671 | case TargetOpcode::G_UADDSAT: |
| 2672 | case TargetOpcode::G_USUBSAT: |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 2673 | case TargetOpcode::G_USHLSAT: |
| 2674 | return widenScalarAddSubShlSat(MI, TypeIdx, WideTy); |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2675 | case TargetOpcode::G_CTTZ: |
| 2676 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: |
| 2677 | case TargetOpcode::G_CTLZ: |
| 2678 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
| 2679 | case TargetOpcode::G_CTPOP: { |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2680 | if (TypeIdx == 0) { |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2681 | Observer.changingInstr(MI); |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2682 | widenScalarDst(MI, WideTy, 0); |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2683 | Observer.changedInstr(MI); |
Matt Arsenault | d5684f7 | 2019-01-31 02:09:57 +0000 | [diff] [blame] | 2684 | return Legalized; |
| 2685 | } |
| 2686 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 2687 | Register SrcReg = MI.getOperand(1).getReg(); |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2688 | |
Jay Foad | 57b9107 | 2021-08-06 11:05:42 +0100 | [diff] [blame] | 2689 | // First extend the input. |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2690 | unsigned ExtOpc = Opcode == TargetOpcode::G_CTTZ || |
| 2691 | Opcode == TargetOpcode::G_CTTZ_ZERO_UNDEF |
Jay Foad | 57b9107 | 2021-08-06 11:05:42 +0100 | [diff] [blame] | 2692 | ? TargetOpcode::G_ANYEXT |
| 2693 | : TargetOpcode::G_ZEXT; |
| 2694 | auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg}); |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2695 | LLT CurTy = MRI.getType(SrcReg); |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2696 | unsigned NewOpc = Opcode; |
Jay Foad | cd2594e | 2021-08-04 14:37:45 +0100 | [diff] [blame] | 2697 | if (NewOpc == TargetOpcode::G_CTTZ) { |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2698 | // The count is the same in the larger type except if the original |
| 2699 | // value was zero. This can be handled by setting the bit just off |
| 2700 | // the top of the original type. |
| 2701 | auto TopBit = |
| 2702 | APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits()); |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2703 | MIBSrc = MIRBuilder.buildOr( |
| 2704 | WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); |
Jay Foad | cd2594e | 2021-08-04 14:37:45 +0100 | [diff] [blame] | 2705 | // Now we know the operand is non-zero, use the more relaxed opcode. |
| 2706 | NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF; |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2707 | } |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2708 | |
Manish Kausik H | 69192e0 | 2024-07-08 18:31:32 +0530 | [diff] [blame] | 2709 | unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits(); |
| 2710 | |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2711 | if (Opcode == TargetOpcode::G_CTLZ_ZERO_UNDEF) { |
Manish Kausik H | 69192e0 | 2024-07-08 18:31:32 +0530 | [diff] [blame] | 2712 | // An optimization where the result is the CTLZ after the left shift by |
| 2713 | // (Difference in widety and current ty), that is, |
| 2714 | // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy)) |
| 2715 | // Result = ctlz MIBSrc |
| 2716 | MIBSrc = MIRBuilder.buildShl(WideTy, MIBSrc, |
| 2717 | MIRBuilder.buildConstant(WideTy, SizeDiff)); |
| 2718 | } |
| 2719 | |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2720 | // Perform the operation at the larger size. |
Jay Foad | cd2594e | 2021-08-04 14:37:45 +0100 | [diff] [blame] | 2721 | auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc}); |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2722 | // This is already the correct result for CTPOP and CTTZs |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2723 | if (Opcode == TargetOpcode::G_CTLZ) { |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2724 | // The correct result is NewOp - (Difference in widety and current ty). |
Jay Foad | 28bb43b | 2020-01-16 12:09:48 +0000 | [diff] [blame] | 2725 | MIBNewOp = MIRBuilder.buildSub( |
| 2726 | WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)); |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2727 | } |
Matt Arsenault | 3d6a49b | 2019-02-04 22:26:33 +0000 | [diff] [blame] | 2728 | |
| 2729 | MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); |
| 2730 | MI.eraseFromParent(); |
Aditya Nandakumar | c106183 | 2018-08-22 17:59:18 +0000 | [diff] [blame] | 2731 | return Legalized; |
| 2732 | } |
Matt Arsenault | d1bfc8d | 2019-01-31 02:34:03 +0000 | [diff] [blame] | 2733 | case TargetOpcode::G_BSWAP: { |
| 2734 | Observer.changingInstr(MI); |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 2735 | Register DstReg = MI.getOperand(0).getReg(); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2736 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 2737 | Register ShrReg = MRI.createGenericVirtualRegister(WideTy); |
| 2738 | Register DstExt = MRI.createGenericVirtualRegister(WideTy); |
| 2739 | Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy); |
Matt Arsenault | d1bfc8d | 2019-01-31 02:34:03 +0000 | [diff] [blame] | 2740 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2741 | |
| 2742 | MI.getOperand(0).setReg(DstExt); |
| 2743 | |
| 2744 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
| 2745 | |
| 2746 | LLT Ty = MRI.getType(DstReg); |
| 2747 | unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); |
| 2748 | MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); |
Jay Foad | 28bb43b | 2020-01-16 12:09:48 +0000 | [diff] [blame] | 2749 | MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg); |
Matt Arsenault | d1bfc8d | 2019-01-31 02:34:03 +0000 | [diff] [blame] | 2750 | |
| 2751 | MIRBuilder.buildTrunc(DstReg, ShrReg); |
| 2752 | Observer.changedInstr(MI); |
| 2753 | return Legalized; |
| 2754 | } |
Matt Arsenault | 5ff310e | 2019-09-04 20:46:15 +0000 | [diff] [blame] | 2755 | case TargetOpcode::G_BITREVERSE: { |
| 2756 | Observer.changingInstr(MI); |
| 2757 | |
| 2758 | Register DstReg = MI.getOperand(0).getReg(); |
| 2759 | LLT Ty = MRI.getType(DstReg); |
| 2760 | unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); |
| 2761 | |
| 2762 | Register DstExt = MRI.createGenericVirtualRegister(WideTy); |
| 2763 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2764 | MI.getOperand(0).setReg(DstExt); |
| 2765 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
| 2766 | |
| 2767 | auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits); |
| 2768 | auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt); |
| 2769 | MIRBuilder.buildTrunc(DstReg, Shift); |
| 2770 | Observer.changedInstr(MI); |
| 2771 | return Legalized; |
| 2772 | } |
Dominik Montada | 55e3a7c | 2020-04-14 11:25:05 +0200 | [diff] [blame] | 2773 | case TargetOpcode::G_FREEZE: |
Yingwei Zheng | 821bcba | 2024-05-22 23:35:37 +0800 | [diff] [blame] | 2774 | case TargetOpcode::G_CONSTANT_FOLD_BARRIER: |
Dominik Montada | 55e3a7c | 2020-04-14 11:25:05 +0200 | [diff] [blame] | 2775 | Observer.changingInstr(MI); |
| 2776 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2777 | widenScalarDst(MI, WideTy); |
| 2778 | Observer.changedInstr(MI); |
| 2779 | return Legalized; |
| 2780 | |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 2781 | case TargetOpcode::G_ABS: |
| 2782 | Observer.changingInstr(MI); |
| 2783 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); |
| 2784 | widenScalarDst(MI, WideTy); |
| 2785 | Observer.changedInstr(MI); |
| 2786 | return Legalized; |
| 2787 | |
Tim Northover | 61c1614 | 2016-08-04 21:39:49 +0000 | [diff] [blame] | 2788 | case TargetOpcode::G_ADD: |
| 2789 | case TargetOpcode::G_AND: |
| 2790 | case TargetOpcode::G_MUL: |
| 2791 | case TargetOpcode::G_OR: |
| 2792 | case TargetOpcode::G_XOR: |
Justin Bogner | ddb80ae | 2017-01-19 07:51:17 +0000 | [diff] [blame] | 2793 | case TargetOpcode::G_SUB: |
Tuan Chuong Goh | 13a78fd | 2024-03-04 14:27:21 +0000 | [diff] [blame] | 2794 | case TargetOpcode::G_SHUFFLE_VECTOR: |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 2795 | // Perform operation at larger width (any extension is fines here, high bits |
Tim Northover | 3233581 | 2016-08-04 18:35:11 +0000 | [diff] [blame] | 2796 | // don't affect the result) and then truncate the result back to the |
| 2797 | // original type. |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2798 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2799 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2800 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); |
| 2801 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2802 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2803 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2804 | |
Brendon Cahoon | f9f5d41 | 2021-04-30 09:57:44 -0400 | [diff] [blame] | 2805 | case TargetOpcode::G_SBFX: |
| 2806 | case TargetOpcode::G_UBFX: |
| 2807 | Observer.changingInstr(MI); |
| 2808 | |
| 2809 | if (TypeIdx == 0) { |
| 2810 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2811 | widenScalarDst(MI, WideTy); |
| 2812 | } else { |
| 2813 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2814 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT); |
| 2815 | } |
| 2816 | |
| 2817 | Observer.changedInstr(MI); |
| 2818 | return Legalized; |
| 2819 | |
Roman Tereshin | 6d26638 | 2018-05-09 21:43:30 +0000 | [diff] [blame] | 2820 | case TargetOpcode::G_SHL: |
Matt Arsenault | 012ecbb | 2019-05-16 04:08:46 +0000 | [diff] [blame] | 2821 | Observer.changingInstr(MI); |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2822 | |
| 2823 | if (TypeIdx == 0) { |
| 2824 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 2825 | widenScalarDst(MI, WideTy); |
| 2826 | } else { |
| 2827 | assert(TypeIdx == 1); |
| 2828 | // The "number of bits to shift" operand must preserve its value as an |
| 2829 | // unsigned integer: |
| 2830 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2831 | } |
| 2832 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2833 | Observer.changedInstr(MI); |
Roman Tereshin | 6d26638 | 2018-05-09 21:43:30 +0000 | [diff] [blame] | 2834 | return Legalized; |
| 2835 | |
Craig Topper | d605d9d | 2023-12-04 13:00:34 -0800 | [diff] [blame] | 2836 | case TargetOpcode::G_ROTR: |
| 2837 | case TargetOpcode::G_ROTL: |
| 2838 | if (TypeIdx != 1) |
| 2839 | return UnableToLegalize; |
| 2840 | |
| 2841 | Observer.changingInstr(MI); |
| 2842 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2843 | Observer.changedInstr(MI); |
| 2844 | return Legalized; |
| 2845 | |
Tim Northover | 7a753d9 | 2016-08-26 17:46:06 +0000 | [diff] [blame] | 2846 | case TargetOpcode::G_SDIV: |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2847 | case TargetOpcode::G_SREM: |
Matt Arsenault | 0f3ba44 | 2019-05-23 17:58:48 +0000 | [diff] [blame] | 2848 | case TargetOpcode::G_SMIN: |
| 2849 | case TargetOpcode::G_SMAX: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2850 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2851 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); |
| 2852 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); |
| 2853 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2854 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2855 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2856 | |
Christudasan Devadasan | 90d7840 | 2021-04-12 15:49:47 +0530 | [diff] [blame] | 2857 | case TargetOpcode::G_SDIVREM: |
| 2858 | Observer.changingInstr(MI); |
| 2859 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); |
| 2860 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); |
| 2861 | widenScalarDst(MI, WideTy); |
| 2862 | widenScalarDst(MI, WideTy, 1); |
| 2863 | Observer.changedInstr(MI); |
| 2864 | return Legalized; |
| 2865 | |
Roman Tereshin | 6d26638 | 2018-05-09 21:43:30 +0000 | [diff] [blame] | 2866 | case TargetOpcode::G_ASHR: |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2867 | case TargetOpcode::G_LSHR: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2868 | Observer.changingInstr(MI); |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2869 | |
| 2870 | if (TypeIdx == 0) { |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 2871 | unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT |
| 2872 | : TargetOpcode::G_ZEXT; |
Matt Arsenault | 30989e4 | 2019-01-22 21:42:11 +0000 | [diff] [blame] | 2873 | |
| 2874 | widenScalarSrc(MI, WideTy, 1, CvtOp); |
| 2875 | widenScalarDst(MI, WideTy); |
| 2876 | } else { |
| 2877 | assert(TypeIdx == 1); |
| 2878 | // The "number of bits to shift" operand must preserve its value as an |
| 2879 | // unsigned integer: |
| 2880 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2881 | } |
| 2882 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2883 | Observer.changedInstr(MI); |
Roman Tereshin | 6d26638 | 2018-05-09 21:43:30 +0000 | [diff] [blame] | 2884 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2885 | case TargetOpcode::G_UDIV: |
| 2886 | case TargetOpcode::G_UREM: |
Matt Arsenault | 0f3ba44 | 2019-05-23 17:58:48 +0000 | [diff] [blame] | 2887 | case TargetOpcode::G_UMIN: |
| 2888 | case TargetOpcode::G_UMAX: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2889 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2890 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); |
| 2891 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2892 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2893 | Observer.changedInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2894 | return Legalized; |
| 2895 | |
Christudasan Devadasan | 90d7840 | 2021-04-12 15:49:47 +0530 | [diff] [blame] | 2896 | case TargetOpcode::G_UDIVREM: |
| 2897 | Observer.changingInstr(MI); |
| 2898 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 2899 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT); |
| 2900 | widenScalarDst(MI, WideTy); |
| 2901 | widenScalarDst(MI, WideTy, 1); |
| 2902 | Observer.changedInstr(MI); |
| 2903 | return Legalized; |
| 2904 | |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2905 | case TargetOpcode::G_SELECT: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2906 | Observer.changingInstr(MI); |
Petar Avramovic | 09dff33 | 2018-12-25 14:42:30 +0000 | [diff] [blame] | 2907 | if (TypeIdx == 0) { |
| 2908 | // Perform operation at larger width (any extension is fine here, high |
| 2909 | // bits don't affect the result) and then truncate the result back to the |
| 2910 | // original type. |
| 2911 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); |
| 2912 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); |
| 2913 | widenScalarDst(MI, WideTy); |
| 2914 | } else { |
Matt Arsenault | 6d8e1b4 | 2019-01-30 02:57:43 +0000 | [diff] [blame] | 2915 | bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector(); |
Petar Avramovic | 09dff33 | 2018-12-25 14:42:30 +0000 | [diff] [blame] | 2916 | // Explicit extension is required here since high bits affect the result. |
Matt Arsenault | 6d8e1b4 | 2019-01-30 02:57:43 +0000 | [diff] [blame] | 2917 | widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); |
Petar Avramovic | 09dff33 | 2018-12-25 14:42:30 +0000 | [diff] [blame] | 2918 | } |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2919 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2920 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2921 | |
Ahmed Bougacha | b613706 | 2017-01-23 21:10:14 +0000 | [diff] [blame] | 2922 | case TargetOpcode::G_FPTOSI: |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2923 | case TargetOpcode::G_FPTOUI: |
David Green | 28d28d5 | 2024-04-15 09:41:08 +0100 | [diff] [blame] | 2924 | case TargetOpcode::G_INTRINSIC_LRINT: |
David Green | 8d49ce1 | 2024-04-17 18:38:24 +0100 | [diff] [blame] | 2925 | case TargetOpcode::G_INTRINSIC_LLRINT: |
Min-Yih Hsu | 7c3c8a1 | 2023-11-22 16:43:20 -0800 | [diff] [blame] | 2926 | case TargetOpcode::G_IS_FPCLASS: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2927 | Observer.changingInstr(MI); |
Matt Arsenault | ed85b0c | 2019-10-01 01:06:48 +0000 | [diff] [blame] | 2928 | |
| 2929 | if (TypeIdx == 0) |
| 2930 | widenScalarDst(MI, WideTy); |
| 2931 | else |
| 2932 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); |
| 2933 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2934 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2935 | return Legalized; |
Ahmed Bougacha | d294823 | 2017-01-20 01:37:24 +0000 | [diff] [blame] | 2936 | case TargetOpcode::G_SITOFP: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2937 | Observer.changingInstr(MI); |
Petar Avramovic | 6850033 | 2020-07-16 16:31:57 +0200 | [diff] [blame] | 2938 | |
| 2939 | if (TypeIdx == 0) |
| 2940 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 2941 | else |
| 2942 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); |
| 2943 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2944 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 2945 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2946 | case TargetOpcode::G_UITOFP: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 2947 | Observer.changingInstr(MI); |
Petar Avramovic | 6850033 | 2020-07-16 16:31:57 +0200 | [diff] [blame] | 2948 | |
| 2949 | if (TypeIdx == 0) |
| 2950 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 2951 | else |
| 2952 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); |
| 2953 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 2954 | Observer.changedInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2955 | return Legalized; |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 2956 | case TargetOpcode::G_FPTOSI_SAT: |
| 2957 | case TargetOpcode::G_FPTOUI_SAT: |
| 2958 | Observer.changingInstr(MI); |
| 2959 | |
| 2960 | if (TypeIdx == 0) { |
| 2961 | Register OldDst = MI.getOperand(0).getReg(); |
| 2962 | LLT Ty = MRI.getType(OldDst); |
| 2963 | Register ExtReg = MRI.createGenericVirtualRegister(WideTy); |
| 2964 | Register NewDst; |
| 2965 | MI.getOperand(0).setReg(ExtReg); |
| 2966 | uint64_t ShortBits = Ty.getScalarSizeInBits(); |
| 2967 | uint64_t WideBits = WideTy.getScalarSizeInBits(); |
| 2968 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
| 2969 | if (Opcode == TargetOpcode::G_FPTOSI_SAT) { |
| 2970 | // z = i16 fptosi_sat(a) |
| 2971 | // -> |
| 2972 | // x = i32 fptosi_sat(a) |
| 2973 | // y = smin(x, 32767) |
| 2974 | // z = smax(y, -32768) |
| 2975 | auto MaxVal = MIRBuilder.buildConstant( |
| 2976 | WideTy, APInt::getSignedMaxValue(ShortBits).sext(WideBits)); |
| 2977 | auto MinVal = MIRBuilder.buildConstant( |
| 2978 | WideTy, APInt::getSignedMinValue(ShortBits).sext(WideBits)); |
| 2979 | Register MidReg = |
| 2980 | MIRBuilder.buildSMin(WideTy, ExtReg, MaxVal).getReg(0); |
| 2981 | NewDst = MIRBuilder.buildSMax(WideTy, MidReg, MinVal).getReg(0); |
| 2982 | } else { |
| 2983 | // z = i16 fptoui_sat(a) |
| 2984 | // -> |
| 2985 | // x = i32 fptoui_sat(a) |
| 2986 | // y = smin(x, 65535) |
| 2987 | auto MaxVal = MIRBuilder.buildConstant( |
| 2988 | WideTy, APInt::getAllOnes(ShortBits).zext(WideBits)); |
| 2989 | NewDst = MIRBuilder.buildUMin(WideTy, ExtReg, MaxVal).getReg(0); |
| 2990 | } |
| 2991 | MIRBuilder.buildTrunc(OldDst, NewDst); |
| 2992 | } else |
| 2993 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); |
| 2994 | |
| 2995 | Observer.changedInstr(MI); |
| 2996 | return Legalized; |
Daniel Sanders | 5eb9f58 | 2018-04-28 18:14:50 +0000 | [diff] [blame] | 2997 | case TargetOpcode::G_LOAD: |
Daniel Sanders | 5eb9f58 | 2018-04-28 18:14:50 +0000 | [diff] [blame] | 2998 | case TargetOpcode::G_SEXTLOAD: |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 2999 | case TargetOpcode::G_ZEXTLOAD: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3000 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3001 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3002 | Observer.changedInstr(MI); |
Tim Northover | 3c73e36 | 2016-08-23 18:20:09 +0000 | [diff] [blame] | 3003 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3004 | |
Tim Northover | 3c73e36 | 2016-08-23 18:20:09 +0000 | [diff] [blame] | 3005 | case TargetOpcode::G_STORE: { |
Matt Arsenault | 92c5001 | 2019-01-30 02:04:31 +0000 | [diff] [blame] | 3006 | if (TypeIdx != 0) |
| 3007 | return UnableToLegalize; |
| 3008 | |
| 3009 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
Matt Arsenault | 88bdcbb | 2020-08-22 12:34:38 -0400 | [diff] [blame] | 3010 | if (!Ty.isScalar()) |
Tim Northover | 548feee | 2017-03-21 22:22:05 +0000 | [diff] [blame] | 3011 | return UnableToLegalize; |
| 3012 | |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3013 | Observer.changingInstr(MI); |
Matt Arsenault | 92c5001 | 2019-01-30 02:04:31 +0000 | [diff] [blame] | 3014 | |
| 3015 | unsigned ExtType = Ty.getScalarSizeInBits() == 1 ? |
| 3016 | TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; |
| 3017 | widenScalarSrc(MI, WideTy, 0, ExtType); |
| 3018 | |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3019 | Observer.changedInstr(MI); |
Tim Northover | 3c73e36 | 2016-08-23 18:20:09 +0000 | [diff] [blame] | 3020 | return Legalized; |
| 3021 | } |
Tim Northover | ea904f9 | 2016-08-19 22:40:00 +0000 | [diff] [blame] | 3022 | case TargetOpcode::G_CONSTANT: { |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3023 | MachineOperand &SrcMO = MI.getOperand(1); |
| 3024 | LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
Aditya Nandakumar | 6da7dbb | 2019-12-03 10:40:03 -0800 | [diff] [blame] | 3025 | unsigned ExtOpc = LI.getExtOpcodeForWideningConstant( |
| 3026 | MRI.getType(MI.getOperand(0).getReg())); |
| 3027 | assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT || |
| 3028 | ExtOpc == TargetOpcode::G_ANYEXT) && |
| 3029 | "Illegal Extend"); |
| 3030 | const APInt &SrcVal = SrcMO.getCImm()->getValue(); |
| 3031 | const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT) |
| 3032 | ? SrcVal.sext(WideTy.getSizeInBits()) |
| 3033 | : SrcVal.zext(WideTy.getSizeInBits()); |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3034 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3035 | SrcMO.setCImm(ConstantInt::get(Ctx, Val)); |
| 3036 | |
| 3037 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3038 | Observer.changedInstr(MI); |
Tim Northover | ea904f9 | 2016-08-19 22:40:00 +0000 | [diff] [blame] | 3039 | return Legalized; |
| 3040 | } |
Tim Northover | a11be04 | 2016-08-19 22:40:08 +0000 | [diff] [blame] | 3041 | case TargetOpcode::G_FCONSTANT: { |
Amara Emerson | d4f84df | 2022-07-14 00:53:59 -0700 | [diff] [blame] | 3042 | // To avoid changing the bits of the constant due to extension to a larger |
| 3043 | // type and then using G_FPTRUNC, we simply convert to a G_CONSTANT. |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3044 | MachineOperand &SrcMO = MI.getOperand(1); |
Amara Emerson | d4f84df | 2022-07-14 00:53:59 -0700 | [diff] [blame] | 3045 | APInt Val = SrcMO.getFPImm()->getValueAPF().bitcastToAPInt(); |
| 3046 | MIRBuilder.setInstrAndDebugLoc(MI); |
| 3047 | auto IntCst = MIRBuilder.buildConstant(MI.getOperand(0).getReg(), Val); |
| 3048 | widenScalarDst(*IntCst, WideTy, 0, TargetOpcode::G_TRUNC); |
| 3049 | MI.eraseFromParent(); |
Roman Tereshin | 25cbfe6 | 2018-05-08 22:53:09 +0000 | [diff] [blame] | 3050 | return Legalized; |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 3051 | } |
Matt Arsenault | befee40 | 2019-01-09 07:34:14 +0000 | [diff] [blame] | 3052 | case TargetOpcode::G_IMPLICIT_DEF: { |
| 3053 | Observer.changingInstr(MI); |
| 3054 | widenScalarDst(MI, WideTy); |
| 3055 | Observer.changedInstr(MI); |
| 3056 | return Legalized; |
| 3057 | } |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3058 | case TargetOpcode::G_BRCOND: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3059 | Observer.changingInstr(MI); |
Petar Avramovic | 5d9b8ee | 2019-02-14 11:39:53 +0000 | [diff] [blame] | 3060 | widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3061 | Observer.changedInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3062 | return Legalized; |
| 3063 | |
| 3064 | case TargetOpcode::G_FCMP: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3065 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3066 | if (TypeIdx == 0) |
| 3067 | widenScalarDst(MI, WideTy); |
| 3068 | else { |
| 3069 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); |
| 3070 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 3071 | } |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3072 | Observer.changedInstr(MI); |
Roman Tereshin | 27bba44 | 2018-05-09 01:43:12 +0000 | [diff] [blame] | 3073 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3074 | |
| 3075 | case TargetOpcode::G_ICMP: |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3076 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3077 | if (TypeIdx == 0) |
| 3078 | widenScalarDst(MI, WideTy); |
| 3079 | else { |
| 3080 | unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>( |
| 3081 | MI.getOperand(1).getPredicate())) |
| 3082 | ? TargetOpcode::G_SEXT |
| 3083 | : TargetOpcode::G_ZEXT; |
| 3084 | widenScalarSrc(MI, WideTy, 2, ExtOpcode); |
| 3085 | widenScalarSrc(MI, WideTy, 3, ExtOpcode); |
| 3086 | } |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3087 | Observer.changedInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3088 | return Legalized; |
| 3089 | |
Daniel Sanders | e74c5b9 | 2019-11-01 13:18:00 -0700 | [diff] [blame] | 3090 | case TargetOpcode::G_PTR_ADD: |
| 3091 | assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD"); |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3092 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3093 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3094 | Observer.changedInstr(MI); |
Tim Northover | 22d82cf | 2016-09-15 11:02:19 +0000 | [diff] [blame] | 3095 | return Legalized; |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3096 | |
Aditya Nandakumar | 892979e | 2017-08-25 04:57:27 +0000 | [diff] [blame] | 3097 | case TargetOpcode::G_PHI: { |
| 3098 | assert(TypeIdx == 0 && "Expecting only Idx 0"); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3099 | |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3100 | Observer.changingInstr(MI); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3101 | for (unsigned I = 1; I < MI.getNumOperands(); I += 2) { |
| 3102 | MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); |
Amara Emerson | 53445f5 | 2022-11-13 01:43:04 -0800 | [diff] [blame] | 3103 | MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3104 | widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT); |
Aditya Nandakumar | 892979e | 2017-08-25 04:57:27 +0000 | [diff] [blame] | 3105 | } |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3106 | |
| 3107 | MachineBasicBlock &MBB = *MI.getParent(); |
Amara Emerson | 9d64721 | 2019-09-16 23:46:03 +0000 | [diff] [blame] | 3108 | MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); |
Roman Tereshin | d5fa9fd | 2018-05-09 17:28:18 +0000 | [diff] [blame] | 3109 | widenScalarDst(MI, WideTy); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3110 | Observer.changedInstr(MI); |
Aditya Nandakumar | 892979e | 2017-08-25 04:57:27 +0000 | [diff] [blame] | 3111 | return Legalized; |
| 3112 | } |
Matt Arsenault | 6378629 | 2019-01-22 20:38:15 +0000 | [diff] [blame] | 3113 | case TargetOpcode::G_EXTRACT_VECTOR_ELT: { |
| 3114 | if (TypeIdx == 0) { |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 3115 | Register VecReg = MI.getOperand(1).getReg(); |
Matt Arsenault | 6378629 | 2019-01-22 20:38:15 +0000 | [diff] [blame] | 3116 | LLT VecTy = MRI.getType(VecReg); |
| 3117 | Observer.changingInstr(MI); |
| 3118 | |
Sander de Smalen | d5e14ba | 2021-06-24 09:58:21 +0100 | [diff] [blame] | 3119 | widenScalarSrc( |
| 3120 | MI, LLT::vector(VecTy.getElementCount(), WideTy.getSizeInBits()), 1, |
Amara Emerson | dafcbfd | 2021-09-24 22:52:30 -0700 | [diff] [blame] | 3121 | TargetOpcode::G_ANYEXT); |
Matt Arsenault | 6378629 | 2019-01-22 20:38:15 +0000 | [diff] [blame] | 3122 | |
| 3123 | widenScalarDst(MI, WideTy, 0); |
| 3124 | Observer.changedInstr(MI); |
| 3125 | return Legalized; |
| 3126 | } |
| 3127 | |
Amara Emerson | cbd86d8 | 2018-10-25 14:04:54 +0000 | [diff] [blame] | 3128 | if (TypeIdx != 2) |
| 3129 | return UnableToLegalize; |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 3130 | Observer.changingInstr(MI); |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3131 | // TODO: Probably should be zext |
Amara Emerson | cbd86d8 | 2018-10-25 14:04:54 +0000 | [diff] [blame] | 3132 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 3133 | Observer.changedInstr(MI); |
Amara Emerson | cbd86d8 | 2018-10-25 14:04:54 +0000 | [diff] [blame] | 3134 | return Legalized; |
Matt Arsenault | 6378629 | 2019-01-22 20:38:15 +0000 | [diff] [blame] | 3135 | } |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3136 | case TargetOpcode::G_INSERT_VECTOR_ELT: { |
Allen | eaf23b2 | 2023-09-12 21:15:01 +0800 | [diff] [blame] | 3137 | if (TypeIdx == 0) { |
| 3138 | Observer.changingInstr(MI); |
| 3139 | const LLT WideEltTy = WideTy.getElementType(); |
| 3140 | |
| 3141 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 3142 | widenScalarSrc(MI, WideEltTy, 2, TargetOpcode::G_ANYEXT); |
| 3143 | widenScalarDst(MI, WideTy, 0); |
| 3144 | Observer.changedInstr(MI); |
| 3145 | return Legalized; |
| 3146 | } |
| 3147 | |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3148 | if (TypeIdx == 1) { |
| 3149 | Observer.changingInstr(MI); |
| 3150 | |
| 3151 | Register VecReg = MI.getOperand(1).getReg(); |
| 3152 | LLT VecTy = MRI.getType(VecReg); |
Sander de Smalen | d5e14ba | 2021-06-24 09:58:21 +0100 | [diff] [blame] | 3153 | LLT WideVecTy = LLT::vector(VecTy.getElementCount(), WideTy); |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3154 | |
| 3155 | widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT); |
| 3156 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT); |
| 3157 | widenScalarDst(MI, WideVecTy, 0); |
| 3158 | Observer.changedInstr(MI); |
| 3159 | return Legalized; |
| 3160 | } |
| 3161 | |
| 3162 | if (TypeIdx == 2) { |
| 3163 | Observer.changingInstr(MI); |
| 3164 | // TODO: Probably should be zext |
| 3165 | widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); |
| 3166 | Observer.changedInstr(MI); |
Matt Arsenault | e4f19d1 | 2020-06-16 11:39:44 -0400 | [diff] [blame] | 3167 | return Legalized; |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3168 | } |
| 3169 | |
Matt Arsenault | e4f19d1 | 2020-06-16 11:39:44 -0400 | [diff] [blame] | 3170 | return UnableToLegalize; |
Matt Arsenault | 1a276d1 | 2019-10-01 15:51:37 -0400 | [diff] [blame] | 3171 | } |
Matt Arsenault | 745fd9f | 2019-01-20 19:10:31 +0000 | [diff] [blame] | 3172 | case TargetOpcode::G_FADD: |
| 3173 | case TargetOpcode::G_FMUL: |
| 3174 | case TargetOpcode::G_FSUB: |
| 3175 | case TargetOpcode::G_FMA: |
Matt Arsenault | cf10372 | 2019-09-06 20:49:10 +0000 | [diff] [blame] | 3176 | case TargetOpcode::G_FMAD: |
Matt Arsenault | 745fd9f | 2019-01-20 19:10:31 +0000 | [diff] [blame] | 3177 | case TargetOpcode::G_FNEG: |
| 3178 | case TargetOpcode::G_FABS: |
Matt Arsenault | 9dba67f | 2019-02-11 17:05:20 +0000 | [diff] [blame] | 3179 | case TargetOpcode::G_FCANONICALIZE: |
Matt Arsenault | 6ce1b4f | 2019-07-10 16:31:19 +0000 | [diff] [blame] | 3180 | case TargetOpcode::G_FMINNUM: |
| 3181 | case TargetOpcode::G_FMAXNUM: |
| 3182 | case TargetOpcode::G_FMINNUM_IEEE: |
| 3183 | case TargetOpcode::G_FMAXNUM_IEEE: |
| 3184 | case TargetOpcode::G_FMINIMUM: |
| 3185 | case TargetOpcode::G_FMAXIMUM: |
Matt Arsenault | 745fd9f | 2019-01-20 19:10:31 +0000 | [diff] [blame] | 3186 | case TargetOpcode::G_FDIV: |
| 3187 | case TargetOpcode::G_FREM: |
Jessica Paquette | 453ab1d | 2018-12-21 17:05:26 +0000 | [diff] [blame] | 3188 | case TargetOpcode::G_FCEIL: |
Jessica Paquette | ebdb021 | 2019-02-11 17:22:58 +0000 | [diff] [blame] | 3189 | case TargetOpcode::G_FFLOOR: |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 3190 | case TargetOpcode::G_FCOS: |
| 3191 | case TargetOpcode::G_FSIN: |
Farzon Lotfi | 1d87433 | 2024-06-05 15:01:33 -0400 | [diff] [blame] | 3192 | case TargetOpcode::G_FTAN: |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 3193 | case TargetOpcode::G_FACOS: |
| 3194 | case TargetOpcode::G_FASIN: |
| 3195 | case TargetOpcode::G_FATAN: |
Tex Riddell | c03d09c | 2024-10-24 17:53:12 -0700 | [diff] [blame] | 3196 | case TargetOpcode::G_FATAN2: |
Farzon Lotfi | 0b58f34 | 2024-07-11 15:58:43 -0400 | [diff] [blame] | 3197 | case TargetOpcode::G_FCOSH: |
| 3198 | case TargetOpcode::G_FSINH: |
| 3199 | case TargetOpcode::G_FTANH: |
Jessica Paquette | c49428a | 2019-01-28 19:53:14 +0000 | [diff] [blame] | 3200 | case TargetOpcode::G_FLOG10: |
Jessica Paquette | 2d73ecd | 2019-01-28 21:27:23 +0000 | [diff] [blame] | 3201 | case TargetOpcode::G_FLOG: |
Jessica Paquette | 0154bd1 | 2019-01-30 21:16:04 +0000 | [diff] [blame] | 3202 | case TargetOpcode::G_FLOG2: |
Jessica Paquette | d5c69e0 | 2019-04-19 23:41:52 +0000 | [diff] [blame] | 3203 | case TargetOpcode::G_FRINT: |
Jessica Paquette | ba55767 | 2019-04-25 16:44:40 +0000 | [diff] [blame] | 3204 | case TargetOpcode::G_FNEARBYINT: |
Jessica Paquette | 22457f8 | 2019-01-30 21:03:52 +0000 | [diff] [blame] | 3205 | case TargetOpcode::G_FSQRT: |
Jessica Paquette | 84bedac | 2019-01-30 23:46:15 +0000 | [diff] [blame] | 3206 | case TargetOpcode::G_FEXP: |
Jessica Paquette | e794121 | 2019-04-03 16:58:32 +0000 | [diff] [blame] | 3207 | case TargetOpcode::G_FEXP2: |
Matt Arsenault | b14e83d | 2023-08-12 07:20:00 -0400 | [diff] [blame] | 3208 | case TargetOpcode::G_FEXP10: |
Jessica Paquette | dfd87f6 | 2019-04-19 16:28:08 +0000 | [diff] [blame] | 3209 | case TargetOpcode::G_FPOW: |
Jessica Paquette | 5634264 | 2019-04-23 18:20:44 +0000 | [diff] [blame] | 3210 | case TargetOpcode::G_INTRINSIC_TRUNC: |
Jessica Paquette | 3cc6d1f | 2019-04-23 21:11:57 +0000 | [diff] [blame] | 3211 | case TargetOpcode::G_INTRINSIC_ROUND: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 3212 | case TargetOpcode::G_INTRINSIC_ROUNDEVEN: |
Matt Arsenault | 745fd9f | 2019-01-20 19:10:31 +0000 | [diff] [blame] | 3213 | assert(TypeIdx == 0); |
Jessica Paquette | 453ab1d | 2018-12-21 17:05:26 +0000 | [diff] [blame] | 3214 | Observer.changingInstr(MI); |
Matt Arsenault | 745fd9f | 2019-01-20 19:10:31 +0000 | [diff] [blame] | 3215 | |
| 3216 | for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) |
| 3217 | widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); |
| 3218 | |
Jessica Paquette | 453ab1d | 2018-12-21 17:05:26 +0000 | [diff] [blame] | 3219 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 3220 | Observer.changedInstr(MI); |
| 3221 | return Legalized; |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 3222 | case TargetOpcode::G_FPOWI: |
| 3223 | case TargetOpcode::G_FLDEXP: |
| 3224 | case TargetOpcode::G_STRICT_FLDEXP: { |
| 3225 | if (TypeIdx == 0) { |
David Green | 10fe531 | 2024-08-18 11:08:49 +0100 | [diff] [blame] | 3226 | if (Opcode == TargetOpcode::G_STRICT_FLDEXP) |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 3227 | return UnableToLegalize; |
| 3228 | |
| 3229 | Observer.changingInstr(MI); |
| 3230 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); |
| 3231 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 3232 | Observer.changedInstr(MI); |
| 3233 | return Legalized; |
| 3234 | } |
| 3235 | |
| 3236 | if (TypeIdx == 1) { |
| 3237 | // For some reason SelectionDAG tries to promote to a libcall without |
| 3238 | // actually changing the integer type for promotion. |
| 3239 | Observer.changingInstr(MI); |
| 3240 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); |
| 3241 | Observer.changedInstr(MI); |
| 3242 | return Legalized; |
| 3243 | } |
| 3244 | |
| 3245 | return UnableToLegalize; |
Matt Arsenault | 7cd8a02 | 2020-07-17 11:01:15 -0400 | [diff] [blame] | 3246 | } |
Matt Arsenault | 003b58f | 2023-04-26 21:57:10 -0400 | [diff] [blame] | 3247 | case TargetOpcode::G_FFREXP: { |
| 3248 | Observer.changingInstr(MI); |
| 3249 | |
| 3250 | if (TypeIdx == 0) { |
| 3251 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT); |
| 3252 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 3253 | } else { |
| 3254 | widenScalarDst(MI, WideTy, 1); |
| 3255 | } |
| 3256 | |
| 3257 | Observer.changedInstr(MI); |
| 3258 | return Legalized; |
| 3259 | } |
Matt Arsenault | cbaada6 | 2019-02-02 23:29:55 +0000 | [diff] [blame] | 3260 | case TargetOpcode::G_INTTOPTR: |
| 3261 | if (TypeIdx != 1) |
| 3262 | return UnableToLegalize; |
| 3263 | |
| 3264 | Observer.changingInstr(MI); |
| 3265 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); |
| 3266 | Observer.changedInstr(MI); |
| 3267 | return Legalized; |
| 3268 | case TargetOpcode::G_PTRTOINT: |
| 3269 | if (TypeIdx != 0) |
| 3270 | return UnableToLegalize; |
| 3271 | |
| 3272 | Observer.changingInstr(MI); |
| 3273 | widenScalarDst(MI, WideTy, 0); |
| 3274 | Observer.changedInstr(MI); |
| 3275 | return Legalized; |
Matt Arsenault | bd791b5 | 2019-07-08 13:48:06 +0000 | [diff] [blame] | 3276 | case TargetOpcode::G_BUILD_VECTOR: { |
| 3277 | Observer.changingInstr(MI); |
| 3278 | |
| 3279 | const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType(); |
| 3280 | for (int I = 1, E = MI.getNumOperands(); I != E; ++I) |
| 3281 | widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); |
| 3282 | |
| 3283 | // Avoid changing the result vector type if the source element type was |
| 3284 | // requested. |
| 3285 | if (TypeIdx == 1) { |
Matt Arsenault | a679f27 | 2020-07-19 12:29:48 -0400 | [diff] [blame] | 3286 | MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); |
Matt Arsenault | bd791b5 | 2019-07-08 13:48:06 +0000 | [diff] [blame] | 3287 | } else { |
| 3288 | widenScalarDst(MI, WideTy, 0); |
| 3289 | } |
| 3290 | |
| 3291 | Observer.changedInstr(MI); |
| 3292 | return Legalized; |
| 3293 | } |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 3294 | case TargetOpcode::G_SEXT_INREG: |
| 3295 | if (TypeIdx != 0) |
| 3296 | return UnableToLegalize; |
| 3297 | |
| 3298 | Observer.changingInstr(MI); |
| 3299 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 3300 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC); |
| 3301 | Observer.changedInstr(MI); |
| 3302 | return Legalized; |
Matt Arsenault | ef3e8312 | 2020-05-23 18:10:34 -0400 | [diff] [blame] | 3303 | case TargetOpcode::G_PTRMASK: { |
| 3304 | if (TypeIdx != 1) |
| 3305 | return UnableToLegalize; |
| 3306 | Observer.changingInstr(MI); |
| 3307 | widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); |
| 3308 | Observer.changedInstr(MI); |
| 3309 | return Legalized; |
| 3310 | } |
David Green | 295edaa | 2023-11-27 08:20:54 +0000 | [diff] [blame] | 3311 | case TargetOpcode::G_VECREDUCE_FADD: |
David Green | 5b5614c | 2024-01-03 07:49:20 +0000 | [diff] [blame] | 3312 | case TargetOpcode::G_VECREDUCE_FMUL: |
David Green | d199478a | 2023-08-14 09:19:47 +0100 | [diff] [blame] | 3313 | case TargetOpcode::G_VECREDUCE_FMIN: |
| 3314 | case TargetOpcode::G_VECREDUCE_FMAX: |
David Green | a3f2751 | 2023-08-14 10:03:25 +0100 | [diff] [blame] | 3315 | case TargetOpcode::G_VECREDUCE_FMINIMUM: |
Nikita Popov | f2f1845 | 2024-06-21 08:33:40 +0200 | [diff] [blame] | 3316 | case TargetOpcode::G_VECREDUCE_FMAXIMUM: { |
David Green | d199478a | 2023-08-14 09:19:47 +0100 | [diff] [blame] | 3317 | if (TypeIdx != 0) |
| 3318 | return UnableToLegalize; |
| 3319 | Observer.changingInstr(MI); |
| 3320 | Register VecReg = MI.getOperand(1).getReg(); |
| 3321 | LLT VecTy = MRI.getType(VecReg); |
| 3322 | LLT WideVecTy = VecTy.isVector() |
| 3323 | ? LLT::vector(VecTy.getElementCount(), WideTy) |
| 3324 | : WideTy; |
| 3325 | widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_FPEXT); |
| 3326 | widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); |
| 3327 | Observer.changedInstr(MI); |
| 3328 | return Legalized; |
Tim Northover | 3233581 | 2016-08-04 18:35:11 +0000 | [diff] [blame] | 3329 | } |
Michael Maitland | 54a9f0e | 2024-03-26 20:17:22 -0400 | [diff] [blame] | 3330 | case TargetOpcode::G_VSCALE: { |
| 3331 | MachineOperand &SrcMO = MI.getOperand(1); |
| 3332 | LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
| 3333 | const APInt &SrcVal = SrcMO.getCImm()->getValue(); |
| 3334 | // The CImm is always a signed value |
| 3335 | const APInt Val = SrcVal.sext(WideTy.getSizeInBits()); |
| 3336 | Observer.changingInstr(MI); |
| 3337 | SrcMO.setCImm(ConstantInt::get(Ctx, Val)); |
| 3338 | widenScalarDst(MI, WideTy); |
| 3339 | Observer.changedInstr(MI); |
| 3340 | return Legalized; |
| 3341 | } |
Michael Maitland | 8aa3a77 | 2024-03-07 13:40:30 -0800 | [diff] [blame] | 3342 | case TargetOpcode::G_SPLAT_VECTOR: { |
| 3343 | if (TypeIdx != 1) |
| 3344 | return UnableToLegalize; |
| 3345 | |
| 3346 | Observer.changingInstr(MI); |
| 3347 | widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); |
| 3348 | Observer.changedInstr(MI); |
| 3349 | return Legalized; |
| 3350 | } |
Michael Maitland | 6bac414 | 2024-10-21 08:49:13 -0400 | [diff] [blame] | 3351 | case TargetOpcode::G_INSERT_SUBVECTOR: { |
| 3352 | if (TypeIdx != 0) |
| 3353 | return UnableToLegalize; |
| 3354 | |
| 3355 | GInsertSubvector &IS = cast<GInsertSubvector>(MI); |
| 3356 | Register BigVec = IS.getBigVec(); |
| 3357 | Register SubVec = IS.getSubVec(); |
| 3358 | |
| 3359 | LLT SubVecTy = MRI.getType(SubVec); |
| 3360 | LLT SubVecWideTy = SubVecTy.changeElementType(WideTy.getElementType()); |
| 3361 | |
| 3362 | // Widen the G_INSERT_SUBVECTOR |
| 3363 | auto BigZExt = MIRBuilder.buildZExt(WideTy, BigVec); |
| 3364 | auto SubZExt = MIRBuilder.buildZExt(SubVecWideTy, SubVec); |
| 3365 | auto WideInsert = MIRBuilder.buildInsertSubvector(WideTy, BigZExt, SubZExt, |
| 3366 | IS.getIndexImm()); |
| 3367 | |
| 3368 | // Truncate back down |
| 3369 | auto SplatZero = MIRBuilder.buildSplatVector( |
| 3370 | WideTy, MIRBuilder.buildConstant(WideTy.getElementType(), 0)); |
| 3371 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, IS.getReg(0), WideInsert, |
| 3372 | SplatZero); |
| 3373 | |
| 3374 | MI.eraseFromParent(); |
| 3375 | |
| 3376 | return Legalized; |
| 3377 | } |
Michael Maitland | 54a9f0e | 2024-03-26 20:17:22 -0400 | [diff] [blame] | 3378 | } |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 3379 | } |
| 3380 | |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3381 | static void getUnmergePieces(SmallVectorImpl<Register> &Pieces, |
| 3382 | MachineIRBuilder &B, Register Src, LLT Ty) { |
| 3383 | auto Unmerge = B.buildUnmerge(Ty, Src); |
| 3384 | for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I) |
| 3385 | Pieces.push_back(Unmerge.getReg(I)); |
| 3386 | } |
| 3387 | |
Mikhail Gudim | 35cfaec | 2024-02-16 18:51:44 -0500 | [diff] [blame] | 3388 | static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, |
| 3389 | MachineIRBuilder &MIRBuilder) { |
| 3390 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3391 | MachineFunction &MF = MIRBuilder.getMF(); |
| 3392 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3393 | unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace(); |
| 3394 | LLT AddrPtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); |
Mikhail Gudim | 35cfaec | 2024-02-16 18:51:44 -0500 | [diff] [blame] | 3395 | LLT DstLLT = MRI.getType(DstReg); |
| 3396 | |
| 3397 | Align Alignment(DL.getABITypeAlign(ConstVal->getType())); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3398 | |
| 3399 | auto Addr = MIRBuilder.buildConstantPool( |
Mikhail Gudim | 35cfaec | 2024-02-16 18:51:44 -0500 | [diff] [blame] | 3400 | AddrPtrTy, |
| 3401 | MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment)); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3402 | |
Mikhail Gudim | 35cfaec | 2024-02-16 18:51:44 -0500 | [diff] [blame] | 3403 | MachineMemOperand *MMO = |
| 3404 | MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF), |
| 3405 | MachineMemOperand::MOLoad, DstLLT, Alignment); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3406 | |
Mikhail Gudim | 35cfaec | 2024-02-16 18:51:44 -0500 | [diff] [blame] | 3407 | MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, DstReg, Addr, *MMO); |
| 3408 | } |
| 3409 | |
| 3410 | LegalizerHelper::LegalizeResult |
| 3411 | LegalizerHelper::lowerConstant(MachineInstr &MI) { |
| 3412 | const MachineOperand &ConstOperand = MI.getOperand(1); |
| 3413 | const Constant *ConstantVal = ConstOperand.getCImm(); |
| 3414 | |
| 3415 | emitLoadFromConstantPool(MI.getOperand(0).getReg(), ConstantVal, MIRBuilder); |
| 3416 | MI.eraseFromParent(); |
| 3417 | |
| 3418 | return Legalized; |
| 3419 | } |
| 3420 | |
| 3421 | LegalizerHelper::LegalizeResult |
| 3422 | LegalizerHelper::lowerFConstant(MachineInstr &MI) { |
| 3423 | const MachineOperand &ConstOperand = MI.getOperand(1); |
| 3424 | const Constant *ConstantVal = ConstOperand.getFPImm(); |
| 3425 | |
| 3426 | emitLoadFromConstantPool(MI.getOperand(0).getReg(), ConstantVal, MIRBuilder); |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 3427 | MI.eraseFromParent(); |
| 3428 | |
| 3429 | return Legalized; |
| 3430 | } |
| 3431 | |
| 3432 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3433 | LegalizerHelper::lowerBitcast(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 3434 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 33e9086 | 2020-06-09 11:19:12 -0400 | [diff] [blame] | 3435 | if (SrcTy.isVector()) { |
| 3436 | LLT SrcEltTy = SrcTy.getElementType(); |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3437 | SmallVector<Register, 8> SrcRegs; |
Matt Arsenault | 33e9086 | 2020-06-09 11:19:12 -0400 | [diff] [blame] | 3438 | |
| 3439 | if (DstTy.isVector()) { |
| 3440 | int NumDstElt = DstTy.getNumElements(); |
| 3441 | int NumSrcElt = SrcTy.getNumElements(); |
| 3442 | |
| 3443 | LLT DstEltTy = DstTy.getElementType(); |
| 3444 | LLT DstCastTy = DstEltTy; // Intermediate bitcast result type |
| 3445 | LLT SrcPartTy = SrcEltTy; // Original unmerge result type. |
| 3446 | |
| 3447 | // If there's an element size mismatch, insert intermediate casts to match |
| 3448 | // the result element type. |
| 3449 | if (NumSrcElt < NumDstElt) { // Source element type is larger. |
| 3450 | // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>) |
| 3451 | // |
| 3452 | // => |
| 3453 | // |
| 3454 | // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0 |
| 3455 | // %3:_(<2 x s8>) = G_BITCAST %2 |
| 3456 | // %4:_(<2 x s8>) = G_BITCAST %3 |
| 3457 | // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4 |
Sander de Smalen | d5e14ba | 2021-06-24 09:58:21 +0100 | [diff] [blame] | 3458 | DstCastTy = LLT::fixed_vector(NumDstElt / NumSrcElt, DstEltTy); |
Matt Arsenault | 33e9086 | 2020-06-09 11:19:12 -0400 | [diff] [blame] | 3459 | SrcPartTy = SrcEltTy; |
| 3460 | } else if (NumSrcElt > NumDstElt) { // Source element type is smaller. |
| 3461 | // |
| 3462 | // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>) |
| 3463 | // |
| 3464 | // => |
| 3465 | // |
| 3466 | // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0 |
| 3467 | // %3:_(s16) = G_BITCAST %2 |
| 3468 | // %4:_(s16) = G_BITCAST %3 |
| 3469 | // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4 |
Sander de Smalen | d5e14ba | 2021-06-24 09:58:21 +0100 | [diff] [blame] | 3470 | SrcPartTy = LLT::fixed_vector(NumSrcElt / NumDstElt, SrcEltTy); |
Matt Arsenault | 33e9086 | 2020-06-09 11:19:12 -0400 | [diff] [blame] | 3471 | DstCastTy = DstEltTy; |
| 3472 | } |
| 3473 | |
| 3474 | getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy); |
| 3475 | for (Register &SrcReg : SrcRegs) |
| 3476 | SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0); |
| 3477 | } else |
| 3478 | getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy); |
| 3479 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 3480 | MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs); |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3481 | MI.eraseFromParent(); |
| 3482 | return Legalized; |
| 3483 | } |
| 3484 | |
Matt Arsenault | 33e9086 | 2020-06-09 11:19:12 -0400 | [diff] [blame] | 3485 | if (DstTy.isVector()) { |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3486 | SmallVector<Register, 8> SrcRegs; |
| 3487 | getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType()); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 3488 | MIRBuilder.buildMergeLikeInstr(Dst, SrcRegs); |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 3489 | MI.eraseFromParent(); |
| 3490 | return Legalized; |
| 3491 | } |
| 3492 | |
| 3493 | return UnableToLegalize; |
| 3494 | } |
| 3495 | |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 3496 | /// Figure out the bit offset into a register when coercing a vector index for |
| 3497 | /// the wide element type. This is only for the case when promoting vector to |
| 3498 | /// one with larger elements. |
| 3499 | // |
| 3500 | /// |
| 3501 | /// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) |
| 3502 | /// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize) |
| 3503 | static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B, |
| 3504 | Register Idx, |
| 3505 | unsigned NewEltSize, |
| 3506 | unsigned OldEltSize) { |
| 3507 | const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize); |
| 3508 | LLT IdxTy = B.getMRI()->getType(Idx); |
| 3509 | |
| 3510 | // Now figure out the amount we need to shift to get the target bits. |
| 3511 | auto OffsetMask = B.buildConstant( |
Chris Lattner | 735f467 | 2021-09-08 22:13:13 -0700 | [diff] [blame] | 3512 | IdxTy, ~(APInt::getAllOnes(IdxTy.getSizeInBits()) << Log2EltRatio)); |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 3513 | auto OffsetIdx = B.buildAnd(IdxTy, Idx, OffsetMask); |
| 3514 | return B.buildShl(IdxTy, OffsetIdx, |
| 3515 | B.buildConstant(IdxTy, Log2_32(OldEltSize))).getReg(0); |
| 3516 | } |
| 3517 | |
Matt Arsenault | 212570a | 2020-06-15 11:54:49 -0400 | [diff] [blame] | 3518 | /// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this |
| 3519 | /// is casting to a vector with a smaller element size, perform multiple element |
| 3520 | /// extracts and merge the results. If this is coercing to a vector with larger |
| 3521 | /// elements, index the bitcasted vector and extract the target element with bit |
| 3522 | /// operations. This is intended to force the indexing in the native register |
| 3523 | /// size for architectures that can dynamically index the register file. |
| 3524 | LegalizerHelper::LegalizeResult |
| 3525 | LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, |
| 3526 | LLT CastTy) { |
| 3527 | if (TypeIdx != 1) |
| 3528 | return UnableToLegalize; |
| 3529 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 3530 | auto [Dst, DstTy, SrcVec, SrcVecTy, Idx, IdxTy] = MI.getFirst3RegLLTs(); |
Matt Arsenault | 212570a | 2020-06-15 11:54:49 -0400 | [diff] [blame] | 3531 | |
| 3532 | LLT SrcEltTy = SrcVecTy.getElementType(); |
| 3533 | unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1; |
| 3534 | unsigned OldNumElts = SrcVecTy.getNumElements(); |
| 3535 | |
| 3536 | LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy; |
| 3537 | Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); |
| 3538 | |
| 3539 | const unsigned NewEltSize = NewEltTy.getSizeInBits(); |
| 3540 | const unsigned OldEltSize = SrcEltTy.getSizeInBits(); |
| 3541 | if (NewNumElts > OldNumElts) { |
| 3542 | // Decreasing the vector element size |
| 3543 | // |
| 3544 | // e.g. i64 = extract_vector_elt x:v2i64, y:i32 |
| 3545 | // => |
| 3546 | // v4i32:castx = bitcast x:v2i64 |
| 3547 | // |
| 3548 | // i64 = bitcast |
| 3549 | // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), |
| 3550 | // (i32 (extract_vector_elt castx, (2 * y + 1))) |
| 3551 | // |
| 3552 | if (NewNumElts % OldNumElts != 0) |
| 3553 | return UnableToLegalize; |
| 3554 | |
| 3555 | // Type of the intermediate result vector. |
| 3556 | const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts; |
Sander de Smalen | 968980e | 2021-06-25 08:25:41 +0100 | [diff] [blame] | 3557 | LLT MidTy = |
| 3558 | LLT::scalarOrVector(ElementCount::getFixed(NewEltsPerOldElt), NewEltTy); |
Matt Arsenault | 212570a | 2020-06-15 11:54:49 -0400 | [diff] [blame] | 3559 | |
| 3560 | auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt); |
| 3561 | |
| 3562 | SmallVector<Register, 8> NewOps(NewEltsPerOldElt); |
| 3563 | auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK); |
| 3564 | |
| 3565 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
| 3566 | auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I); |
| 3567 | auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset); |
| 3568 | auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx); |
| 3569 | NewOps[I] = Elt.getReg(0); |
| 3570 | } |
| 3571 | |
| 3572 | auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps); |
| 3573 | MIRBuilder.buildBitcast(Dst, NewVec); |
| 3574 | MI.eraseFromParent(); |
| 3575 | return Legalized; |
| 3576 | } |
| 3577 | |
| 3578 | if (NewNumElts < OldNumElts) { |
| 3579 | if (NewEltSize % OldEltSize != 0) |
| 3580 | return UnableToLegalize; |
| 3581 | |
| 3582 | // This only depends on powers of 2 because we use bit tricks to figure out |
| 3583 | // the bit offset we need to shift to get the target element. A general |
| 3584 | // expansion could emit division/multiply. |
| 3585 | if (!isPowerOf2_32(NewEltSize / OldEltSize)) |
| 3586 | return UnableToLegalize; |
| 3587 | |
| 3588 | // Increasing the vector element size. |
| 3589 | // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx |
| 3590 | // |
| 3591 | // => |
| 3592 | // |
| 3593 | // %cast = G_BITCAST %vec |
| 3594 | // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize) |
| 3595 | // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx |
| 3596 | // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) |
| 3597 | // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize) |
| 3598 | // %elt_bits = G_LSHR %wide_elt, %offset_bits |
| 3599 | // %elt = G_TRUNC %elt_bits |
| 3600 | |
| 3601 | const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize); |
| 3602 | auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); |
| 3603 | |
| 3604 | // Divide to get the index in the wider element type. |
| 3605 | auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); |
| 3606 | |
| 3607 | Register WideElt = CastVec; |
| 3608 | if (CastTy.isVector()) { |
| 3609 | WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, |
| 3610 | ScaledIdx).getReg(0); |
| 3611 | } |
| 3612 | |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 3613 | // Compute the bit offset into the register of the target element. |
| 3614 | Register OffsetBits = getBitcastWiderVectorElementOffset( |
| 3615 | MIRBuilder, Idx, NewEltSize, OldEltSize); |
Matt Arsenault | 212570a | 2020-06-15 11:54:49 -0400 | [diff] [blame] | 3616 | |
| 3617 | // Shift the wide element to get the target element. |
| 3618 | auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits); |
| 3619 | MIRBuilder.buildTrunc(Dst, ExtractedBits); |
| 3620 | MI.eraseFromParent(); |
| 3621 | return Legalized; |
| 3622 | } |
| 3623 | |
| 3624 | return UnableToLegalize; |
| 3625 | } |
| 3626 | |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 3627 | /// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p |
| 3628 | /// TargetReg, while preserving other bits in \p TargetReg. |
| 3629 | /// |
| 3630 | /// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset) |
| 3631 | static Register buildBitFieldInsert(MachineIRBuilder &B, |
| 3632 | Register TargetReg, Register InsertReg, |
| 3633 | Register OffsetBits) { |
| 3634 | LLT TargetTy = B.getMRI()->getType(TargetReg); |
| 3635 | LLT InsertTy = B.getMRI()->getType(InsertReg); |
| 3636 | auto ZextVal = B.buildZExt(TargetTy, InsertReg); |
| 3637 | auto ShiftedInsertVal = B.buildShl(TargetTy, ZextVal, OffsetBits); |
| 3638 | |
| 3639 | // Produce a bitmask of the value to insert |
| 3640 | auto EltMask = B.buildConstant( |
| 3641 | TargetTy, APInt::getLowBitsSet(TargetTy.getSizeInBits(), |
| 3642 | InsertTy.getSizeInBits())); |
| 3643 | // Shift it into position |
| 3644 | auto ShiftedMask = B.buildShl(TargetTy, EltMask, OffsetBits); |
| 3645 | auto InvShiftedMask = B.buildNot(TargetTy, ShiftedMask); |
| 3646 | |
| 3647 | // Clear out the bits in the wide element |
| 3648 | auto MaskedOldElt = B.buildAnd(TargetTy, TargetReg, InvShiftedMask); |
| 3649 | |
| 3650 | // The value to insert has all zeros already, so stick it into the masked |
| 3651 | // wide element. |
| 3652 | return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0); |
| 3653 | } |
| 3654 | |
| 3655 | /// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this |
| 3656 | /// is increasing the element size, perform the indexing in the target element |
| 3657 | /// type, and use bit operations to insert at the element position. This is |
| 3658 | /// intended for architectures that can dynamically index the register file and |
| 3659 | /// want to force indexing in the native register size. |
| 3660 | LegalizerHelper::LegalizeResult |
| 3661 | LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, |
| 3662 | LLT CastTy) { |
| 3663 | if (TypeIdx != 0) |
| 3664 | return UnableToLegalize; |
| 3665 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 3666 | auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy, Idx, IdxTy] = |
| 3667 | MI.getFirst4RegLLTs(); |
| 3668 | LLT VecTy = DstTy; |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 3669 | |
| 3670 | LLT VecEltTy = VecTy.getElementType(); |
| 3671 | LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy; |
| 3672 | const unsigned NewEltSize = NewEltTy.getSizeInBits(); |
| 3673 | const unsigned OldEltSize = VecEltTy.getSizeInBits(); |
| 3674 | |
| 3675 | unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1; |
| 3676 | unsigned OldNumElts = VecTy.getNumElements(); |
| 3677 | |
| 3678 | Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0); |
| 3679 | if (NewNumElts < OldNumElts) { |
| 3680 | if (NewEltSize % OldEltSize != 0) |
| 3681 | return UnableToLegalize; |
| 3682 | |
| 3683 | // This only depends on powers of 2 because we use bit tricks to figure out |
| 3684 | // the bit offset we need to shift to get the target element. A general |
| 3685 | // expansion could emit division/multiply. |
| 3686 | if (!isPowerOf2_32(NewEltSize / OldEltSize)) |
| 3687 | return UnableToLegalize; |
| 3688 | |
| 3689 | const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize); |
| 3690 | auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio); |
| 3691 | |
| 3692 | // Divide to get the index in the wider element type. |
| 3693 | auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio); |
| 3694 | |
| 3695 | Register ExtractedElt = CastVec; |
| 3696 | if (CastTy.isVector()) { |
| 3697 | ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, |
| 3698 | ScaledIdx).getReg(0); |
| 3699 | } |
| 3700 | |
| 3701 | // Compute the bit offset into the register of the target element. |
| 3702 | Register OffsetBits = getBitcastWiderVectorElementOffset( |
| 3703 | MIRBuilder, Idx, NewEltSize, OldEltSize); |
| 3704 | |
| 3705 | Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt, |
| 3706 | Val, OffsetBits); |
| 3707 | if (CastTy.isVector()) { |
| 3708 | InsertedElt = MIRBuilder.buildInsertVectorElement( |
| 3709 | CastTy, CastVec, InsertedElt, ScaledIdx).getReg(0); |
| 3710 | } |
| 3711 | |
| 3712 | MIRBuilder.buildBitcast(Dst, InsertedElt); |
| 3713 | MI.eraseFromParent(); |
| 3714 | return Legalized; |
| 3715 | } |
| 3716 | |
| 3717 | return UnableToLegalize; |
| 3718 | } |
| 3719 | |
chuongg3 | 0d5db4e | 2024-07-15 12:00:47 +0100 | [diff] [blame] | 3720 | // This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly |
| 3721 | // those that have smaller than legal operands. |
| 3722 | // |
| 3723 | // <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8> |
| 3724 | // |
| 3725 | // ===> |
| 3726 | // |
| 3727 | // s32 = G_BITCAST <4 x s8> |
| 3728 | // s32 = G_BITCAST <4 x s8> |
| 3729 | // s32 = G_BITCAST <4 x s8> |
| 3730 | // s32 = G_BITCAST <4 x s8> |
| 3731 | // <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32 |
| 3732 | // <16 x s8> = G_BITCAST <4 x s32> |
| 3733 | LegalizerHelper::LegalizeResult |
| 3734 | LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx, |
| 3735 | LLT CastTy) { |
| 3736 | // Convert it to CONCAT instruction |
| 3737 | auto ConcatMI = dyn_cast<GConcatVectors>(&MI); |
| 3738 | if (!ConcatMI) { |
| 3739 | return UnableToLegalize; |
| 3740 | } |
| 3741 | |
| 3742 | // Check if bitcast is Legal |
| 3743 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
| 3744 | LLT SrcScalTy = LLT::scalar(SrcTy.getSizeInBits()); |
| 3745 | |
| 3746 | // Check if the build vector is Legal |
| 3747 | if (!LI.isLegal({TargetOpcode::G_BUILD_VECTOR, {CastTy, SrcScalTy}})) { |
| 3748 | return UnableToLegalize; |
| 3749 | } |
| 3750 | |
| 3751 | // Bitcast the sources |
| 3752 | SmallVector<Register> BitcastRegs; |
| 3753 | for (unsigned i = 0; i < ConcatMI->getNumSources(); i++) { |
| 3754 | BitcastRegs.push_back( |
| 3755 | MIRBuilder.buildBitcast(SrcScalTy, ConcatMI->getSourceReg(i)) |
| 3756 | .getReg(0)); |
| 3757 | } |
| 3758 | |
| 3759 | // Build the scalar values into a vector |
| 3760 | Register BuildReg = |
| 3761 | MIRBuilder.buildBuildVector(CastTy, BitcastRegs).getReg(0); |
| 3762 | MIRBuilder.buildBitcast(DstReg, BuildReg); |
| 3763 | |
| 3764 | MI.eraseFromParent(); |
| 3765 | return Legalized; |
| 3766 | } |
| 3767 | |
David Green | d3ce069 | 2024-11-23 17:00:51 +0000 | [diff] [blame] | 3768 | // This bitcasts a shuffle vector to a different type currently of the same |
| 3769 | // element size. Mostly used to legalize ptr vectors, where ptrtoint/inttoptr |
| 3770 | // will be used instead. |
| 3771 | // |
| 3772 | // <16 x p0> = G_CONCAT_VECTORS <4 x p0>, <4 x p0>, mask |
| 3773 | // ===> |
| 3774 | // <4 x s64> = G_PTRTOINT <4 x p0> |
| 3775 | // <4 x s64> = G_PTRTOINT <4 x p0> |
| 3776 | // <16 x s64> = G_CONCAT_VECTORS <4 x s64>, <4 x s64>, mask |
| 3777 | // <16 x p0> = G_INTTOPTR <16 x s64> |
| 3778 | LegalizerHelper::LegalizeResult |
| 3779 | LegalizerHelper::bitcastShuffleVector(MachineInstr &MI, unsigned TypeIdx, |
| 3780 | LLT CastTy) { |
| 3781 | auto ShuffleMI = cast<GShuffleVector>(&MI); |
| 3782 | LLT DstTy = MRI.getType(ShuffleMI->getReg(0)); |
| 3783 | LLT SrcTy = MRI.getType(ShuffleMI->getReg(1)); |
| 3784 | |
| 3785 | // We currently only handle vectors of the same size. |
| 3786 | if (TypeIdx != 0 || |
| 3787 | CastTy.getScalarSizeInBits() != DstTy.getScalarSizeInBits() || |
| 3788 | CastTy.getElementCount() != DstTy.getElementCount()) |
| 3789 | return UnableToLegalize; |
| 3790 | |
| 3791 | LLT NewSrcTy = SrcTy.changeElementType(CastTy.getScalarType()); |
| 3792 | |
| 3793 | auto Inp1 = MIRBuilder.buildCast(NewSrcTy, ShuffleMI->getReg(1)); |
| 3794 | auto Inp2 = MIRBuilder.buildCast(NewSrcTy, ShuffleMI->getReg(2)); |
| 3795 | auto Shuf = |
| 3796 | MIRBuilder.buildShuffleVector(CastTy, Inp1, Inp2, ShuffleMI->getMask()); |
| 3797 | MIRBuilder.buildCast(ShuffleMI->getReg(0), Shuf); |
| 3798 | |
| 3799 | MI.eraseFromParent(); |
| 3800 | return Legalized; |
| 3801 | } |
| 3802 | |
Michael Maitland | f957d08 | 2024-10-01 14:08:49 -0400 | [diff] [blame] | 3803 | /// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy. |
| 3804 | /// |
| 3805 | /// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N |
| 3806 | /// |
| 3807 | /// ===> |
| 3808 | /// |
| 3809 | /// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1> |
| 3810 | /// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8 |
| 3811 | /// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8> |
| 3812 | LegalizerHelper::LegalizeResult |
| 3813 | LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx, |
| 3814 | LLT CastTy) { |
| 3815 | auto ES = cast<GExtractSubvector>(&MI); |
| 3816 | |
| 3817 | if (!CastTy.isVector()) |
| 3818 | return UnableToLegalize; |
| 3819 | |
| 3820 | if (TypeIdx != 0) |
| 3821 | return UnableToLegalize; |
| 3822 | |
| 3823 | Register Dst = ES->getReg(0); |
| 3824 | Register Src = ES->getSrcVec(); |
| 3825 | uint64_t Idx = ES->getIndexImm(); |
| 3826 | |
| 3827 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); |
| 3828 | |
| 3829 | LLT DstTy = MRI.getType(Dst); |
| 3830 | LLT SrcTy = MRI.getType(Src); |
| 3831 | ElementCount DstTyEC = DstTy.getElementCount(); |
| 3832 | ElementCount SrcTyEC = SrcTy.getElementCount(); |
| 3833 | auto DstTyMinElts = DstTyEC.getKnownMinValue(); |
| 3834 | auto SrcTyMinElts = SrcTyEC.getKnownMinValue(); |
| 3835 | |
| 3836 | if (DstTy == CastTy) |
| 3837 | return Legalized; |
| 3838 | |
| 3839 | if (DstTy.getSizeInBits() != CastTy.getSizeInBits()) |
| 3840 | return UnableToLegalize; |
| 3841 | |
| 3842 | unsigned CastEltSize = CastTy.getElementType().getSizeInBits(); |
| 3843 | unsigned DstEltSize = DstTy.getElementType().getSizeInBits(); |
| 3844 | if (CastEltSize < DstEltSize) |
| 3845 | return UnableToLegalize; |
| 3846 | |
| 3847 | auto AdjustAmt = CastEltSize / DstEltSize; |
| 3848 | if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 || |
| 3849 | SrcTyMinElts % AdjustAmt != 0) |
| 3850 | return UnableToLegalize; |
| 3851 | |
| 3852 | Idx /= AdjustAmt; |
| 3853 | SrcTy = LLT::vector(SrcTyEC.divideCoefficientBy(AdjustAmt), AdjustAmt); |
| 3854 | auto CastVec = MIRBuilder.buildBitcast(SrcTy, Src); |
| 3855 | auto PromotedES = MIRBuilder.buildExtractSubvector(CastTy, CastVec, Idx); |
| 3856 | MIRBuilder.buildBitcast(Dst, PromotedES); |
| 3857 | |
| 3858 | ES->eraseFromParent(); |
| 3859 | return Legalized; |
| 3860 | } |
| 3861 | |
Michael Maitland | 6bac414 | 2024-10-21 08:49:13 -0400 | [diff] [blame] | 3862 | /// This attempts to bitcast G_INSERT_SUBVECTOR to CastTy. |
| 3863 | /// |
| 3864 | /// <vscale x 16 x i1> = G_INSERT_SUBVECTOR <vscale x 16 x i1>, |
| 3865 | /// <vscale x 8 x i1>, |
| 3866 | /// N |
| 3867 | /// |
| 3868 | /// ===> |
| 3869 | /// |
| 3870 | /// <vscale x 2 x i8> = G_BITCAST <vscale x 16 x i1> |
| 3871 | /// <vscale x 1 x i8> = G_BITCAST <vscale x 8 x i1> |
| 3872 | /// <vscale x 2 x i8> = G_INSERT_SUBVECTOR <vscale x 2 x i8>, |
| 3873 | /// <vscale x 1 x i8>, N / 8 |
| 3874 | /// <vscale x 16 x i1> = G_BITCAST <vscale x 2 x i8> |
| 3875 | LegalizerHelper::LegalizeResult |
| 3876 | LegalizerHelper::bitcastInsertSubvector(MachineInstr &MI, unsigned TypeIdx, |
| 3877 | LLT CastTy) { |
| 3878 | auto ES = cast<GInsertSubvector>(&MI); |
| 3879 | |
| 3880 | if (!CastTy.isVector()) |
| 3881 | return UnableToLegalize; |
| 3882 | |
| 3883 | if (TypeIdx != 0) |
| 3884 | return UnableToLegalize; |
| 3885 | |
| 3886 | Register Dst = ES->getReg(0); |
| 3887 | Register BigVec = ES->getBigVec(); |
| 3888 | Register SubVec = ES->getSubVec(); |
| 3889 | uint64_t Idx = ES->getIndexImm(); |
| 3890 | |
| 3891 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); |
| 3892 | |
| 3893 | LLT DstTy = MRI.getType(Dst); |
| 3894 | LLT BigVecTy = MRI.getType(BigVec); |
| 3895 | LLT SubVecTy = MRI.getType(SubVec); |
| 3896 | |
| 3897 | if (DstTy == CastTy) |
| 3898 | return Legalized; |
| 3899 | |
| 3900 | if (DstTy.getSizeInBits() != CastTy.getSizeInBits()) |
| 3901 | return UnableToLegalize; |
| 3902 | |
| 3903 | ElementCount DstTyEC = DstTy.getElementCount(); |
| 3904 | ElementCount BigVecTyEC = BigVecTy.getElementCount(); |
| 3905 | ElementCount SubVecTyEC = SubVecTy.getElementCount(); |
| 3906 | auto DstTyMinElts = DstTyEC.getKnownMinValue(); |
| 3907 | auto BigVecTyMinElts = BigVecTyEC.getKnownMinValue(); |
| 3908 | auto SubVecTyMinElts = SubVecTyEC.getKnownMinValue(); |
| 3909 | |
| 3910 | unsigned CastEltSize = CastTy.getElementType().getSizeInBits(); |
| 3911 | unsigned DstEltSize = DstTy.getElementType().getSizeInBits(); |
| 3912 | if (CastEltSize < DstEltSize) |
| 3913 | return UnableToLegalize; |
| 3914 | |
| 3915 | auto AdjustAmt = CastEltSize / DstEltSize; |
| 3916 | if (Idx % AdjustAmt != 0 || DstTyMinElts % AdjustAmt != 0 || |
| 3917 | BigVecTyMinElts % AdjustAmt != 0 || SubVecTyMinElts % AdjustAmt != 0) |
| 3918 | return UnableToLegalize; |
| 3919 | |
| 3920 | Idx /= AdjustAmt; |
| 3921 | BigVecTy = LLT::vector(BigVecTyEC.divideCoefficientBy(AdjustAmt), AdjustAmt); |
| 3922 | SubVecTy = LLT::vector(SubVecTyEC.divideCoefficientBy(AdjustAmt), AdjustAmt); |
| 3923 | auto CastBigVec = MIRBuilder.buildBitcast(BigVecTy, BigVec); |
| 3924 | auto CastSubVec = MIRBuilder.buildBitcast(SubVecTy, SubVec); |
| 3925 | auto PromotedIS = |
| 3926 | MIRBuilder.buildInsertSubvector(CastTy, CastBigVec, CastSubVec, Idx); |
| 3927 | MIRBuilder.buildBitcast(Dst, PromotedIS); |
| 3928 | |
| 3929 | ES->eraseFromParent(); |
| 3930 | return Legalized; |
| 3931 | } |
| 3932 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 3933 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) { |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 3934 | // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 3935 | Register DstReg = LoadMI.getDstReg(); |
| 3936 | Register PtrReg = LoadMI.getPointerReg(); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 3937 | LLT DstTy = MRI.getType(DstReg); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 3938 | MachineMemOperand &MMO = LoadMI.getMMO(); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3939 | LLT MemTy = MMO.getMemoryType(); |
| 3940 | MachineFunction &MF = MIRBuilder.getMF(); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3941 | |
| 3942 | unsigned MemSizeInBits = MemTy.getSizeInBits(); |
| 3943 | unsigned MemStoreSizeInBits = 8 * MemTy.getSizeInBytes(); |
| 3944 | |
| 3945 | if (MemSizeInBits != MemStoreSizeInBits) { |
Matt Arsenault | e46badd | 2021-07-26 14:10:26 -0400 | [diff] [blame] | 3946 | if (MemTy.isVector()) |
| 3947 | return UnableToLegalize; |
| 3948 | |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3949 | // Promote to a byte-sized load if not loading an integral number of |
| 3950 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. |
| 3951 | LLT WideMemTy = LLT::scalar(MemStoreSizeInBits); |
| 3952 | MachineMemOperand *NewMMO = |
| 3953 | MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideMemTy); |
| 3954 | |
| 3955 | Register LoadReg = DstReg; |
| 3956 | LLT LoadTy = DstTy; |
| 3957 | |
| 3958 | // If this wasn't already an extending load, we need to widen the result |
| 3959 | // register to avoid creating a load with a narrower result than the source. |
| 3960 | if (MemStoreSizeInBits > DstTy.getSizeInBits()) { |
| 3961 | LoadTy = WideMemTy; |
| 3962 | LoadReg = MRI.createGenericVirtualRegister(WideMemTy); |
| 3963 | } |
| 3964 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 3965 | if (isa<GSExtLoad>(LoadMI)) { |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3966 | auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); |
| 3967 | MIRBuilder.buildSExtInReg(LoadReg, NewLoad, MemSizeInBits); |
Matt Arsenault | d1f97a3 | 2022-04-10 19:50:47 -0400 | [diff] [blame] | 3968 | } else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) { |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3969 | auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); |
| 3970 | // The extra bits are guaranteed to be zero, since we stored them that |
| 3971 | // way. A zext load from Wide thus automatically gives zext from MemVT. |
| 3972 | MIRBuilder.buildAssertZExt(LoadReg, NewLoad, MemSizeInBits); |
| 3973 | } else { |
| 3974 | MIRBuilder.buildLoad(LoadReg, PtrReg, *NewMMO); |
| 3975 | } |
| 3976 | |
| 3977 | if (DstTy != LoadTy) |
| 3978 | MIRBuilder.buildTrunc(DstReg, LoadReg); |
| 3979 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 3980 | LoadMI.eraseFromParent(); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 3981 | return Legalized; |
| 3982 | } |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 3983 | |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 3984 | // Big endian lowering not implemented. |
| 3985 | if (MIRBuilder.getDataLayout().isBigEndian()) |
Matt Arsenault | 9d7299b | 2021-06-09 21:22:00 -0400 | [diff] [blame] | 3986 | return UnableToLegalize; |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 3987 | |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 3988 | // This load needs splitting into power of 2 sized loads. |
| 3989 | // |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 3990 | // Our strategy here is to generate anyextending loads for the smaller |
| 3991 | // types up to next power-2 result type, and then combine the two larger |
| 3992 | // result values together, before truncating back down to the non-pow-2 |
| 3993 | // type. |
| 3994 | // E.g. v1 = i24 load => |
| 3995 | // v2 = i32 zextload (2 byte) |
| 3996 | // v3 = i32 load (1 byte) |
| 3997 | // v4 = i32 shl v3, 16 |
| 3998 | // v5 = i32 or v4, v2 |
| 3999 | // v1 = i24 trunc v5 |
| 4000 | // By doing this we generate the correct truncate which should get |
| 4001 | // combined away as an artifact with a matching extend. |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 4002 | |
| 4003 | uint64_t LargeSplitSize, SmallSplitSize; |
| 4004 | |
| 4005 | if (!isPowerOf2_32(MemSizeInBits)) { |
Matt Arsenault | e46badd | 2021-07-26 14:10:26 -0400 | [diff] [blame] | 4006 | // This load needs splitting into power of 2 sized loads. |
Kazu Hirata | f20b507 | 2023-01-28 09:06:31 -0800 | [diff] [blame] | 4007 | LargeSplitSize = llvm::bit_floor(MemSizeInBits); |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 4008 | SmallSplitSize = MemSizeInBits - LargeSplitSize; |
| 4009 | } else { |
Matt Arsenault | e46badd | 2021-07-26 14:10:26 -0400 | [diff] [blame] | 4010 | // This is already a power of 2, but we still need to split this in half. |
| 4011 | // |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 4012 | // Assume we're being asked to decompose an unaligned load. |
| 4013 | // TODO: If this requires multiple splits, handle them all at once. |
| 4014 | auto &Ctx = MF.getFunction().getContext(); |
| 4015 | if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO)) |
| 4016 | return UnableToLegalize; |
| 4017 | |
| 4018 | SmallSplitSize = LargeSplitSize = MemSizeInBits / 2; |
| 4019 | } |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4020 | |
Matt Arsenault | e46badd | 2021-07-26 14:10:26 -0400 | [diff] [blame] | 4021 | if (MemTy.isVector()) { |
| 4022 | // TODO: Handle vector extloads |
| 4023 | if (MemTy != DstTy) |
| 4024 | return UnableToLegalize; |
| 4025 | |
| 4026 | // TODO: We can do better than scalarizing the vector and at least split it |
| 4027 | // in half. |
| 4028 | return reduceLoadStoreWidth(LoadMI, 0, DstTy.getElementType()); |
| 4029 | } |
| 4030 | |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4031 | MachineMemOperand *LargeMMO = |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4032 | MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); |
| 4033 | MachineMemOperand *SmallMMO = |
| 4034 | MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4035 | |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4036 | LLT PtrTy = MRI.getType(PtrReg); |
| 4037 | unsigned AnyExtSize = PowerOf2Ceil(DstTy.getSizeInBits()); |
| 4038 | LLT AnyExtTy = LLT::scalar(AnyExtSize); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4039 | auto LargeLoad = MIRBuilder.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, AnyExtTy, |
| 4040 | PtrReg, *LargeMMO); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4041 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4042 | auto OffsetCst = MIRBuilder.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), |
| 4043 | LargeSplitSize / 8); |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4044 | Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4045 | auto SmallPtr = MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst); |
| 4046 | auto SmallLoad = MIRBuilder.buildLoadInstr(LoadMI.getOpcode(), AnyExtTy, |
| 4047 | SmallPtr, *SmallMMO); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4048 | |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4049 | auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize); |
| 4050 | auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4051 | |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4052 | if (AnyExtTy == DstTy) |
| 4053 | MIRBuilder.buildOr(DstReg, Shift, LargeLoad); |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 4054 | else if (AnyExtTy.getSizeInBits() != DstTy.getSizeInBits()) { |
Matt Arsenault | 9d7299b | 2021-06-09 21:22:00 -0400 | [diff] [blame] | 4055 | auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); |
| 4056 | MIRBuilder.buildTrunc(DstReg, {Or}); |
Matt Arsenault | f19226d | 2021-07-22 08:11:14 -0400 | [diff] [blame] | 4057 | } else { |
| 4058 | assert(DstTy.isPointer() && "expected pointer"); |
| 4059 | auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad); |
| 4060 | |
| 4061 | // FIXME: We currently consider this to be illegal for non-integral address |
| 4062 | // spaces, but we need still need a way to reinterpret the bits. |
| 4063 | MIRBuilder.buildIntToPtr(DstReg, Or); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4064 | } |
| 4065 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4066 | LoadMI.eraseFromParent(); |
Matt Arsenault | 47269da | 2021-06-10 09:28:20 -0400 | [diff] [blame] | 4067 | return Legalized; |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4068 | } |
| 4069 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4070 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) { |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4071 | // Lower a non-power of 2 store into multiple pow-2 stores. |
| 4072 | // E.g. split an i24 store into an i16 store + i8 store. |
| 4073 | // We do this by first extending the stored value to the next largest power |
| 4074 | // of 2 type, and then using truncating stores to store the components. |
| 4075 | // By doing this, likewise with G_LOAD, generate an extend that can be |
| 4076 | // artifact-combined away instead of leaving behind extracts. |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4077 | Register SrcReg = StoreMI.getValueReg(); |
| 4078 | Register PtrReg = StoreMI.getPointerReg(); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4079 | LLT SrcTy = MRI.getType(SrcReg); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 4080 | MachineFunction &MF = MIRBuilder.getMF(); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4081 | MachineMemOperand &MMO = **StoreMI.memoperands_begin(); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 4082 | LLT MemTy = MMO.getMemoryType(); |
| 4083 | |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 4084 | unsigned StoreWidth = MemTy.getSizeInBits(); |
| 4085 | unsigned StoreSizeInBits = 8 * MemTy.getSizeInBytes(); |
| 4086 | |
| 4087 | if (StoreWidth != StoreSizeInBits) { |
Matt Arsenault | ebc17a0 | 2021-07-27 11:08:06 -0400 | [diff] [blame] | 4088 | if (SrcTy.isVector()) |
| 4089 | return UnableToLegalize; |
| 4090 | |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 4091 | // Promote to a byte-sized store with upper bits zero if not |
| 4092 | // storing an integral number of bytes. For example, promote |
| 4093 | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) |
| 4094 | LLT WideTy = LLT::scalar(StoreSizeInBits); |
| 4095 | |
| 4096 | if (StoreSizeInBits > SrcTy.getSizeInBits()) { |
| 4097 | // Avoid creating a store with a narrower source than result. |
| 4098 | SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0); |
| 4099 | SrcTy = WideTy; |
| 4100 | } |
| 4101 | |
| 4102 | auto ZextInReg = MIRBuilder.buildZExtInReg(SrcTy, SrcReg, StoreWidth); |
| 4103 | |
| 4104 | MachineMemOperand *NewMMO = |
| 4105 | MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), WideTy); |
| 4106 | MIRBuilder.buildStore(ZextInReg, PtrReg, *NewMMO); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4107 | StoreMI.eraseFromParent(); |
Matt Arsenault | a601b30 | 2021-06-08 17:11:12 -0400 | [diff] [blame] | 4108 | return Legalized; |
| 4109 | } |
| 4110 | |
Matt Arsenault | ebc17a0 | 2021-07-27 11:08:06 -0400 | [diff] [blame] | 4111 | if (MemTy.isVector()) { |
| 4112 | // TODO: Handle vector trunc stores |
| 4113 | if (MemTy != SrcTy) |
| 4114 | return UnableToLegalize; |
| 4115 | |
| 4116 | // TODO: We can do better than scalarizing the vector and at least split it |
| 4117 | // in half. |
| 4118 | return reduceLoadStoreWidth(StoreMI, 0, SrcTy.getElementType()); |
| 4119 | } |
| 4120 | |
Matt Arsenault | bc2cb91 | 2021-07-26 19:41:48 -0400 | [diff] [blame] | 4121 | unsigned MemSizeInBits = MemTy.getSizeInBits(); |
| 4122 | uint64_t LargeSplitSize, SmallSplitSize; |
| 4123 | |
| 4124 | if (!isPowerOf2_32(MemSizeInBits)) { |
Kazu Hirata | f20b507 | 2023-01-28 09:06:31 -0800 | [diff] [blame] | 4125 | LargeSplitSize = llvm::bit_floor<uint64_t>(MemTy.getSizeInBits()); |
Matt Arsenault | bc2cb91 | 2021-07-26 19:41:48 -0400 | [diff] [blame] | 4126 | SmallSplitSize = MemTy.getSizeInBits() - LargeSplitSize; |
| 4127 | } else { |
| 4128 | auto &Ctx = MF.getFunction().getContext(); |
| 4129 | if (TLI.allowsMemoryAccess(Ctx, MIRBuilder.getDataLayout(), MemTy, MMO)) |
| 4130 | return UnableToLegalize; // Don't know what we're being asked to do. |
| 4131 | |
| 4132 | SmallSplitSize = LargeSplitSize = MemSizeInBits / 2; |
| 4133 | } |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4134 | |
Amara Emerson | 9637848 | 2021-07-16 12:56:11 -0700 | [diff] [blame] | 4135 | // Extend to the next pow-2. If this store was itself the result of lowering, |
| 4136 | // e.g. an s56 store being broken into s32 + s24, we might have a stored type |
Matt Arsenault | bc2cb91 | 2021-07-26 19:41:48 -0400 | [diff] [blame] | 4137 | // that's wider than the stored size. |
| 4138 | unsigned AnyExtSize = PowerOf2Ceil(MemTy.getSizeInBits()); |
| 4139 | const LLT NewSrcTy = LLT::scalar(AnyExtSize); |
| 4140 | |
| 4141 | if (SrcTy.isPointer()) { |
| 4142 | const LLT IntPtrTy = LLT::scalar(SrcTy.getSizeInBits()); |
| 4143 | SrcReg = MIRBuilder.buildPtrToInt(IntPtrTy, SrcReg).getReg(0); |
| 4144 | } |
| 4145 | |
Amara Emerson | 9637848 | 2021-07-16 12:56:11 -0700 | [diff] [blame] | 4146 | auto ExtVal = MIRBuilder.buildAnyExtOrTrunc(NewSrcTy, SrcReg); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4147 | |
| 4148 | // Obtain the smaller value by shifting away the larger value. |
Amara Emerson | 9637848 | 2021-07-16 12:56:11 -0700 | [diff] [blame] | 4149 | auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, LargeSplitSize); |
| 4150 | auto SmallVal = MIRBuilder.buildLShr(NewSrcTy, ExtVal, ShiftAmt); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4151 | |
| 4152 | // Generate the PtrAdd and truncating stores. |
| 4153 | LLT PtrTy = MRI.getType(PtrReg); |
| 4154 | auto OffsetCst = MIRBuilder.buildConstant( |
| 4155 | LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4156 | auto SmallPtr = |
Matt Arsenault | bc2cb91 | 2021-07-26 19:41:48 -0400 | [diff] [blame] | 4157 | MIRBuilder.buildPtrAdd(PtrTy, PtrReg, OffsetCst); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4158 | |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4159 | MachineMemOperand *LargeMMO = |
| 4160 | MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8); |
| 4161 | MachineMemOperand *SmallMMO = |
| 4162 | MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8); |
Matt Arsenault | f6555b9 | 2021-06-07 14:11:52 -0400 | [diff] [blame] | 4163 | MIRBuilder.buildStore(ExtVal, PtrReg, *LargeMMO); |
| 4164 | MIRBuilder.buildStore(SmallVal, SmallPtr, *SmallMMO); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4165 | StoreMI.eraseFromParent(); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4166 | return Legalized; |
| 4167 | } |
| 4168 | |
| 4169 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4170 | LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4171 | switch (MI.getOpcode()) { |
| 4172 | case TargetOpcode::G_LOAD: { |
| 4173 | if (TypeIdx != 0) |
| 4174 | return UnableToLegalize; |
Matt Arsenault | 9236125 | 2021-06-10 19:32:41 -0400 | [diff] [blame] | 4175 | MachineMemOperand &MMO = **MI.memoperands_begin(); |
| 4176 | |
| 4177 | // Not sure how to interpret a bitcast of an extending load. |
| 4178 | if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits()) |
| 4179 | return UnableToLegalize; |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4180 | |
| 4181 | Observer.changingInstr(MI); |
| 4182 | bitcastDst(MI, CastTy, 0); |
Matt Arsenault | 9236125 | 2021-06-10 19:32:41 -0400 | [diff] [blame] | 4183 | MMO.setType(CastTy); |
Matt Arsenault | 7032076 | 2024-07-01 15:26:09 +0200 | [diff] [blame] | 4184 | // The range metadata is no longer valid when reinterpreted as a different |
| 4185 | // type. |
| 4186 | MMO.clearRanges(); |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4187 | Observer.changedInstr(MI); |
| 4188 | return Legalized; |
| 4189 | } |
| 4190 | case TargetOpcode::G_STORE: { |
| 4191 | if (TypeIdx != 0) |
| 4192 | return UnableToLegalize; |
| 4193 | |
Matt Arsenault | 9236125 | 2021-06-10 19:32:41 -0400 | [diff] [blame] | 4194 | MachineMemOperand &MMO = **MI.memoperands_begin(); |
| 4195 | |
| 4196 | // Not sure how to interpret a bitcast of a truncating store. |
| 4197 | if (MMO.getMemoryType().getSizeInBits() != CastTy.getSizeInBits()) |
| 4198 | return UnableToLegalize; |
| 4199 | |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4200 | Observer.changingInstr(MI); |
| 4201 | bitcastSrc(MI, CastTy, 0); |
Matt Arsenault | 9236125 | 2021-06-10 19:32:41 -0400 | [diff] [blame] | 4202 | MMO.setType(CastTy); |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4203 | Observer.changedInstr(MI); |
| 4204 | return Legalized; |
| 4205 | } |
| 4206 | case TargetOpcode::G_SELECT: { |
| 4207 | if (TypeIdx != 0) |
| 4208 | return UnableToLegalize; |
| 4209 | |
| 4210 | if (MRI.getType(MI.getOperand(1).getReg()).isVector()) { |
| 4211 | LLVM_DEBUG( |
| 4212 | dbgs() << "bitcast action not implemented for vector select\n"); |
| 4213 | return UnableToLegalize; |
| 4214 | } |
| 4215 | |
| 4216 | Observer.changingInstr(MI); |
| 4217 | bitcastSrc(MI, CastTy, 2); |
| 4218 | bitcastSrc(MI, CastTy, 3); |
| 4219 | bitcastDst(MI, CastTy, 0); |
| 4220 | Observer.changedInstr(MI); |
| 4221 | return Legalized; |
| 4222 | } |
| 4223 | case TargetOpcode::G_AND: |
| 4224 | case TargetOpcode::G_OR: |
| 4225 | case TargetOpcode::G_XOR: { |
| 4226 | Observer.changingInstr(MI); |
| 4227 | bitcastSrc(MI, CastTy, 1); |
| 4228 | bitcastSrc(MI, CastTy, 2); |
| 4229 | bitcastDst(MI, CastTy, 0); |
| 4230 | Observer.changedInstr(MI); |
| 4231 | return Legalized; |
| 4232 | } |
Matt Arsenault | 212570a | 2020-06-15 11:54:49 -0400 | [diff] [blame] | 4233 | case TargetOpcode::G_EXTRACT_VECTOR_ELT: |
| 4234 | return bitcastExtractVectorElt(MI, TypeIdx, CastTy); |
Matt Arsenault | e2f1b48 | 2020-06-15 21:35:15 -0400 | [diff] [blame] | 4235 | case TargetOpcode::G_INSERT_VECTOR_ELT: |
| 4236 | return bitcastInsertVectorElt(MI, TypeIdx, CastTy); |
chuongg3 | 0d5db4e | 2024-07-15 12:00:47 +0100 | [diff] [blame] | 4237 | case TargetOpcode::G_CONCAT_VECTORS: |
| 4238 | return bitcastConcatVector(MI, TypeIdx, CastTy); |
David Green | d3ce069 | 2024-11-23 17:00:51 +0000 | [diff] [blame] | 4239 | case TargetOpcode::G_SHUFFLE_VECTOR: |
| 4240 | return bitcastShuffleVector(MI, TypeIdx, CastTy); |
Michael Maitland | f957d08 | 2024-10-01 14:08:49 -0400 | [diff] [blame] | 4241 | case TargetOpcode::G_EXTRACT_SUBVECTOR: |
| 4242 | return bitcastExtractSubvector(MI, TypeIdx, CastTy); |
Michael Maitland | 6bac414 | 2024-10-21 08:49:13 -0400 | [diff] [blame] | 4243 | case TargetOpcode::G_INSERT_SUBVECTOR: |
| 4244 | return bitcastInsertSubvector(MI, TypeIdx, CastTy); |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4245 | default: |
| 4246 | return UnableToLegalize; |
| 4247 | } |
| 4248 | } |
| 4249 | |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 4250 | // Legalize an instruction by changing the opcode in place. |
| 4251 | void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) { |
| 4252 | Observer.changingInstr(MI); |
| 4253 | MI.setDesc(MIRBuilder.getTII().get(NewOpcode)); |
| 4254 | Observer.changedInstr(MI); |
| 4255 | } |
| 4256 | |
Matt Arsenault | 39c55ce | 2020-02-13 15:52:32 -0500 | [diff] [blame] | 4257 | LegalizerHelper::LegalizeResult |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4258 | LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4259 | using namespace TargetOpcode; |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4260 | |
| 4261 | switch(MI.getOpcode()) { |
| 4262 | default: |
| 4263 | return UnableToLegalize; |
Chen Zheng | 6ee2f77 | 2022-12-12 09:53:53 +0000 | [diff] [blame] | 4264 | case TargetOpcode::G_FCONSTANT: |
| 4265 | return lowerFConstant(MI); |
Matt Arsenault | 936483f | 2020-01-09 21:53:28 -0500 | [diff] [blame] | 4266 | case TargetOpcode::G_BITCAST: |
| 4267 | return lowerBitcast(MI); |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4268 | case TargetOpcode::G_SREM: |
| 4269 | case TargetOpcode::G_UREM: { |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4270 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4271 | auto Quot = |
| 4272 | MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty}, |
| 4273 | {MI.getOperand(1), MI.getOperand(2)}); |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4274 | |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4275 | auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2)); |
| 4276 | MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod); |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4277 | MI.eraseFromParent(); |
| 4278 | return Legalized; |
| 4279 | } |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 4280 | case TargetOpcode::G_SADDO: |
| 4281 | case TargetOpcode::G_SSUBO: |
| 4282 | return lowerSADDO_SSUBO(MI); |
Pushpinder Singh | 41d6669 | 2020-08-10 05:47:50 -0400 | [diff] [blame] | 4283 | case TargetOpcode::G_UMULH: |
| 4284 | case TargetOpcode::G_SMULH: |
| 4285 | return lowerSMULH_UMULH(MI); |
Tim Northover | 0a9b279 | 2017-02-08 21:22:15 +0000 | [diff] [blame] | 4286 | case TargetOpcode::G_SMULO: |
| 4287 | case TargetOpcode::G_UMULO: { |
| 4288 | // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the |
| 4289 | // result. |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4290 | auto [Res, Overflow, LHS, RHS] = MI.getFirst4Regs(); |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4291 | LLT Ty = MRI.getType(Res); |
Tim Northover | 0a9b279 | 2017-02-08 21:22:15 +0000 | [diff] [blame] | 4292 | |
Tim Northover | 0a9b279 | 2017-02-08 21:22:15 +0000 | [diff] [blame] | 4293 | unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO |
| 4294 | ? TargetOpcode::G_SMULH |
| 4295 | : TargetOpcode::G_UMULH; |
| 4296 | |
Jay Foad | f465b1a | 2020-01-16 14:46:36 +0000 | [diff] [blame] | 4297 | Observer.changingInstr(MI); |
| 4298 | const auto &TII = MIRBuilder.getTII(); |
| 4299 | MI.setDesc(TII.get(TargetOpcode::G_MUL)); |
Shengchen Kan | 37b3783 | 2022-03-16 20:21:25 +0800 | [diff] [blame] | 4300 | MI.removeOperand(1); |
Jay Foad | f465b1a | 2020-01-16 14:46:36 +0000 | [diff] [blame] | 4301 | Observer.changedInstr(MI); |
| 4302 | |
Jay Foad | f465b1a | 2020-01-16 14:46:36 +0000 | [diff] [blame] | 4303 | auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4304 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
Amara Emerson | 9de6213 | 2018-01-03 04:56:56 +0000 | [diff] [blame] | 4305 | |
Amara Emerson | 1d54e75 | 2020-09-29 14:39:54 -0700 | [diff] [blame] | 4306 | // Move insert point forward so we can use the Res register if needed. |
| 4307 | MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); |
| 4308 | |
Amara Emerson | 9de6213 | 2018-01-03 04:56:56 +0000 | [diff] [blame] | 4309 | // For *signed* multiply, overflow is detected by checking: |
| 4310 | // (hi != (lo >> bitwidth-1)) |
| 4311 | if (Opcode == TargetOpcode::G_SMULH) { |
Jay Foad | f465b1a | 2020-01-16 14:46:36 +0000 | [diff] [blame] | 4312 | auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1); |
| 4313 | auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt); |
Amara Emerson | 9de6213 | 2018-01-03 04:56:56 +0000 | [diff] [blame] | 4314 | MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted); |
| 4315 | } else { |
| 4316 | MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero); |
| 4317 | } |
Tim Northover | 0a9b279 | 2017-02-08 21:22:15 +0000 | [diff] [blame] | 4318 | return Legalized; |
| 4319 | } |
Volkan Keles | 5698b2a | 2017-03-08 18:09:14 +0000 | [diff] [blame] | 4320 | case TargetOpcode::G_FNEG: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4321 | auto [Res, SubByReg] = MI.getFirst2Regs(); |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4322 | LLT Ty = MRI.getType(Res); |
| 4323 | |
David Green | 9f255d8 | 2024-09-27 07:43:58 +0100 | [diff] [blame] | 4324 | auto SignMask = MIRBuilder.buildConstant( |
| 4325 | Ty, APInt::getSignMask(Ty.getScalarSizeInBits())); |
Eli Friedman | 3f739f7 | 2020-09-23 14:10:33 -0700 | [diff] [blame] | 4326 | MIRBuilder.buildXor(Res, SubByReg, SignMask); |
Volkan Keles | 5698b2a | 2017-03-08 18:09:14 +0000 | [diff] [blame] | 4327 | MI.eraseFromParent(); |
| 4328 | return Legalized; |
| 4329 | } |
Matt Arsenault | 1fe1299 | 2022-11-17 23:03:23 -0800 | [diff] [blame] | 4330 | case TargetOpcode::G_FSUB: |
| 4331 | case TargetOpcode::G_STRICT_FSUB: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4332 | auto [Res, LHS, RHS] = MI.getFirst3Regs(); |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4333 | LLT Ty = MRI.getType(Res); |
| 4334 | |
Volkan Keles | 225921a | 2017-03-10 21:25:09 +0000 | [diff] [blame] | 4335 | // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)). |
Matt Arsenault | 1fe1299 | 2022-11-17 23:03:23 -0800 | [diff] [blame] | 4336 | auto Neg = MIRBuilder.buildFNeg(Ty, RHS); |
| 4337 | |
| 4338 | if (MI.getOpcode() == TargetOpcode::G_STRICT_FSUB) |
| 4339 | MIRBuilder.buildStrictFAdd(Res, LHS, Neg, MI.getFlags()); |
| 4340 | else |
| 4341 | MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags()); |
| 4342 | |
Volkan Keles | 225921a | 2017-03-10 21:25:09 +0000 | [diff] [blame] | 4343 | MI.eraseFromParent(); |
| 4344 | return Legalized; |
| 4345 | } |
Matt Arsenault | 4d33918 | 2019-09-13 00:44:35 +0000 | [diff] [blame] | 4346 | case TargetOpcode::G_FMAD: |
| 4347 | return lowerFMad(MI); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 4348 | case TargetOpcode::G_FFLOOR: |
| 4349 | return lowerFFloor(MI); |
Sumanth Gundapaneni | fc832d5 | 2024-07-23 11:34:34 -0500 | [diff] [blame] | 4350 | case TargetOpcode::G_LROUND: |
| 4351 | case TargetOpcode::G_LLROUND: { |
| 4352 | Register DstReg = MI.getOperand(0).getReg(); |
| 4353 | Register SrcReg = MI.getOperand(1).getReg(); |
| 4354 | LLT SrcTy = MRI.getType(SrcReg); |
| 4355 | auto Round = MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_ROUND, {SrcTy}, |
| 4356 | {SrcReg}); |
| 4357 | MIRBuilder.buildFPTOSI(DstReg, Round); |
| 4358 | MI.eraseFromParent(); |
| 4359 | return Legalized; |
| 4360 | } |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 4361 | case TargetOpcode::G_INTRINSIC_ROUND: |
| 4362 | return lowerIntrinsicRound(MI); |
Acim-Maravic | f313852 | 2023-11-14 18:49:21 +0100 | [diff] [blame] | 4363 | case TargetOpcode::G_FRINT: { |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 4364 | // Since round even is the assumed rounding mode for unconstrained FP |
| 4365 | // operations, rint and roundeven are the same operation. |
Acim-Maravic | f313852 | 2023-11-14 18:49:21 +0100 | [diff] [blame] | 4366 | changeOpcode(MI, TargetOpcode::G_INTRINSIC_ROUNDEVEN); |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 4367 | return Legalized; |
| 4368 | } |
Sumanth Gundapaneni | 0ee32c4 | 2024-07-24 14:34:31 -0500 | [diff] [blame] | 4369 | case TargetOpcode::G_INTRINSIC_LRINT: |
| 4370 | case TargetOpcode::G_INTRINSIC_LLRINT: { |
| 4371 | Register DstReg = MI.getOperand(0).getReg(); |
| 4372 | Register SrcReg = MI.getOperand(1).getReg(); |
| 4373 | LLT SrcTy = MRI.getType(SrcReg); |
| 4374 | auto Round = |
| 4375 | MIRBuilder.buildInstr(TargetOpcode::G_FRINT, {SrcTy}, {SrcReg}); |
| 4376 | MIRBuilder.buildFPTOSI(DstReg, Round); |
| 4377 | MI.eraseFromParent(); |
| 4378 | return Legalized; |
| 4379 | } |
Daniel Sanders | aef1dfc | 2017-11-30 20:11:42 +0000 | [diff] [blame] | 4380 | case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4381 | auto [OldValRes, SuccessRes, Addr, CmpVal, NewVal] = MI.getFirst5Regs(); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4382 | Register NewOldValRes = MRI.cloneVirtualRegister(OldValRes); |
| 4383 | MIRBuilder.buildAtomicCmpXchg(NewOldValRes, Addr, CmpVal, NewVal, |
Daniel Sanders | aef1dfc | 2017-11-30 20:11:42 +0000 | [diff] [blame] | 4384 | **MI.memoperands_begin()); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4385 | MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, NewOldValRes, CmpVal); |
| 4386 | MIRBuilder.buildCopy(OldValRes, NewOldValRes); |
Daniel Sanders | aef1dfc | 2017-11-30 20:11:42 +0000 | [diff] [blame] | 4387 | MI.eraseFromParent(); |
| 4388 | return Legalized; |
| 4389 | } |
Daniel Sanders | 5eb9f58 | 2018-04-28 18:14:50 +0000 | [diff] [blame] | 4390 | case TargetOpcode::G_LOAD: |
| 4391 | case TargetOpcode::G_SEXTLOAD: |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4392 | case TargetOpcode::G_ZEXTLOAD: |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4393 | return lowerLoad(cast<GAnyLoad>(MI)); |
Matt Arsenault | 54615ec | 2020-07-31 10:09:00 -0400 | [diff] [blame] | 4394 | case TargetOpcode::G_STORE: |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 4395 | return lowerStore(cast<GStore>(MI)); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 4396 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: |
| 4397 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: |
| 4398 | case TargetOpcode::G_CTLZ: |
| 4399 | case TargetOpcode::G_CTTZ: |
| 4400 | case TargetOpcode::G_CTPOP: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4401 | return lowerBitCount(MI); |
Petar Avramovic | bd39569 | 2019-02-26 17:22:42 +0000 | [diff] [blame] | 4402 | case G_UADDO: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4403 | auto [Res, CarryOut, LHS, RHS] = MI.getFirst4Regs(); |
Petar Avramovic | bd39569 | 2019-02-26 17:22:42 +0000 | [diff] [blame] | 4404 | |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4405 | Register NewRes = MRI.cloneVirtualRegister(Res); |
| 4406 | |
| 4407 | MIRBuilder.buildAdd(NewRes, LHS, RHS); |
| 4408 | MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, NewRes, RHS); |
| 4409 | |
| 4410 | MIRBuilder.buildCopy(Res, NewRes); |
Petar Avramovic | bd39569 | 2019-02-26 17:22:42 +0000 | [diff] [blame] | 4411 | |
| 4412 | MI.eraseFromParent(); |
| 4413 | return Legalized; |
| 4414 | } |
Petar Avramovic | b8276f2 | 2018-12-17 12:31:07 +0000 | [diff] [blame] | 4415 | case G_UADDE: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4416 | auto [Res, CarryOut, LHS, RHS, CarryIn] = MI.getFirst5Regs(); |
Craig Topper | ebb2e5e | 2023-08-17 14:27:45 -0700 | [diff] [blame] | 4417 | const LLT CondTy = MRI.getType(CarryOut); |
| 4418 | const LLT Ty = MRI.getType(Res); |
Petar Avramovic | b8276f2 | 2018-12-17 12:31:07 +0000 | [diff] [blame] | 4419 | |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4420 | Register NewRes = MRI.cloneVirtualRegister(Res); |
| 4421 | |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4422 | // Initial add of the two operands. |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4423 | auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS); |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4424 | |
| 4425 | // Initial check for carry. |
| 4426 | auto Carry = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, TmpRes, LHS); |
| 4427 | |
| 4428 | // Add the sum and the carry. |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4429 | auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4430 | MIRBuilder.buildAdd(NewRes, TmpRes, ZExtCarryIn); |
Craig Topper | ebb2e5e | 2023-08-17 14:27:45 -0700 | [diff] [blame] | 4431 | |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4432 | // Second check for carry. We can only carry if the initial sum is all 1s |
| 4433 | // and the carry is set, resulting in a new sum of 0. |
| 4434 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4435 | auto ResEqZero = |
| 4436 | MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, NewRes, Zero); |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4437 | auto Carry2 = MIRBuilder.buildAnd(CondTy, ResEqZero, CarryIn); |
| 4438 | MIRBuilder.buildOr(CarryOut, Carry, Carry2); |
Petar Avramovic | b8276f2 | 2018-12-17 12:31:07 +0000 | [diff] [blame] | 4439 | |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 4440 | MIRBuilder.buildCopy(Res, NewRes); |
| 4441 | |
Petar Avramovic | b8276f2 | 2018-12-17 12:31:07 +0000 | [diff] [blame] | 4442 | MI.eraseFromParent(); |
| 4443 | return Legalized; |
| 4444 | } |
Petar Avramovic | 7cecadb | 2019-01-28 12:10:17 +0000 | [diff] [blame] | 4445 | case G_USUBO: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4446 | auto [Res, BorrowOut, LHS, RHS] = MI.getFirst4Regs(); |
Petar Avramovic | 7cecadb | 2019-01-28 12:10:17 +0000 | [diff] [blame] | 4447 | |
| 4448 | MIRBuilder.buildSub(Res, LHS, RHS); |
| 4449 | MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); |
| 4450 | |
| 4451 | MI.eraseFromParent(); |
| 4452 | return Legalized; |
| 4453 | } |
| 4454 | case G_USUBE: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4455 | auto [Res, BorrowOut, LHS, RHS, BorrowIn] = MI.getFirst5Regs(); |
Matt Arsenault | 6fc0d00 | 2020-02-26 17:21:10 -0500 | [diff] [blame] | 4456 | const LLT CondTy = MRI.getType(BorrowOut); |
| 4457 | const LLT Ty = MRI.getType(Res); |
Petar Avramovic | 7cecadb | 2019-01-28 12:10:17 +0000 | [diff] [blame] | 4458 | |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4459 | // Initial subtract of the two operands. |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4460 | auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS); |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4461 | |
| 4462 | // Initial check for borrow. |
| 4463 | auto Borrow = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, CondTy, TmpRes, LHS); |
| 4464 | |
| 4465 | // Subtract the borrow from the first subtract. |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4466 | auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn); |
Petar Avramovic | 7cecadb | 2019-01-28 12:10:17 +0000 | [diff] [blame] | 4467 | MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); |
Matt Arsenault | c7e8d8b | 2020-02-26 17:18:43 -0500 | [diff] [blame] | 4468 | |
Craig Topper | c6dee69 | 2023-08-17 20:32:37 -0700 | [diff] [blame] | 4469 | // Second check for borrow. We can only borrow if the initial difference is |
| 4470 | // 0 and the borrow is set, resulting in a new difference of all 1s. |
| 4471 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
| 4472 | auto TmpResEqZero = |
| 4473 | MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, TmpRes, Zero); |
| 4474 | auto Borrow2 = MIRBuilder.buildAnd(CondTy, TmpResEqZero, BorrowIn); |
| 4475 | MIRBuilder.buildOr(BorrowOut, Borrow, Borrow2); |
Petar Avramovic | 7cecadb | 2019-01-28 12:10:17 +0000 | [diff] [blame] | 4476 | |
| 4477 | MI.eraseFromParent(); |
| 4478 | return Legalized; |
| 4479 | } |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 4480 | case G_UITOFP: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4481 | return lowerUITOFP(MI); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 4482 | case G_SITOFP: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4483 | return lowerSITOFP(MI); |
Petar Avramovic | 6412b56 | 2019-08-30 05:44:02 +0000 | [diff] [blame] | 4484 | case G_FPTOUI: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4485 | return lowerFPTOUI(MI); |
Matt Arsenault | ea95668 | 2020-01-04 17:09:48 -0500 | [diff] [blame] | 4486 | case G_FPTOSI: |
| 4487 | return lowerFPTOSI(MI); |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 4488 | case G_FPTOUI_SAT: |
| 4489 | case G_FPTOSI_SAT: |
| 4490 | return lowerFPTOINT_SAT(MI); |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 4491 | case G_FPTRUNC: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4492 | return lowerFPTRUNC(MI); |
Matt Arsenault | 7cd8a02 | 2020-07-17 11:01:15 -0400 | [diff] [blame] | 4493 | case G_FPOWI: |
| 4494 | return lowerFPOWI(MI); |
Matt Arsenault | 6f74f55 | 2019-07-01 17:18:03 +0000 | [diff] [blame] | 4495 | case G_SMIN: |
| 4496 | case G_SMAX: |
| 4497 | case G_UMIN: |
| 4498 | case G_UMAX: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4499 | return lowerMinMax(MI); |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 4500 | case G_SCMP: |
| 4501 | case G_UCMP: |
| 4502 | return lowerThreewayCompare(MI); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 4503 | case G_FCOPYSIGN: |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 4504 | return lowerFCopySign(MI); |
Matt Arsenault | 6ce1b4f | 2019-07-10 16:31:19 +0000 | [diff] [blame] | 4505 | case G_FMINNUM: |
| 4506 | case G_FMAXNUM: |
| 4507 | return lowerFMinNumMaxNum(MI); |
Matt Arsenault | 6999960 | 2020-03-29 15:51:54 -0400 | [diff] [blame] | 4508 | case G_MERGE_VALUES: |
| 4509 | return lowerMergeValues(MI); |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 4510 | case G_UNMERGE_VALUES: |
| 4511 | return lowerUnmergeValues(MI); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 4512 | case TargetOpcode::G_SEXT_INREG: { |
| 4513 | assert(MI.getOperand(2).isImm() && "Expected immediate"); |
| 4514 | int64_t SizeInBits = MI.getOperand(2).getImm(); |
| 4515 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4516 | auto [DstReg, SrcReg] = MI.getFirst2Regs(); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 4517 | LLT DstTy = MRI.getType(DstReg); |
| 4518 | Register TmpRes = MRI.createGenericVirtualRegister(DstTy); |
| 4519 | |
| 4520 | auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 4521 | MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0)); |
| 4522 | MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0)); |
Daniel Sanders | e9a57c2 | 2019-08-09 21:11:20 +0000 | [diff] [blame] | 4523 | MI.eraseFromParent(); |
| 4524 | return Legalized; |
| 4525 | } |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4526 | case G_EXTRACT_VECTOR_ELT: |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 4527 | case G_INSERT_VECTOR_ELT: |
| 4528 | return lowerExtractInsertVectorElt(MI); |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 4529 | case G_SHUFFLE_VECTOR: |
| 4530 | return lowerShuffleVector(MI); |
Lawrence Benson | 177ce19 | 2024-07-17 14:24:24 +0200 | [diff] [blame] | 4531 | case G_VECTOR_COMPRESS: |
| 4532 | return lowerVECTOR_COMPRESS(MI); |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 4533 | case G_DYN_STACKALLOC: |
| 4534 | return lowerDynStackAlloc(MI); |
Matt Arsenault | 1ca0808 | 2023-07-29 19:12:24 -0400 | [diff] [blame] | 4535 | case G_STACKSAVE: |
| 4536 | return lowerStackSave(MI); |
| 4537 | case G_STACKRESTORE: |
| 4538 | return lowerStackRestore(MI); |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 4539 | case G_EXTRACT: |
| 4540 | return lowerExtract(MI); |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 4541 | case G_INSERT: |
| 4542 | return lowerInsert(MI); |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 4543 | case G_BSWAP: |
| 4544 | return lowerBswap(MI); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 4545 | case G_BITREVERSE: |
| 4546 | return lowerBitreverse(MI); |
Matt Arsenault | 0ea3c72 | 2019-12-27 19:26:51 -0500 | [diff] [blame] | 4547 | case G_READ_REGISTER: |
Matt Arsenault | c5c1bb3 | 2020-01-12 13:29:44 -0500 | [diff] [blame] | 4548 | case G_WRITE_REGISTER: |
| 4549 | return lowerReadWriteRegister(MI); |
Jay Foad | b35833b | 2020-07-12 14:18:45 -0400 | [diff] [blame] | 4550 | case G_UADDSAT: |
| 4551 | case G_USUBSAT: { |
| 4552 | // Try to make a reasonable guess about which lowering strategy to use. The |
| 4553 | // target can override this with custom lowering and calling the |
| 4554 | // implementation functions. |
| 4555 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
AtariDreams | d582958 | 2024-05-28 12:25:43 -0400 | [diff] [blame] | 4556 | if (LI.isLegalOrCustom({G_UMIN, Ty})) |
Jay Foad | b35833b | 2020-07-12 14:18:45 -0400 | [diff] [blame] | 4557 | return lowerAddSubSatToMinMax(MI); |
| 4558 | return lowerAddSubSatToAddoSubo(MI); |
| 4559 | } |
| 4560 | case G_SADDSAT: |
| 4561 | case G_SSUBSAT: { |
| 4562 | LLT Ty = MRI.getType(MI.getOperand(0).getReg()); |
| 4563 | |
| 4564 | // FIXME: It would probably make more sense to see if G_SADDO is preferred, |
| 4565 | // since it's a shorter expansion. However, we would need to figure out the |
| 4566 | // preferred boolean type for the carry out for the query. |
| 4567 | if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty})) |
| 4568 | return lowerAddSubSatToMinMax(MI); |
| 4569 | return lowerAddSubSatToAddoSubo(MI); |
| 4570 | } |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 4571 | case G_SSHLSAT: |
| 4572 | case G_USHLSAT: |
| 4573 | return lowerShlSat(MI); |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 4574 | case G_ABS: |
| 4575 | return lowerAbsToAddXor(MI); |
Him188 | 0748f42 | 2024-09-03 12:47:26 +0100 | [diff] [blame] | 4576 | case G_FABS: |
| 4577 | return lowerFAbs(MI); |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 4578 | case G_SELECT: |
| 4579 | return lowerSelect(MI); |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 4580 | case G_IS_FPCLASS: |
| 4581 | return lowerISFPCLASS(MI); |
Christudasan Devadasan | 4c6ab48 | 2021-03-10 18:03:10 +0530 | [diff] [blame] | 4582 | case G_SDIVREM: |
| 4583 | case G_UDIVREM: |
| 4584 | return lowerDIVREM(MI); |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 4585 | case G_FSHL: |
| 4586 | case G_FSHR: |
| 4587 | return lowerFunnelShift(MI); |
Amara Emerson | f5e9be6 | 2021-03-26 15:27:15 -0700 | [diff] [blame] | 4588 | case G_ROTL: |
| 4589 | case G_ROTR: |
| 4590 | return lowerRotate(MI); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 4591 | case G_MEMSET: |
| 4592 | case G_MEMCPY: |
| 4593 | case G_MEMMOVE: |
| 4594 | return lowerMemCpyFamily(MI); |
| 4595 | case G_MEMCPY_INLINE: |
| 4596 | return lowerMemcpyInline(MI); |
Tuan Chuong Goh | a40c984 | 2023-08-17 16:31:54 +0100 | [diff] [blame] | 4597 | case G_ZEXT: |
| 4598 | case G_SEXT: |
| 4599 | case G_ANYEXT: |
| 4600 | return lowerEXT(MI); |
chuongg3 | d88d983 | 2023-10-11 16:05:25 +0100 | [diff] [blame] | 4601 | case G_TRUNC: |
| 4602 | return lowerTRUNC(MI); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 4603 | GISEL_VECREDUCE_CASES_NONSEQ |
| 4604 | return lowerVectorReduction(MI); |
Michael Maitland | 6f9cb9a7 | 2023-12-08 13:24:27 -0500 | [diff] [blame] | 4605 | case G_VAARG: |
| 4606 | return lowerVAArg(MI); |
Tim Northover | cecee56 | 2016-08-26 17:46:13 +0000 | [diff] [blame] | 4607 | } |
| 4608 | } |
| 4609 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4610 | Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty, |
| 4611 | Align MinAlign) const { |
| 4612 | // FIXME: We're missing a way to go back from LLT to llvm::Type to query the |
| 4613 | // datalayout for the preferred alignment. Also there should be a target hook |
| 4614 | // for this to allow targets to reduce the alignment and ignore the |
| 4615 | // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of |
| 4616 | // the type. |
| 4617 | return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign); |
| 4618 | } |
| 4619 | |
| 4620 | MachineInstrBuilder |
| 4621 | LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment, |
| 4622 | MachinePointerInfo &PtrInfo) { |
| 4623 | MachineFunction &MF = MIRBuilder.getMF(); |
| 4624 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 4625 | int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false); |
| 4626 | |
| 4627 | unsigned AddrSpace = DL.getAllocaAddrSpace(); |
| 4628 | LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)); |
| 4629 | |
| 4630 | PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx); |
| 4631 | return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx); |
| 4632 | } |
| 4633 | |
Owen Anderson | 44b717d | 2024-02-21 00:42:22 -0500 | [diff] [blame] | 4634 | static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg, |
| 4635 | LLT VecTy) { |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4636 | LLT IdxTy = B.getMRI()->getType(IdxReg); |
| 4637 | unsigned NElts = VecTy.getNumElements(); |
Owen Anderson | 44b717d | 2024-02-21 00:42:22 -0500 | [diff] [blame] | 4638 | |
| 4639 | int64_t IdxVal; |
| 4640 | if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal))) { |
| 4641 | if (IdxVal < VecTy.getNumElements()) |
| 4642 | return IdxReg; |
| 4643 | // If a constant index would be out of bounds, clamp it as well. |
| 4644 | } |
| 4645 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4646 | if (isPowerOf2_32(NElts)) { |
| 4647 | APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts)); |
| 4648 | return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0); |
| 4649 | } |
| 4650 | |
| 4651 | return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1)) |
| 4652 | .getReg(0); |
| 4653 | } |
| 4654 | |
| 4655 | Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy, |
| 4656 | Register Index) { |
| 4657 | LLT EltTy = VecTy.getElementType(); |
| 4658 | |
| 4659 | // Calculate the element offset and add it to the pointer. |
| 4660 | unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size. |
| 4661 | assert(EltSize * 8 == EltTy.getSizeInBits() && |
| 4662 | "Converting bits to bytes lost precision"); |
| 4663 | |
Owen Anderson | 44b717d | 2024-02-21 00:42:22 -0500 | [diff] [blame] | 4664 | Index = clampVectorIndex(MIRBuilder, Index, VecTy); |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4665 | |
Jay Foad | fd3eaf7 | 2024-03-09 09:07:22 +0000 | [diff] [blame] | 4666 | // Convert index to the correct size for the address space. |
| 4667 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 4668 | unsigned AS = MRI.getType(VecPtr).getAddressSpace(); |
| 4669 | unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8; |
| 4670 | LLT IdxTy = MRI.getType(Index).changeElementSize(IndexSizeInBits); |
| 4671 | if (IdxTy != MRI.getType(Index)) |
| 4672 | Index = MIRBuilder.buildSExtOrTrunc(IdxTy, Index).getReg(0); |
| 4673 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 4674 | auto Mul = MIRBuilder.buildMul(IdxTy, Index, |
| 4675 | MIRBuilder.buildConstant(IdxTy, EltSize)); |
| 4676 | |
| 4677 | LLT PtrTy = MRI.getType(VecPtr); |
| 4678 | return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0); |
| 4679 | } |
| 4680 | |
Fangrui Song | ea2d4c5 | 2021-12-24 00:55:54 -0800 | [diff] [blame] | 4681 | #ifndef NDEBUG |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4682 | /// Check that all vector operands have same number of elements. Other operands |
| 4683 | /// should be listed in NonVecOp. |
| 4684 | static bool hasSameNumEltsOnAllVectorOperands( |
| 4685 | GenericMachineInstr &MI, MachineRegisterInfo &MRI, |
| 4686 | std::initializer_list<unsigned> NonVecOpIndices) { |
| 4687 | if (MI.getNumMemOperands() != 0) |
| 4688 | return false; |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 4689 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4690 | LLT VecTy = MRI.getType(MI.getReg(0)); |
| 4691 | if (!VecTy.isVector()) |
| 4692 | return false; |
| 4693 | unsigned NumElts = VecTy.getNumElements(); |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 4694 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4695 | for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) { |
| 4696 | MachineOperand &Op = MI.getOperand(OpIdx); |
| 4697 | if (!Op.isReg()) { |
| 4698 | if (!is_contained(NonVecOpIndices, OpIdx)) |
| 4699 | return false; |
| 4700 | continue; |
| 4701 | } |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 4702 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4703 | LLT Ty = MRI.getType(Op.getReg()); |
| 4704 | if (!Ty.isVector()) { |
| 4705 | if (!is_contained(NonVecOpIndices, OpIdx)) |
| 4706 | return false; |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4707 | continue; |
| 4708 | } |
| 4709 | |
| 4710 | if (Ty.getNumElements() != NumElts) |
| 4711 | return false; |
| 4712 | } |
| 4713 | |
| 4714 | return true; |
| 4715 | } |
Fangrui Song | ea2d4c5 | 2021-12-24 00:55:54 -0800 | [diff] [blame] | 4716 | #endif |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4717 | |
| 4718 | /// Fill \p DstOps with DstOps that have same number of elements combined as |
| 4719 | /// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are |
| 4720 | /// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple |
| 4721 | /// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements. |
| 4722 | static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty, |
| 4723 | unsigned NumElts) { |
| 4724 | LLT LeftoverTy; |
| 4725 | assert(Ty.isVector() && "Expected vector type"); |
| 4726 | LLT EltTy = Ty.getElementType(); |
| 4727 | LLT NarrowTy = (NumElts == 1) ? EltTy : LLT::fixed_vector(NumElts, EltTy); |
| 4728 | int NumParts, NumLeftover; |
| 4729 | std::tie(NumParts, NumLeftover) = |
| 4730 | getNarrowTypeBreakDown(Ty, NarrowTy, LeftoverTy); |
| 4731 | |
| 4732 | assert(NumParts > 0 && "Error in getNarrowTypeBreakDown"); |
| 4733 | for (int i = 0; i < NumParts; ++i) { |
| 4734 | DstOps.push_back(NarrowTy); |
| 4735 | } |
| 4736 | |
| 4737 | if (LeftoverTy.isValid()) { |
| 4738 | assert(NumLeftover == 1 && "expected exactly one leftover"); |
| 4739 | DstOps.push_back(LeftoverTy); |
| 4740 | } |
| 4741 | } |
| 4742 | |
| 4743 | /// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps |
| 4744 | /// made from \p Op depending on operand type. |
| 4745 | static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N, |
| 4746 | MachineOperand &Op) { |
| 4747 | for (unsigned i = 0; i < N; ++i) { |
| 4748 | if (Op.isReg()) |
| 4749 | Ops.push_back(Op.getReg()); |
| 4750 | else if (Op.isImm()) |
| 4751 | Ops.push_back(Op.getImm()); |
| 4752 | else if (Op.isPredicate()) |
| 4753 | Ops.push_back(static_cast<CmpInst::Predicate>(Op.getPredicate())); |
| 4754 | else |
| 4755 | llvm_unreachable("Unsupported type"); |
| 4756 | } |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 4757 | } |
| 4758 | |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4759 | // Handle splitting vector operations which need to have the same number of |
| 4760 | // elements in each type index, but each type index may have a different element |
| 4761 | // type. |
| 4762 | // |
| 4763 | // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> |
| 4764 | // <2 x s64> = G_SHL <2 x s64>, <2 x s32> |
| 4765 | // <2 x s64> = G_SHL <2 x s64>, <2 x s32> |
| 4766 | // |
| 4767 | // Also handles some irregular breakdown cases, e.g. |
| 4768 | // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> |
| 4769 | // <2 x s64> = G_SHL <2 x s64>, <2 x s32> |
| 4770 | // s64 = G_SHL s64, s32 |
| 4771 | LegalizerHelper::LegalizeResult |
| 4772 | LegalizerHelper::fewerElementsVectorMultiEltType( |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4773 | GenericMachineInstr &MI, unsigned NumElts, |
| 4774 | std::initializer_list<unsigned> NonVecOpIndices) { |
| 4775 | assert(hasSameNumEltsOnAllVectorOperands(MI, MRI, NonVecOpIndices) && |
| 4776 | "Non-compatible opcode or not specified non-vector operands"); |
| 4777 | unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements(); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4778 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4779 | unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs(); |
| 4780 | unsigned NumDefs = MI.getNumDefs(); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4781 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4782 | // Create DstOps (sub-vectors with NumElts elts + Leftover) for each output. |
| 4783 | // Build instructions with DstOps to use instruction found by CSE directly. |
| 4784 | // CSE copies found instruction into given vreg when building with vreg dest. |
| 4785 | SmallVector<SmallVector<DstOp, 8>, 2> OutputOpsPieces(NumDefs); |
| 4786 | // Output registers will be taken from created instructions. |
| 4787 | SmallVector<SmallVector<Register, 8>, 2> OutputRegs(NumDefs); |
| 4788 | for (unsigned i = 0; i < NumDefs; ++i) { |
| 4789 | makeDstOps(OutputOpsPieces[i], MRI.getType(MI.getReg(i)), NumElts); |
| 4790 | } |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4791 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4792 | // Split vector input operands into sub-vectors with NumElts elts + Leftover. |
| 4793 | // Operands listed in NonVecOpIndices will be used as is without splitting; |
| 4794 | // examples: compare predicate in icmp and fcmp (op 1), vector select with i1 |
| 4795 | // scalar condition (op 1), immediate in sext_inreg (op 2). |
| 4796 | SmallVector<SmallVector<SrcOp, 8>, 3> InputOpsPieces(NumInputs); |
| 4797 | for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands(); |
| 4798 | ++UseIdx, ++UseNo) { |
| 4799 | if (is_contained(NonVecOpIndices, UseIdx)) { |
| 4800 | broadcastSrcOp(InputOpsPieces[UseNo], OutputOpsPieces[0].size(), |
| 4801 | MI.getOperand(UseIdx)); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4802 | } else { |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4803 | SmallVector<Register, 8> SplitPieces; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 4804 | extractVectorParts(MI.getReg(UseIdx), NumElts, SplitPieces, MIRBuilder, |
| 4805 | MRI); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4806 | for (auto Reg : SplitPieces) |
| 4807 | InputOpsPieces[UseNo].push_back(Reg); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4808 | } |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4809 | } |
| 4810 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4811 | unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0; |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4812 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4813 | // Take i-th piece of each input operand split and build sub-vector/scalar |
| 4814 | // instruction. Set i-th DstOp(s) from OutputOpsPieces as destination(s). |
| 4815 | for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) { |
| 4816 | SmallVector<DstOp, 2> Defs; |
| 4817 | for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo) |
| 4818 | Defs.push_back(OutputOpsPieces[DstNo][i]); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4819 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4820 | SmallVector<SrcOp, 3> Uses; |
| 4821 | for (unsigned InputNo = 0; InputNo < NumInputs; ++InputNo) |
| 4822 | Uses.push_back(InputOpsPieces[InputNo][i]); |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 4823 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4824 | auto I = MIRBuilder.buildInstr(MI.getOpcode(), Defs, Uses, MI.getFlags()); |
| 4825 | for (unsigned DstNo = 0; DstNo < NumDefs; ++DstNo) |
| 4826 | OutputRegs[DstNo].push_back(I.getReg(DstNo)); |
| 4827 | } |
Matt Arsenault | ca67634 | 2019-01-25 02:36:32 +0000 | [diff] [blame] | 4828 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4829 | // Merge small outputs into MI's output for each def operand. |
| 4830 | if (NumLeftovers) { |
| 4831 | for (unsigned i = 0; i < NumDefs; ++i) |
| 4832 | mergeMixedSubvectors(MI.getReg(i), OutputRegs[i]); |
Matt Arsenault | cbaada6 | 2019-02-02 23:29:55 +0000 | [diff] [blame] | 4833 | } else { |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4834 | for (unsigned i = 0; i < NumDefs; ++i) |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 4835 | MIRBuilder.buildMergeLikeInstr(MI.getReg(i), OutputRegs[i]); |
Matt Arsenault | ca67634 | 2019-01-25 02:36:32 +0000 | [diff] [blame] | 4836 | } |
| 4837 | |
Matt Arsenault | 1b1e685 | 2019-01-25 02:59:34 +0000 | [diff] [blame] | 4838 | MI.eraseFromParent(); |
| 4839 | return Legalized; |
| 4840 | } |
| 4841 | |
| 4842 | LegalizerHelper::LegalizeResult |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4843 | LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI, |
| 4844 | unsigned NumElts) { |
| 4845 | unsigned OrigNumElts = MRI.getType(MI.getReg(0)).getNumElements(); |
Matt Arsenault | 1b1e685 | 2019-01-25 02:59:34 +0000 | [diff] [blame] | 4846 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4847 | unsigned NumInputs = MI.getNumOperands() - MI.getNumDefs(); |
| 4848 | unsigned NumDefs = MI.getNumDefs(); |
Matt Arsenault | 1b1e685 | 2019-01-25 02:59:34 +0000 | [diff] [blame] | 4849 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4850 | SmallVector<DstOp, 8> OutputOpsPieces; |
| 4851 | SmallVector<Register, 8> OutputRegs; |
| 4852 | makeDstOps(OutputOpsPieces, MRI.getType(MI.getReg(0)), NumElts); |
Matt Arsenault | 1b1e685 | 2019-01-25 02:59:34 +0000 | [diff] [blame] | 4853 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4854 | // Instructions that perform register split will be inserted in basic block |
| 4855 | // where register is defined (basic block is in the next operand). |
| 4856 | SmallVector<SmallVector<Register, 8>, 3> InputOpsPieces(NumInputs / 2); |
| 4857 | for (unsigned UseIdx = NumDefs, UseNo = 0; UseIdx < MI.getNumOperands(); |
| 4858 | UseIdx += 2, ++UseNo) { |
| 4859 | MachineBasicBlock &OpMBB = *MI.getOperand(UseIdx + 1).getMBB(); |
Amara Emerson | 53445f5 | 2022-11-13 01:43:04 -0800 | [diff] [blame] | 4860 | MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminatorForward()); |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 4861 | extractVectorParts(MI.getReg(UseIdx), NumElts, InputOpsPieces[UseNo], |
| 4862 | MIRBuilder, MRI); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4863 | } |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 4864 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4865 | // Build PHIs with fewer elements. |
| 4866 | unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0; |
| 4867 | MIRBuilder.setInsertPt(*MI.getParent(), MI); |
| 4868 | for (unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) { |
| 4869 | auto Phi = MIRBuilder.buildInstr(TargetOpcode::G_PHI); |
| 4870 | Phi.addDef( |
| 4871 | MRI.createGenericVirtualRegister(OutputOpsPieces[i].getLLTTy(MRI))); |
| 4872 | OutputRegs.push_back(Phi.getReg(0)); |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 4873 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4874 | for (unsigned j = 0; j < NumInputs / 2; ++j) { |
| 4875 | Phi.addUse(InputOpsPieces[j][i]); |
| 4876 | Phi.add(MI.getOperand(1 + j * 2 + 1)); |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 4877 | } |
| 4878 | } |
| 4879 | |
Dávid Ferenc Szabó | 2347020 | 2024-04-15 11:01:55 +0200 | [diff] [blame] | 4880 | // Set the insert point after the existing PHIs |
| 4881 | MachineBasicBlock &MBB = *MI.getParent(); |
| 4882 | MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI()); |
| 4883 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4884 | // Merge small outputs into MI's def. |
| 4885 | if (NumLeftovers) { |
| 4886 | mergeMixedSubvectors(MI.getReg(0), OutputRegs); |
| 4887 | } else { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 4888 | MIRBuilder.buildMergeLikeInstr(MI.getReg(0), OutputRegs); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4889 | } |
| 4890 | |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 4891 | MI.eraseFromParent(); |
| 4892 | return Legalized; |
| 4893 | } |
| 4894 | |
| 4895 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4896 | LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI, |
| 4897 | unsigned TypeIdx, |
| 4898 | LLT NarrowTy) { |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4899 | const int NumDst = MI.getNumOperands() - 1; |
| 4900 | const Register SrcReg = MI.getOperand(NumDst).getReg(); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4901 | LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4902 | LLT SrcTy = MRI.getType(SrcReg); |
| 4903 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4904 | if (TypeIdx != 1 || NarrowTy == DstTy) |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4905 | return UnableToLegalize; |
| 4906 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4907 | // Requires compatible types. Otherwise SrcReg should have been defined by |
| 4908 | // merge-like instruction that would get artifact combined. Most likely |
| 4909 | // instruction that defines SrcReg has to perform more/fewer elements |
| 4910 | // legalization compatible with NarrowTy. |
| 4911 | assert(SrcTy.isVector() && NarrowTy.isVector() && "Expected vector types"); |
| 4912 | assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4913 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4914 | if ((SrcTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || |
| 4915 | (NarrowTy.getSizeInBits() % DstTy.getSizeInBits() != 0)) |
| 4916 | return UnableToLegalize; |
| 4917 | |
| 4918 | // This is most likely DstTy (smaller then register size) packed in SrcTy |
| 4919 | // (larger then register size) and since unmerge was not combined it will be |
| 4920 | // lowered to bit sequence extracts from register. Unpack SrcTy to NarrowTy |
| 4921 | // (register size) pieces first. Then unpack each of NarrowTy pieces to DstTy. |
| 4922 | |
| 4923 | // %1:_(DstTy), %2, %3, %4 = G_UNMERGE_VALUES %0:_(SrcTy) |
| 4924 | // |
| 4925 | // %5:_(NarrowTy), %6 = G_UNMERGE_VALUES %0:_(SrcTy) - reg sequence |
| 4926 | // %1:_(DstTy), %2 = G_UNMERGE_VALUES %5:_(NarrowTy) - sequence of bits in reg |
| 4927 | // %3:_(DstTy), %4 = G_UNMERGE_VALUES %6:_(NarrowTy) |
| 4928 | auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, SrcReg); |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 4929 | const int NumUnmerge = Unmerge->getNumOperands() - 1; |
| 4930 | const int PartsPerUnmerge = NumDst / NumUnmerge; |
| 4931 | |
| 4932 | for (int I = 0; I != NumUnmerge; ++I) { |
| 4933 | auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES); |
| 4934 | |
| 4935 | for (int J = 0; J != PartsPerUnmerge; ++J) |
| 4936 | MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg()); |
| 4937 | MIB.addUse(Unmerge.getReg(I)); |
| 4938 | } |
| 4939 | |
| 4940 | MI.eraseFromParent(); |
| 4941 | return Legalized; |
| 4942 | } |
| 4943 | |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 4944 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 901e331 | 2020-08-03 18:37:29 -0400 | [diff] [blame] | 4945 | LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, |
| 4946 | LLT NarrowTy) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 4947 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4948 | // Requires compatible types. Otherwise user of DstReg did not perform unmerge |
| 4949 | // that should have been artifact combined. Most likely instruction that uses |
| 4950 | // DstReg has to do more/fewer elements legalization compatible with NarrowTy. |
| 4951 | assert(DstTy.isVector() && NarrowTy.isVector() && "Expected vector types"); |
| 4952 | assert((DstTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); |
| 4953 | if (NarrowTy == SrcTy) |
| 4954 | return UnableToLegalize; |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 4955 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4956 | // This attempts to lower part of LCMTy merge/unmerge sequence. Intended use |
| 4957 | // is for old mir tests. Since the changes to more/fewer elements it should no |
| 4958 | // longer be possible to generate MIR like this when starting from llvm-ir |
| 4959 | // because LCMTy approach was replaced with merge/unmerge to vector elements. |
| 4960 | if (TypeIdx == 1) { |
| 4961 | assert(SrcTy.isVector() && "Expected vector types"); |
| 4962 | assert((SrcTy.getScalarType() == NarrowTy.getScalarType()) && "bad type"); |
| 4963 | if ((DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0) || |
| 4964 | (NarrowTy.getNumElements() >= SrcTy.getNumElements())) |
| 4965 | return UnableToLegalize; |
| 4966 | // %2:_(DstTy) = G_CONCAT_VECTORS %0:_(SrcTy), %1:_(SrcTy) |
| 4967 | // |
| 4968 | // %3:_(EltTy), %4, %5 = G_UNMERGE_VALUES %0:_(SrcTy) |
| 4969 | // %6:_(EltTy), %7, %8 = G_UNMERGE_VALUES %1:_(SrcTy) |
| 4970 | // %9:_(NarrowTy) = G_BUILD_VECTOR %3:_(EltTy), %4 |
| 4971 | // %10:_(NarrowTy) = G_BUILD_VECTOR %5:_(EltTy), %6 |
| 4972 | // %11:_(NarrowTy) = G_BUILD_VECTOR %7:_(EltTy), %8 |
| 4973 | // %2:_(DstTy) = G_CONCAT_VECTORS %9:_(NarrowTy), %10, %11 |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 4974 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4975 | SmallVector<Register, 8> Elts; |
| 4976 | LLT EltTy = MRI.getType(MI.getOperand(1).getReg()).getScalarType(); |
| 4977 | for (unsigned i = 1; i < MI.getNumOperands(); ++i) { |
| 4978 | auto Unmerge = MIRBuilder.buildUnmerge(EltTy, MI.getOperand(i).getReg()); |
| 4979 | for (unsigned j = 0; j < Unmerge->getNumDefs(); ++j) |
| 4980 | Elts.push_back(Unmerge.getReg(j)); |
| 4981 | } |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 4982 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4983 | SmallVector<Register, 8> NarrowTyElts; |
| 4984 | unsigned NumNarrowTyElts = NarrowTy.getNumElements(); |
| 4985 | unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts; |
| 4986 | for (unsigned i = 0, Offset = 0; i < NumNarrowTyPieces; |
| 4987 | ++i, Offset += NumNarrowTyElts) { |
| 4988 | ArrayRef<Register> Pieces(&Elts[Offset], NumNarrowTyElts); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 4989 | NarrowTyElts.push_back( |
| 4990 | MIRBuilder.buildMergeLikeInstr(NarrowTy, Pieces).getReg(0)); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4991 | } |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 4992 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 4993 | MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 4994 | MI.eraseFromParent(); |
| 4995 | return Legalized; |
| 4996 | } |
| 4997 | |
| 4998 | assert(TypeIdx == 0 && "Bad type index"); |
| 4999 | if ((NarrowTy.getSizeInBits() % SrcTy.getSizeInBits() != 0) || |
| 5000 | (DstTy.getSizeInBits() % NarrowTy.getSizeInBits() != 0)) |
| 5001 | return UnableToLegalize; |
| 5002 | |
| 5003 | // This is most likely SrcTy (smaller then register size) packed in DstTy |
| 5004 | // (larger then register size) and since merge was not combined it will be |
| 5005 | // lowered to bit sequence packing into register. Merge SrcTy to NarrowTy |
| 5006 | // (register size) pieces first. Then merge each of NarrowTy pieces to DstTy. |
| 5007 | |
| 5008 | // %0:_(DstTy) = G_MERGE_VALUES %1:_(SrcTy), %2, %3, %4 |
| 5009 | // |
| 5010 | // %5:_(NarrowTy) = G_MERGE_VALUES %1:_(SrcTy), %2 - sequence of bits in reg |
| 5011 | // %6:_(NarrowTy) = G_MERGE_VALUES %3:_(SrcTy), %4 |
| 5012 | // %0:_(DstTy) = G_MERGE_VALUES %5:_(NarrowTy), %6 - reg sequence |
| 5013 | SmallVector<Register, 8> NarrowTyElts; |
| 5014 | unsigned NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); |
| 5015 | unsigned NumSrcElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1; |
| 5016 | unsigned NumElts = NarrowTy.getNumElements() / NumSrcElts; |
| 5017 | for (unsigned i = 0; i < NumParts; ++i) { |
| 5018 | SmallVector<Register, 8> Sources; |
| 5019 | for (unsigned j = 0; j < NumElts; ++j) |
| 5020 | Sources.push_back(MI.getOperand(1 + i * NumElts + j).getReg()); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 5021 | NarrowTyElts.push_back( |
| 5022 | MIRBuilder.buildMergeLikeInstr(NarrowTy, Sources).getReg(0)); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5023 | } |
| 5024 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 5025 | MIRBuilder.buildMergeLikeInstr(DstReg, NarrowTyElts); |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 5026 | MI.eraseFromParent(); |
| 5027 | return Legalized; |
| 5028 | } |
| 5029 | |
| 5030 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5031 | LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, |
| 5032 | unsigned TypeIdx, |
| 5033 | LLT NarrowVecTy) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 5034 | auto [DstReg, SrcVec] = MI.getFirst2Regs(); |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5035 | Register InsertVal; |
| 5036 | bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT; |
| 5037 | |
| 5038 | assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index"); |
| 5039 | if (IsInsert) |
| 5040 | InsertVal = MI.getOperand(2).getReg(); |
| 5041 | |
| 5042 | Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg(); |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5043 | |
| 5044 | // TODO: Handle total scalarization case. |
| 5045 | if (!NarrowVecTy.isVector()) |
| 5046 | return UnableToLegalize; |
| 5047 | |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5048 | LLT VecTy = MRI.getType(SrcVec); |
| 5049 | |
| 5050 | // If the index is a constant, we can really break this down as you would |
| 5051 | // expect, and index into the target size pieces. |
| 5052 | int64_t IdxVal; |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 5053 | auto MaybeCst = getIConstantVRegValWithLookThrough(Idx, MRI); |
Amara Emerson | 59a4ee9 | 2021-05-26 23:28:44 -0700 | [diff] [blame] | 5054 | if (MaybeCst) { |
| 5055 | IdxVal = MaybeCst->Value.getSExtValue(); |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5056 | // Avoid out of bounds indexing the pieces. |
| 5057 | if (IdxVal >= VecTy.getNumElements()) { |
| 5058 | MIRBuilder.buildUndef(DstReg); |
| 5059 | MI.eraseFromParent(); |
| 5060 | return Legalized; |
| 5061 | } |
| 5062 | |
| 5063 | SmallVector<Register, 8> VecParts; |
| 5064 | LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec); |
| 5065 | |
| 5066 | // Build a sequence of NarrowTy pieces in VecParts for this operand. |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5067 | LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts, |
| 5068 | TargetOpcode::G_ANYEXT); |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5069 | |
| 5070 | unsigned NewNumElts = NarrowVecTy.getNumElements(); |
| 5071 | |
| 5072 | LLT IdxTy = MRI.getType(Idx); |
| 5073 | int64_t PartIdx = IdxVal / NewNumElts; |
| 5074 | auto NewIdx = |
| 5075 | MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx); |
| 5076 | |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5077 | if (IsInsert) { |
| 5078 | LLT PartTy = MRI.getType(VecParts[PartIdx]); |
| 5079 | |
| 5080 | // Use the adjusted index to insert into one of the subvectors. |
| 5081 | auto InsertPart = MIRBuilder.buildInsertVectorElement( |
| 5082 | PartTy, VecParts[PartIdx], InsertVal, NewIdx); |
| 5083 | VecParts[PartIdx] = InsertPart.getReg(0); |
| 5084 | |
| 5085 | // Recombine the inserted subvector with the others to reform the result |
| 5086 | // vector. |
| 5087 | buildWidenedRemergeToDst(DstReg, LCMTy, VecParts); |
| 5088 | } else { |
| 5089 | MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx); |
| 5090 | } |
| 5091 | |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5092 | MI.eraseFromParent(); |
| 5093 | return Legalized; |
| 5094 | } |
| 5095 | |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5096 | // With a variable index, we can't perform the operation in a smaller type, so |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5097 | // we're forced to expand this. |
| 5098 | // |
| 5099 | // TODO: We could emit a chain of compare/select to figure out which piece to |
| 5100 | // index. |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 5101 | return lowerExtractInsertVectorElt(MI); |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5102 | } |
| 5103 | |
| 5104 | LegalizerHelper::LegalizeResult |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5105 | LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx, |
Matt Arsenault | 7f09fd6 | 2019-02-05 00:26:12 +0000 | [diff] [blame] | 5106 | LLT NarrowTy) { |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5107 | // FIXME: Don't know how to handle secondary types yet. |
| 5108 | if (TypeIdx != 0) |
| 5109 | return UnableToLegalize; |
| 5110 | |
Matt Arsenault | cfca2a7 | 2019-01-27 22:36:24 +0000 | [diff] [blame] | 5111 | // This implementation doesn't work for atomics. Give up instead of doing |
| 5112 | // something invalid. |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5113 | if (LdStMI.isAtomic()) |
Matt Arsenault | cfca2a7 | 2019-01-27 22:36:24 +0000 | [diff] [blame] | 5114 | return UnableToLegalize; |
| 5115 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5116 | bool IsLoad = isa<GLoad>(LdStMI); |
| 5117 | Register ValReg = LdStMI.getReg(0); |
| 5118 | Register AddrReg = LdStMI.getPointerReg(); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5119 | LLT ValTy = MRI.getType(ValReg); |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5120 | |
Matt Arsenault | c0ad75e | 2020-02-13 15:08:59 -0500 | [diff] [blame] | 5121 | // FIXME: Do we need a distinct NarrowMemory legalize action? |
David Green | 601e102 | 2024-03-17 18:15:56 +0000 | [diff] [blame] | 5122 | if (ValTy.getSizeInBits() != 8 * LdStMI.getMemSize().getValue()) { |
Matt Arsenault | c0ad75e | 2020-02-13 15:08:59 -0500 | [diff] [blame] | 5123 | LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n"); |
| 5124 | return UnableToLegalize; |
| 5125 | } |
| 5126 | |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5127 | int NumParts = -1; |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 5128 | int NumLeftover = -1; |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5129 | LLT LeftoverTy; |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 5130 | SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs; |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5131 | if (IsLoad) { |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 5132 | std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5133 | } else { |
| 5134 | if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 5135 | NarrowLeftoverRegs, MIRBuilder, MRI)) { |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5136 | NumParts = NarrowRegs.size(); |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 5137 | NumLeftover = NarrowLeftoverRegs.size(); |
| 5138 | } |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5139 | } |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5140 | |
| 5141 | if (NumParts == -1) |
| 5142 | return UnableToLegalize; |
| 5143 | |
Matt Arsenault | 1ea182c | 2020-07-31 10:19:02 -0400 | [diff] [blame] | 5144 | LLT PtrTy = MRI.getType(AddrReg); |
| 5145 | const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits()); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5146 | |
| 5147 | unsigned TotalSize = ValTy.getSizeInBits(); |
| 5148 | |
| 5149 | // Split the load/store into PartTy sized pieces starting at Offset. If this |
| 5150 | // is a load, return the new registers in ValRegs. For a store, each elements |
| 5151 | // of ValRegs should be PartTy. Returns the next offset that needs to be |
| 5152 | // handled. |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5153 | bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian(); |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5154 | auto MMO = LdStMI.getMMO(); |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 5155 | auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs, |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5156 | unsigned NumParts, unsigned Offset) -> unsigned { |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5157 | MachineFunction &MF = MIRBuilder.getMF(); |
| 5158 | unsigned PartSize = PartTy.getSizeInBits(); |
| 5159 | for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5160 | ++Idx) { |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5161 | unsigned ByteOffset = Offset / 8; |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 5162 | Register NewAddrReg; |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5163 | |
Daniel Sanders | e74c5b9 | 2019-11-01 13:18:00 -0700 | [diff] [blame] | 5164 | MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5165 | |
| 5166 | MachineMemOperand *NewMMO = |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5167 | MF.getMachineMemOperand(&MMO, ByteOffset, PartTy); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5168 | |
| 5169 | if (IsLoad) { |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 5170 | Register Dst = MRI.createGenericVirtualRegister(PartTy); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5171 | ValRegs.push_back(Dst); |
| 5172 | MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); |
| 5173 | } else { |
| 5174 | MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); |
| 5175 | } |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5176 | Offset = isBigEndian ? Offset - PartSize : Offset + PartSize; |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5177 | } |
| 5178 | |
| 5179 | return Offset; |
| 5180 | }; |
| 5181 | |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5182 | unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0; |
| 5183 | unsigned HandledOffset = |
| 5184 | splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5185 | |
| 5186 | // Handle the rest of the register if this isn't an even type breakdown. |
| 5187 | if (LeftoverTy.isValid()) |
Sheng | 146c782 | 2022-02-07 19:04:27 -0500 | [diff] [blame] | 5188 | splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset); |
Matt Arsenault | c7bce73 | 2019-01-31 02:46:05 +0000 | [diff] [blame] | 5189 | |
| 5190 | if (IsLoad) { |
| 5191 | insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, |
| 5192 | LeftoverTy, NarrowLeftoverRegs); |
| 5193 | } |
| 5194 | |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5195 | LdStMI.eraseFromParent(); |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5196 | return Legalized; |
| 5197 | } |
| 5198 | |
| 5199 | LegalizerHelper::LegalizeResult |
Tim Northover | 69fa84a | 2016-10-14 22:18:18 +0000 | [diff] [blame] | 5200 | LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, |
| 5201 | LLT NarrowTy) { |
Matt Arsenault | 1b1e685 | 2019-01-25 02:59:34 +0000 | [diff] [blame] | 5202 | using namespace TargetOpcode; |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5203 | GenericMachineInstr &GMI = cast<GenericMachineInstr>(MI); |
| 5204 | unsigned NumElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; |
Volkan Keles | 574d737 | 2018-12-14 22:11:20 +0000 | [diff] [blame] | 5205 | |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5206 | switch (MI.getOpcode()) { |
| 5207 | case G_IMPLICIT_DEF: |
Matt Arsenault | ce8a1f7 | 2020-02-15 20:24:36 -0500 | [diff] [blame] | 5208 | case G_TRUNC: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5209 | case G_AND: |
| 5210 | case G_OR: |
| 5211 | case G_XOR: |
| 5212 | case G_ADD: |
| 5213 | case G_SUB: |
| 5214 | case G_MUL: |
Matt Arsenault | 3e8bb7a | 2020-07-25 10:47:33 -0400 | [diff] [blame] | 5215 | case G_PTR_ADD: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5216 | case G_SMULH: |
| 5217 | case G_UMULH: |
| 5218 | case G_FADD: |
| 5219 | case G_FMUL: |
| 5220 | case G_FSUB: |
| 5221 | case G_FNEG: |
| 5222 | case G_FABS: |
Matt Arsenault | 9dba67f | 2019-02-11 17:05:20 +0000 | [diff] [blame] | 5223 | case G_FCANONICALIZE: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5224 | case G_FDIV: |
| 5225 | case G_FREM: |
| 5226 | case G_FMA: |
Matt Arsenault | cf10372 | 2019-09-06 20:49:10 +0000 | [diff] [blame] | 5227 | case G_FMAD: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5228 | case G_FPOW: |
| 5229 | case G_FEXP: |
| 5230 | case G_FEXP2: |
Matt Arsenault | b14e83d | 2023-08-12 07:20:00 -0400 | [diff] [blame] | 5231 | case G_FEXP10: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5232 | case G_FLOG: |
| 5233 | case G_FLOG2: |
| 5234 | case G_FLOG10: |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 5235 | case G_FLDEXP: |
Jessica Paquette | ba55767 | 2019-04-25 16:44:40 +0000 | [diff] [blame] | 5236 | case G_FNEARBYINT: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5237 | case G_FCEIL: |
Jessica Paquette | ebdb021 | 2019-02-11 17:22:58 +0000 | [diff] [blame] | 5238 | case G_FFLOOR: |
Jessica Paquette | d5c69e0 | 2019-04-19 23:41:52 +0000 | [diff] [blame] | 5239 | case G_FRINT: |
Sumanth Gundapaneni | 0ee32c4 | 2024-07-24 14:34:31 -0500 | [diff] [blame] | 5240 | case G_INTRINSIC_LRINT: |
| 5241 | case G_INTRINSIC_LLRINT: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5242 | case G_INTRINSIC_ROUND: |
Matt Arsenault | 0da582d | 2020-07-19 09:56:15 -0400 | [diff] [blame] | 5243 | case G_INTRINSIC_ROUNDEVEN: |
Sumanth Gundapaneni | e78156a | 2024-08-21 12:13:56 -0500 | [diff] [blame] | 5244 | case G_LROUND: |
| 5245 | case G_LLROUND: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5246 | case G_INTRINSIC_TRUNC: |
Jessica Paquette | 7db82d7 | 2019-01-28 18:34:18 +0000 | [diff] [blame] | 5247 | case G_FCOS: |
| 5248 | case G_FSIN: |
Farzon Lotfi | 1d87433 | 2024-06-05 15:01:33 -0400 | [diff] [blame] | 5249 | case G_FTAN: |
Farzon Lotfi | e2f463b | 2024-07-19 10:18:23 -0400 | [diff] [blame] | 5250 | case G_FACOS: |
| 5251 | case G_FASIN: |
| 5252 | case G_FATAN: |
Tex Riddell | c03d09c | 2024-10-24 17:53:12 -0700 | [diff] [blame] | 5253 | case G_FATAN2: |
Farzon Lotfi | e2f463b | 2024-07-19 10:18:23 -0400 | [diff] [blame] | 5254 | case G_FCOSH: |
| 5255 | case G_FSINH: |
| 5256 | case G_FTANH: |
Jessica Paquette | 22457f8 | 2019-01-30 21:03:52 +0000 | [diff] [blame] | 5257 | case G_FSQRT: |
Matt Arsenault | d1bfc8d | 2019-01-31 02:34:03 +0000 | [diff] [blame] | 5258 | case G_BSWAP: |
Matt Arsenault | 5ff310e | 2019-09-04 20:46:15 +0000 | [diff] [blame] | 5259 | case G_BITREVERSE: |
Amara Emerson | ae878da | 2019-04-10 23:06:08 +0000 | [diff] [blame] | 5260 | case G_SDIV: |
Matt Arsenault | d12f2a2 | 2020-01-04 13:24:09 -0500 | [diff] [blame] | 5261 | case G_UDIV: |
| 5262 | case G_SREM: |
| 5263 | case G_UREM: |
Christudasan Devadasan | 90d7840 | 2021-04-12 15:49:47 +0530 | [diff] [blame] | 5264 | case G_SDIVREM: |
| 5265 | case G_UDIVREM: |
Matt Arsenault | 0f3ba44 | 2019-05-23 17:58:48 +0000 | [diff] [blame] | 5266 | case G_SMIN: |
| 5267 | case G_SMAX: |
| 5268 | case G_UMIN: |
| 5269 | case G_UMAX: |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 5270 | case G_ABS: |
Matt Arsenault | 6ce1b4f | 2019-07-10 16:31:19 +0000 | [diff] [blame] | 5271 | case G_FMINNUM: |
| 5272 | case G_FMAXNUM: |
| 5273 | case G_FMINNUM_IEEE: |
| 5274 | case G_FMAXNUM_IEEE: |
| 5275 | case G_FMINIMUM: |
| 5276 | case G_FMAXIMUM: |
Matt Arsenault | 4919f2e | 2020-03-19 21:25:27 -0400 | [diff] [blame] | 5277 | case G_FSHL: |
| 5278 | case G_FSHR: |
Mirko Brkusanin | 5263bf5 | 2021-09-07 16:18:19 +0200 | [diff] [blame] | 5279 | case G_ROTL: |
| 5280 | case G_ROTR: |
Dominik Montada | 55e3a7c | 2020-04-14 11:25:05 +0200 | [diff] [blame] | 5281 | case G_FREEZE: |
Matt Arsenault | 23ec773 | 2020-07-12 16:11:53 -0400 | [diff] [blame] | 5282 | case G_SADDSAT: |
| 5283 | case G_SSUBSAT: |
| 5284 | case G_UADDSAT: |
| 5285 | case G_USUBSAT: |
Pushpinder Singh | d0e5422 | 2021-03-09 06:10:00 +0000 | [diff] [blame] | 5286 | case G_UMULO: |
| 5287 | case G_SMULO: |
Matt Arsenault | c83b823 | 2019-02-07 17:38:00 +0000 | [diff] [blame] | 5288 | case G_SHL: |
| 5289 | case G_LSHR: |
| 5290 | case G_ASHR: |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 5291 | case G_SSHLSAT: |
| 5292 | case G_USHLSAT: |
Matt Arsenault | 75e30c4 | 2019-02-20 16:42:52 +0000 | [diff] [blame] | 5293 | case G_CTLZ: |
| 5294 | case G_CTLZ_ZERO_UNDEF: |
| 5295 | case G_CTTZ: |
| 5296 | case G_CTTZ_ZERO_UNDEF: |
| 5297 | case G_CTPOP: |
Matt Arsenault | 1448f56 | 2019-05-17 12:19:52 +0000 | [diff] [blame] | 5298 | case G_FCOPYSIGN: |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5299 | case G_ZEXT: |
| 5300 | case G_SEXT: |
| 5301 | case G_ANYEXT: |
| 5302 | case G_FPEXT: |
| 5303 | case G_FPTRUNC: |
| 5304 | case G_SITOFP: |
| 5305 | case G_UITOFP: |
| 5306 | case G_FPTOSI: |
| 5307 | case G_FPTOUI: |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 5308 | case G_FPTOSI_SAT: |
| 5309 | case G_FPTOUI_SAT: |
Matt Arsenault | cbaada6 | 2019-02-02 23:29:55 +0000 | [diff] [blame] | 5310 | case G_INTTOPTR: |
| 5311 | case G_PTRTOINT: |
Matt Arsenault | a8b4339 | 2019-02-08 02:40:47 +0000 | [diff] [blame] | 5312 | case G_ADDRSPACE_CAST: |
Abinav Puthan Purayil | 898d577 | 2022-03-31 16:33:28 +0530 | [diff] [blame] | 5313 | case G_UADDO: |
| 5314 | case G_USUBO: |
| 5315 | case G_UADDE: |
| 5316 | case G_USUBE: |
| 5317 | case G_SADDO: |
| 5318 | case G_SSUBO: |
| 5319 | case G_SADDE: |
| 5320 | case G_SSUBE: |
Matt Arsenault | fe5b9a6 | 2020-05-31 13:23:20 -0400 | [diff] [blame] | 5321 | case G_STRICT_FADD: |
Matt Arsenault | 1fe1299 | 2022-11-17 23:03:23 -0800 | [diff] [blame] | 5322 | case G_STRICT_FSUB: |
Matt Arsenault | fe5b9a6 | 2020-05-31 13:23:20 -0400 | [diff] [blame] | 5323 | case G_STRICT_FMUL: |
| 5324 | case G_STRICT_FMA: |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 5325 | case G_STRICT_FLDEXP: |
Matt Arsenault | 003b58f | 2023-04-26 21:57:10 -0400 | [diff] [blame] | 5326 | case G_FFREXP: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5327 | return fewerElementsVectorMultiEltType(GMI, NumElts); |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5328 | case G_ICMP: |
| 5329 | case G_FCMP: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5330 | return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*cpm predicate*/}); |
Janek van Oirschot | 322966f | 2022-11-28 15:40:31 -0500 | [diff] [blame] | 5331 | case G_IS_FPCLASS: |
| 5332 | return fewerElementsVectorMultiEltType(GMI, NumElts, {2, 3 /*mask,fpsem*/}); |
Matt Arsenault | dc6c785 | 2019-01-30 04:19:31 +0000 | [diff] [blame] | 5333 | case G_SELECT: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5334 | if (MRI.getType(MI.getOperand(1).getReg()).isVector()) |
| 5335 | return fewerElementsVectorMultiEltType(GMI, NumElts); |
| 5336 | return fewerElementsVectorMultiEltType(GMI, NumElts, {1 /*scalar cond*/}); |
Matt Arsenault | d3093c2 | 2019-02-28 00:16:32 +0000 | [diff] [blame] | 5337 | case G_PHI: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5338 | return fewerElementsVectorPhi(GMI, NumElts); |
Matt Arsenault | 28215ca | 2019-08-13 16:26:28 +0000 | [diff] [blame] | 5339 | case G_UNMERGE_VALUES: |
| 5340 | return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy); |
Matt Arsenault | 3cd3959 | 2019-10-09 22:44:43 +0000 | [diff] [blame] | 5341 | case G_BUILD_VECTOR: |
Matt Arsenault | 901e331 | 2020-08-03 18:37:29 -0400 | [diff] [blame] | 5342 | assert(TypeIdx == 0 && "not a vector type index"); |
| 5343 | return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); |
Matt Arsenault | 31adc28 | 2020-08-03 14:13:38 -0400 | [diff] [blame] | 5344 | case G_CONCAT_VECTORS: |
Matt Arsenault | 901e331 | 2020-08-03 18:37:29 -0400 | [diff] [blame] | 5345 | if (TypeIdx != 1) // TODO: This probably does work as expected already. |
| 5346 | return UnableToLegalize; |
| 5347 | return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy); |
Matt Arsenault | e002015 | 2020-07-27 09:58:17 -0400 | [diff] [blame] | 5348 | case G_EXTRACT_VECTOR_ELT: |
Matt Arsenault | 5a15f66 | 2020-07-27 22:00:50 -0400 | [diff] [blame] | 5349 | case G_INSERT_VECTOR_ELT: |
| 5350 | return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy); |
Matt Arsenault | 816c9b3e | 2019-01-27 21:53:09 +0000 | [diff] [blame] | 5351 | case G_LOAD: |
| 5352 | case G_STORE: |
Amara Emerson | 4e3dc6b | 2021-07-09 15:48:47 -0700 | [diff] [blame] | 5353 | return reduceLoadStoreWidth(cast<GLoadStore>(MI), TypeIdx, NarrowTy); |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 5354 | case G_SEXT_INREG: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5355 | return fewerElementsVectorMultiEltType(GMI, NumElts, {2 /*imm*/}); |
Amara Emerson | a35c2c7 | 2021-02-21 14:17:03 -0800 | [diff] [blame] | 5356 | GISEL_VECREDUCE_CASES_NONSEQ |
| 5357 | return fewerElementsVectorReductions(MI, TypeIdx, NarrowTy); |
David Green | 77b124c | 2024-01-05 08:11:44 +0000 | [diff] [blame] | 5358 | case TargetOpcode::G_VECREDUCE_SEQ_FADD: |
| 5359 | case TargetOpcode::G_VECREDUCE_SEQ_FMUL: |
| 5360 | return fewerElementsVectorSeqReductions(MI, TypeIdx, NarrowTy); |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5361 | case G_SHUFFLE_VECTOR: |
| 5362 | return fewerElementsVectorShuffle(MI, TypeIdx, NarrowTy); |
David Green | 5550e9c | 2024-01-04 07:26:23 +0000 | [diff] [blame] | 5363 | case G_FPOWI: |
| 5364 | return fewerElementsVectorMultiEltType(GMI, NumElts, {2 /*pow*/}); |
chuongg3 | 0fb3d42 | 2024-02-21 13:24:45 +0000 | [diff] [blame] | 5365 | case G_BITCAST: |
| 5366 | return fewerElementsBitcast(MI, TypeIdx, NarrowTy); |
Matt Arsenault | 401658c | 2024-04-24 12:25:02 +0200 | [diff] [blame] | 5367 | case G_INTRINSIC_FPTRUNC_ROUND: |
| 5368 | return fewerElementsVectorMultiEltType(GMI, NumElts, {2}); |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 5369 | default: |
| 5370 | return UnableToLegalize; |
Tim Northover | 33b07d6 | 2016-07-22 20:03:43 +0000 | [diff] [blame] | 5371 | } |
| 5372 | } |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 5373 | |
chuongg3 | 0fb3d42 | 2024-02-21 13:24:45 +0000 | [diff] [blame] | 5374 | LegalizerHelper::LegalizeResult |
| 5375 | LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx, |
| 5376 | LLT NarrowTy) { |
| 5377 | assert(MI.getOpcode() == TargetOpcode::G_BITCAST && |
| 5378 | "Not a bitcast operation"); |
| 5379 | |
| 5380 | if (TypeIdx != 0) |
| 5381 | return UnableToLegalize; |
| 5382 | |
| 5383 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
| 5384 | |
Tim Gymnich | 2db2dc8a | 2024-12-12 18:47:46 +0100 | [diff] [blame] | 5385 | unsigned NewElemCount = |
| 5386 | NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits(); |
| 5387 | LLT SrcNarrowTy = LLT::fixed_vector(NewElemCount, SrcTy.getElementType()); |
chuongg3 | 0fb3d42 | 2024-02-21 13:24:45 +0000 | [diff] [blame] | 5388 | |
| 5389 | // Split the Src and Dst Reg into smaller registers |
| 5390 | SmallVector<Register> SrcVRegs, BitcastVRegs; |
| 5391 | if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy) |
| 5392 | return UnableToLegalize; |
| 5393 | |
| 5394 | // Build new smaller bitcast instructions |
| 5395 | // Not supporting Leftover types for now but will have to |
| 5396 | for (unsigned i = 0; i < SrcVRegs.size(); i++) |
| 5397 | BitcastVRegs.push_back( |
| 5398 | MIRBuilder.buildBitcast(NarrowTy, SrcVRegs[i]).getReg(0)); |
| 5399 | |
| 5400 | MIRBuilder.buildMergeLikeInstr(DstReg, BitcastVRegs); |
| 5401 | MI.eraseFromParent(); |
| 5402 | return Legalized; |
| 5403 | } |
| 5404 | |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5405 | LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle( |
| 5406 | MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { |
| 5407 | assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); |
| 5408 | if (TypeIdx != 0) |
| 5409 | return UnableToLegalize; |
| 5410 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 5411 | auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] = |
| 5412 | MI.getFirst3RegLLTs(); |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5413 | ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5414 | // The shuffle should be canonicalized by now. |
| 5415 | if (DstTy != Src1Ty) |
| 5416 | return UnableToLegalize; |
| 5417 | if (DstTy != Src2Ty) |
| 5418 | return UnableToLegalize; |
| 5419 | |
| 5420 | if (!isPowerOf2_32(DstTy.getNumElements())) |
| 5421 | return UnableToLegalize; |
| 5422 | |
| 5423 | // We only support splitting a shuffle into 2, so adjust NarrowTy accordingly. |
| 5424 | // Further legalization attempts will be needed to do split further. |
Sander de Smalen | c9acd2f | 2021-06-25 11:27:41 +0100 | [diff] [blame] | 5425 | NarrowTy = |
| 5426 | DstTy.changeElementCount(DstTy.getElementCount().divideCoefficientBy(2)); |
David Green | 4c8c130 | 2024-12-15 10:44:40 +0000 | [diff] [blame] | 5427 | unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5428 | |
| 5429 | SmallVector<Register> SplitSrc1Regs, SplitSrc2Regs; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 5430 | extractParts(Src1Reg, NarrowTy, 2, SplitSrc1Regs, MIRBuilder, MRI); |
| 5431 | extractParts(Src2Reg, NarrowTy, 2, SplitSrc2Regs, MIRBuilder, MRI); |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5432 | Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0], |
| 5433 | SplitSrc2Regs[1]}; |
| 5434 | |
| 5435 | Register Hi, Lo; |
| 5436 | |
| 5437 | // If Lo or Hi uses elements from at most two of the four input vectors, then |
| 5438 | // express it as a vector shuffle of those two inputs. Otherwise extract the |
| 5439 | // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. |
| 5440 | SmallVector<int, 16> Ops; |
| 5441 | for (unsigned High = 0; High < 2; ++High) { |
| 5442 | Register &Output = High ? Hi : Lo; |
| 5443 | |
| 5444 | // Build a shuffle mask for the output, discovering on the fly which |
| 5445 | // input vectors to use as shuffle operands (recorded in InputUsed). |
| 5446 | // If building a suitable shuffle vector proves too hard, then bail |
| 5447 | // out with useBuildVector set. |
| 5448 | unsigned InputUsed[2] = {-1U, -1U}; // Not yet discovered. |
| 5449 | unsigned FirstMaskIdx = High * NewElts; |
| 5450 | bool UseBuildVector = false; |
| 5451 | for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { |
| 5452 | // The mask element. This indexes into the input. |
| 5453 | int Idx = Mask[FirstMaskIdx + MaskOffset]; |
| 5454 | |
| 5455 | // The input vector this mask element indexes into. |
| 5456 | unsigned Input = (unsigned)Idx / NewElts; |
| 5457 | |
Joe Loser | 5e96cea | 2022-09-06 18:06:58 -0600 | [diff] [blame] | 5458 | if (Input >= std::size(Inputs)) { |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5459 | // The mask element does not index into any input vector. |
| 5460 | Ops.push_back(-1); |
| 5461 | continue; |
| 5462 | } |
| 5463 | |
| 5464 | // Turn the index into an offset from the start of the input vector. |
| 5465 | Idx -= Input * NewElts; |
| 5466 | |
| 5467 | // Find or create a shuffle vector operand to hold this input. |
| 5468 | unsigned OpNo; |
Joe Loser | 5e96cea | 2022-09-06 18:06:58 -0600 | [diff] [blame] | 5469 | for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) { |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5470 | if (InputUsed[OpNo] == Input) { |
| 5471 | // This input vector is already an operand. |
| 5472 | break; |
| 5473 | } else if (InputUsed[OpNo] == -1U) { |
| 5474 | // Create a new operand for this input vector. |
| 5475 | InputUsed[OpNo] = Input; |
| 5476 | break; |
| 5477 | } |
| 5478 | } |
| 5479 | |
Joe Loser | 5e96cea | 2022-09-06 18:06:58 -0600 | [diff] [blame] | 5480 | if (OpNo >= std::size(InputUsed)) { |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5481 | // More than two input vectors used! Give up on trying to create a |
| 5482 | // shuffle vector. Insert all elements into a BUILD_VECTOR instead. |
| 5483 | UseBuildVector = true; |
| 5484 | break; |
| 5485 | } |
| 5486 | |
| 5487 | // Add the mask index for the new shuffle vector. |
| 5488 | Ops.push_back(Idx + OpNo * NewElts); |
| 5489 | } |
| 5490 | |
| 5491 | if (UseBuildVector) { |
| 5492 | LLT EltTy = NarrowTy.getElementType(); |
| 5493 | SmallVector<Register, 16> SVOps; |
| 5494 | |
| 5495 | // Extract the input elements by hand. |
| 5496 | for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { |
| 5497 | // The mask element. This indexes into the input. |
| 5498 | int Idx = Mask[FirstMaskIdx + MaskOffset]; |
| 5499 | |
| 5500 | // The input vector this mask element indexes into. |
| 5501 | unsigned Input = (unsigned)Idx / NewElts; |
| 5502 | |
Joe Loser | 5e96cea | 2022-09-06 18:06:58 -0600 | [diff] [blame] | 5503 | if (Input >= std::size(Inputs)) { |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5504 | // The mask element is "undef" or indexes off the end of the input. |
| 5505 | SVOps.push_back(MIRBuilder.buildUndef(EltTy).getReg(0)); |
| 5506 | continue; |
| 5507 | } |
| 5508 | |
| 5509 | // Turn the index into an offset from the start of the input vector. |
| 5510 | Idx -= Input * NewElts; |
| 5511 | |
| 5512 | // Extract the vector element by hand. |
| 5513 | SVOps.push_back(MIRBuilder |
| 5514 | .buildExtractVectorElement( |
| 5515 | EltTy, Inputs[Input], |
| 5516 | MIRBuilder.buildConstant(LLT::scalar(32), Idx)) |
| 5517 | .getReg(0)); |
| 5518 | } |
| 5519 | |
| 5520 | // Construct the Lo/Hi output using a G_BUILD_VECTOR. |
| 5521 | Output = MIRBuilder.buildBuildVector(NarrowTy, SVOps).getReg(0); |
| 5522 | } else if (InputUsed[0] == -1U) { |
| 5523 | // No input vectors were used! The result is undefined. |
| 5524 | Output = MIRBuilder.buildUndef(NarrowTy).getReg(0); |
| 5525 | } else { |
| 5526 | Register Op0 = Inputs[InputUsed[0]]; |
| 5527 | // If only one input was used, use an undefined vector for the other. |
| 5528 | Register Op1 = InputUsed[1] == -1U |
| 5529 | ? MIRBuilder.buildUndef(NarrowTy).getReg(0) |
| 5530 | : Inputs[InputUsed[1]]; |
| 5531 | // At least one input vector was used. Create a new shuffle vector. |
| 5532 | Output = MIRBuilder.buildShuffleVector(NarrowTy, Op0, Op1, Ops).getReg(0); |
| 5533 | } |
| 5534 | |
| 5535 | Ops.clear(); |
| 5536 | } |
| 5537 | |
David Green | 4c8c130 | 2024-12-15 10:44:40 +0000 | [diff] [blame] | 5538 | MIRBuilder.buildMergeLikeInstr(DstReg, {Lo, Hi}); |
Amara Emerson | 9f39ba1 | 2021-05-19 21:35:05 -0700 | [diff] [blame] | 5539 | MI.eraseFromParent(); |
| 5540 | return Legalized; |
| 5541 | } |
| 5542 | |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5543 | LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions( |
| 5544 | MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { |
Amara Emerson | b966978 | 2023-08-12 13:55:08 -0700 | [diff] [blame] | 5545 | auto &RdxMI = cast<GVecReduce>(MI); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5546 | |
| 5547 | if (TypeIdx != 1) |
| 5548 | return UnableToLegalize; |
| 5549 | |
| 5550 | // The semantics of the normal non-sequential reductions allow us to freely |
| 5551 | // re-associate the operation. |
Amara Emerson | b966978 | 2023-08-12 13:55:08 -0700 | [diff] [blame] | 5552 | auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs(); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5553 | |
| 5554 | if (NarrowTy.isVector() && |
| 5555 | (SrcTy.getNumElements() % NarrowTy.getNumElements() != 0)) |
| 5556 | return UnableToLegalize; |
| 5557 | |
Amara Emerson | b966978 | 2023-08-12 13:55:08 -0700 | [diff] [blame] | 5558 | unsigned ScalarOpc = RdxMI.getScalarOpcForReduction(); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5559 | SmallVector<Register> SplitSrcs; |
| 5560 | // If NarrowTy is a scalar then we're being asked to scalarize. |
| 5561 | const unsigned NumParts = |
| 5562 | NarrowTy.isVector() ? SrcTy.getNumElements() / NarrowTy.getNumElements() |
| 5563 | : SrcTy.getNumElements(); |
| 5564 | |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 5565 | extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5566 | if (NarrowTy.isScalar()) { |
| 5567 | if (DstTy != NarrowTy) |
| 5568 | return UnableToLegalize; // FIXME: handle implicit extensions. |
| 5569 | |
| 5570 | if (isPowerOf2_32(NumParts)) { |
| 5571 | // Generate a tree of scalar operations to reduce the critical path. |
| 5572 | SmallVector<Register> PartialResults; |
| 5573 | unsigned NumPartsLeft = NumParts; |
| 5574 | while (NumPartsLeft > 1) { |
| 5575 | for (unsigned Idx = 0; Idx < NumPartsLeft - 1; Idx += 2) { |
| 5576 | PartialResults.emplace_back( |
| 5577 | MIRBuilder |
| 5578 | .buildInstr(ScalarOpc, {NarrowTy}, |
| 5579 | {SplitSrcs[Idx], SplitSrcs[Idx + 1]}) |
| 5580 | .getReg(0)); |
| 5581 | } |
| 5582 | SplitSrcs = PartialResults; |
| 5583 | PartialResults.clear(); |
| 5584 | NumPartsLeft = SplitSrcs.size(); |
| 5585 | } |
| 5586 | assert(SplitSrcs.size() == 1); |
| 5587 | MIRBuilder.buildCopy(DstReg, SplitSrcs[0]); |
| 5588 | MI.eraseFromParent(); |
| 5589 | return Legalized; |
| 5590 | } |
| 5591 | // If we can't generate a tree, then just do sequential operations. |
| 5592 | Register Acc = SplitSrcs[0]; |
| 5593 | for (unsigned Idx = 1; Idx < NumParts; ++Idx) |
| 5594 | Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[Idx]}) |
| 5595 | .getReg(0); |
| 5596 | MIRBuilder.buildCopy(DstReg, Acc); |
| 5597 | MI.eraseFromParent(); |
| 5598 | return Legalized; |
| 5599 | } |
| 5600 | SmallVector<Register> PartialReductions; |
| 5601 | for (unsigned Part = 0; Part < NumParts; ++Part) { |
| 5602 | PartialReductions.push_back( |
Amara Emerson | b966978 | 2023-08-12 13:55:08 -0700 | [diff] [blame] | 5603 | MIRBuilder.buildInstr(RdxMI.getOpcode(), {DstTy}, {SplitSrcs[Part]}) |
| 5604 | .getReg(0)); |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 5605 | } |
| 5606 | |
Amara Emerson | a35c2c7 | 2021-02-21 14:17:03 -0800 | [diff] [blame] | 5607 | // If the types involved are powers of 2, we can generate intermediate vector |
| 5608 | // ops, before generating a final reduction operation. |
| 5609 | if (isPowerOf2_32(SrcTy.getNumElements()) && |
| 5610 | isPowerOf2_32(NarrowTy.getNumElements())) { |
| 5611 | return tryNarrowPow2Reduction(MI, SrcReg, SrcTy, NarrowTy, ScalarOpc); |
| 5612 | } |
| 5613 | |
| 5614 | Register Acc = PartialReductions[0]; |
| 5615 | for (unsigned Part = 1; Part < NumParts; ++Part) { |
| 5616 | if (Part == NumParts - 1) { |
| 5617 | MIRBuilder.buildInstr(ScalarOpc, {DstReg}, |
| 5618 | {Acc, PartialReductions[Part]}); |
| 5619 | } else { |
| 5620 | Acc = MIRBuilder |
| 5621 | .buildInstr(ScalarOpc, {DstTy}, {Acc, PartialReductions[Part]}) |
| 5622 | .getReg(0); |
| 5623 | } |
| 5624 | } |
| 5625 | MI.eraseFromParent(); |
| 5626 | return Legalized; |
| 5627 | } |
| 5628 | |
| 5629 | LegalizerHelper::LegalizeResult |
David Green | 77b124c | 2024-01-05 08:11:44 +0000 | [diff] [blame] | 5630 | LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI, |
| 5631 | unsigned int TypeIdx, |
| 5632 | LLT NarrowTy) { |
| 5633 | auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] = |
| 5634 | MI.getFirst3RegLLTs(); |
| 5635 | if (!NarrowTy.isScalar() || TypeIdx != 2 || DstTy != ScalarTy || |
| 5636 | DstTy != NarrowTy) |
| 5637 | return UnableToLegalize; |
| 5638 | |
| 5639 | assert((MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD || |
| 5640 | MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) && |
| 5641 | "Unexpected vecreduce opcode"); |
| 5642 | unsigned ScalarOpc = MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD |
| 5643 | ? TargetOpcode::G_FADD |
| 5644 | : TargetOpcode::G_FMUL; |
| 5645 | |
| 5646 | SmallVector<Register> SplitSrcs; |
| 5647 | unsigned NumParts = SrcTy.getNumElements(); |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 5648 | extractParts(SrcReg, NarrowTy, NumParts, SplitSrcs, MIRBuilder, MRI); |
David Green | 77b124c | 2024-01-05 08:11:44 +0000 | [diff] [blame] | 5649 | Register Acc = ScalarReg; |
| 5650 | for (unsigned i = 0; i < NumParts; i++) |
| 5651 | Acc = MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {Acc, SplitSrcs[i]}) |
| 5652 | .getReg(0); |
| 5653 | |
| 5654 | MIRBuilder.buildCopy(DstReg, Acc); |
| 5655 | MI.eraseFromParent(); |
| 5656 | return Legalized; |
| 5657 | } |
| 5658 | |
| 5659 | LegalizerHelper::LegalizeResult |
Amara Emerson | a35c2c7 | 2021-02-21 14:17:03 -0800 | [diff] [blame] | 5660 | LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, |
| 5661 | LLT SrcTy, LLT NarrowTy, |
| 5662 | unsigned ScalarOpc) { |
| 5663 | SmallVector<Register> SplitSrcs; |
| 5664 | // Split the sources into NarrowTy size pieces. |
| 5665 | extractParts(SrcReg, NarrowTy, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 5666 | SrcTy.getNumElements() / NarrowTy.getNumElements(), SplitSrcs, |
| 5667 | MIRBuilder, MRI); |
Amara Emerson | a35c2c7 | 2021-02-21 14:17:03 -0800 | [diff] [blame] | 5668 | // We're going to do a tree reduction using vector operations until we have |
| 5669 | // one NarrowTy size value left. |
| 5670 | while (SplitSrcs.size() > 1) { |
| 5671 | SmallVector<Register> PartialRdxs; |
| 5672 | for (unsigned Idx = 0; Idx < SplitSrcs.size()-1; Idx += 2) { |
| 5673 | Register LHS = SplitSrcs[Idx]; |
| 5674 | Register RHS = SplitSrcs[Idx + 1]; |
| 5675 | // Create the intermediate vector op. |
| 5676 | Register Res = |
| 5677 | MIRBuilder.buildInstr(ScalarOpc, {NarrowTy}, {LHS, RHS}).getReg(0); |
| 5678 | PartialRdxs.push_back(Res); |
| 5679 | } |
| 5680 | SplitSrcs = std::move(PartialRdxs); |
| 5681 | } |
| 5682 | // Finally generate the requested NarrowTy based reduction. |
| 5683 | Observer.changingInstr(MI); |
| 5684 | MI.getOperand(1).setReg(SplitSrcs[0]); |
| 5685 | Observer.changedInstr(MI); |
| 5686 | return Legalized; |
| 5687 | } |
| 5688 | |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 5689 | LegalizerHelper::LegalizeResult |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5690 | LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, |
| 5691 | const LLT HalfTy, const LLT AmtTy) { |
| 5692 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 5693 | Register InL = MRI.createGenericVirtualRegister(HalfTy); |
| 5694 | Register InH = MRI.createGenericVirtualRegister(HalfTy); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 5695 | MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5696 | |
Jay Foad | a9bceb2 | 2021-09-30 09:54:57 +0100 | [diff] [blame] | 5697 | if (Amt.isZero()) { |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 5698 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), {InL, InH}); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5699 | MI.eraseFromParent(); |
| 5700 | return Legalized; |
| 5701 | } |
| 5702 | |
| 5703 | LLT NVT = HalfTy; |
| 5704 | unsigned NVTBits = HalfTy.getSizeInBits(); |
| 5705 | unsigned VTBits = 2 * NVTBits; |
| 5706 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 5707 | SrcOp Lo(Register(0)), Hi(Register(0)); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5708 | if (MI.getOpcode() == TargetOpcode::G_SHL) { |
| 5709 | if (Amt.ugt(VTBits)) { |
| 5710 | Lo = Hi = MIRBuilder.buildConstant(NVT, 0); |
| 5711 | } else if (Amt.ugt(NVTBits)) { |
| 5712 | Lo = MIRBuilder.buildConstant(NVT, 0); |
| 5713 | Hi = MIRBuilder.buildShl(NVT, InL, |
| 5714 | MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); |
| 5715 | } else if (Amt == NVTBits) { |
| 5716 | Lo = MIRBuilder.buildConstant(NVT, 0); |
| 5717 | Hi = InL; |
| 5718 | } else { |
| 5719 | Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); |
Matt Arsenault | e98cab1 | 2019-02-07 20:44:08 +0000 | [diff] [blame] | 5720 | auto OrLHS = |
| 5721 | MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); |
| 5722 | auto OrRHS = MIRBuilder.buildLShr( |
| 5723 | NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); |
| 5724 | Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5725 | } |
| 5726 | } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { |
| 5727 | if (Amt.ugt(VTBits)) { |
| 5728 | Lo = Hi = MIRBuilder.buildConstant(NVT, 0); |
| 5729 | } else if (Amt.ugt(NVTBits)) { |
| 5730 | Lo = MIRBuilder.buildLShr(NVT, InH, |
| 5731 | MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); |
| 5732 | Hi = MIRBuilder.buildConstant(NVT, 0); |
| 5733 | } else if (Amt == NVTBits) { |
| 5734 | Lo = InH; |
| 5735 | Hi = MIRBuilder.buildConstant(NVT, 0); |
| 5736 | } else { |
| 5737 | auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); |
| 5738 | |
| 5739 | auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); |
| 5740 | auto OrRHS = MIRBuilder.buildShl( |
| 5741 | NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); |
| 5742 | |
| 5743 | Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); |
| 5744 | Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); |
| 5745 | } |
| 5746 | } else { |
| 5747 | if (Amt.ugt(VTBits)) { |
| 5748 | Hi = Lo = MIRBuilder.buildAShr( |
| 5749 | NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); |
| 5750 | } else if (Amt.ugt(NVTBits)) { |
| 5751 | Lo = MIRBuilder.buildAShr(NVT, InH, |
| 5752 | MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); |
| 5753 | Hi = MIRBuilder.buildAShr(NVT, InH, |
| 5754 | MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); |
| 5755 | } else if (Amt == NVTBits) { |
| 5756 | Lo = InH; |
| 5757 | Hi = MIRBuilder.buildAShr(NVT, InH, |
| 5758 | MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); |
| 5759 | } else { |
| 5760 | auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); |
| 5761 | |
| 5762 | auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); |
| 5763 | auto OrRHS = MIRBuilder.buildShl( |
| 5764 | NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); |
| 5765 | |
| 5766 | Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); |
| 5767 | Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); |
| 5768 | } |
| 5769 | } |
| 5770 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 5771 | MIRBuilder.buildMergeLikeInstr(MI.getOperand(0), {Lo, Hi}); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5772 | MI.eraseFromParent(); |
| 5773 | |
| 5774 | return Legalized; |
| 5775 | } |
| 5776 | |
| 5777 | // TODO: Optimize if constant shift amount. |
| 5778 | LegalizerHelper::LegalizeResult |
| 5779 | LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, |
| 5780 | LLT RequestedTy) { |
| 5781 | if (TypeIdx == 1) { |
| 5782 | Observer.changingInstr(MI); |
| 5783 | narrowScalarSrc(MI, RequestedTy, 2); |
| 5784 | Observer.changedInstr(MI); |
| 5785 | return Legalized; |
| 5786 | } |
| 5787 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 5788 | Register DstReg = MI.getOperand(0).getReg(); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5789 | LLT DstTy = MRI.getType(DstReg); |
| 5790 | if (DstTy.isVector()) |
| 5791 | return UnableToLegalize; |
| 5792 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 5793 | Register Amt = MI.getOperand(2).getReg(); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5794 | LLT ShiftAmtTy = MRI.getType(Amt); |
| 5795 | const unsigned DstEltSize = DstTy.getScalarSizeInBits(); |
| 5796 | if (DstEltSize % 2 != 0) |
| 5797 | return UnableToLegalize; |
| 5798 | |
| 5799 | // Ignore the input type. We can only go to exactly half the size of the |
| 5800 | // input. If that isn't small enough, the resulting pieces will be further |
| 5801 | // legalized. |
| 5802 | const unsigned NewBitSize = DstEltSize / 2; |
| 5803 | const LLT HalfTy = LLT::scalar(NewBitSize); |
| 5804 | const LLT CondTy = LLT::scalar(1); |
| 5805 | |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 5806 | if (auto VRegAndVal = getIConstantVRegValWithLookThrough(Amt, MRI)) { |
Konstantin Schwarz | 64bef13 | 2020-10-08 14:30:33 +0200 | [diff] [blame] | 5807 | return narrowScalarShiftByConstant(MI, VRegAndVal->Value, HalfTy, |
| 5808 | ShiftAmtTy); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5809 | } |
| 5810 | |
| 5811 | // TODO: Expand with known bits. |
| 5812 | |
| 5813 | // Handle the fully general expansion by an unknown amount. |
| 5814 | auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); |
| 5815 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 5816 | Register InL = MRI.createGenericVirtualRegister(HalfTy); |
| 5817 | Register InH = MRI.createGenericVirtualRegister(HalfTy); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 5818 | MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1)); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5819 | |
| 5820 | auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); |
| 5821 | auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); |
| 5822 | |
| 5823 | auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); |
| 5824 | auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); |
| 5825 | auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); |
| 5826 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 5827 | Register ResultRegs[2]; |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5828 | switch (MI.getOpcode()) { |
| 5829 | case TargetOpcode::G_SHL: { |
| 5830 | // Short: ShAmt < NewBitSize |
Petar Avramovic | d568ed4 | 2019-08-27 14:22:32 +0000 | [diff] [blame] | 5831 | auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5832 | |
Petar Avramovic | d568ed4 | 2019-08-27 14:22:32 +0000 | [diff] [blame] | 5833 | auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); |
| 5834 | auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt); |
| 5835 | auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5836 | |
| 5837 | // Long: ShAmt >= NewBitSize |
| 5838 | auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. |
| 5839 | auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. |
| 5840 | |
| 5841 | auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); |
| 5842 | auto Hi = MIRBuilder.buildSelect( |
| 5843 | HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); |
| 5844 | |
| 5845 | ResultRegs[0] = Lo.getReg(0); |
| 5846 | ResultRegs[1] = Hi.getReg(0); |
| 5847 | break; |
| 5848 | } |
Petar Avramovic | a393238 | 2019-08-27 14:33:05 +0000 | [diff] [blame] | 5849 | case TargetOpcode::G_LSHR: |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5850 | case TargetOpcode::G_ASHR: { |
| 5851 | // Short: ShAmt < NewBitSize |
Petar Avramovic | a393238 | 2019-08-27 14:33:05 +0000 | [diff] [blame] | 5852 | auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt}); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5853 | |
Petar Avramovic | d568ed4 | 2019-08-27 14:22:32 +0000 | [diff] [blame] | 5854 | auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt); |
| 5855 | auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack); |
| 5856 | auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5857 | |
| 5858 | // Long: ShAmt >= NewBitSize |
Petar Avramovic | a393238 | 2019-08-27 14:33:05 +0000 | [diff] [blame] | 5859 | MachineInstrBuilder HiL; |
| 5860 | if (MI.getOpcode() == TargetOpcode::G_LSHR) { |
| 5861 | HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. |
| 5862 | } else { |
| 5863 | auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1); |
| 5864 | HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part. |
| 5865 | } |
| 5866 | auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, |
| 5867 | {InH, AmtExcess}); // Lo from Hi part. |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5868 | |
| 5869 | auto Lo = MIRBuilder.buildSelect( |
| 5870 | HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); |
| 5871 | |
| 5872 | auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); |
| 5873 | |
| 5874 | ResultRegs[0] = Lo.getReg(0); |
| 5875 | ResultRegs[1] = Hi.getReg(0); |
| 5876 | break; |
| 5877 | } |
| 5878 | default: |
| 5879 | llvm_unreachable("not a shift"); |
| 5880 | } |
| 5881 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 5882 | MIRBuilder.buildMergeLikeInstr(DstReg, ResultRegs); |
Matt Arsenault | fbec8fe | 2019-02-07 19:37:44 +0000 | [diff] [blame] | 5883 | MI.eraseFromParent(); |
| 5884 | return Legalized; |
| 5885 | } |
| 5886 | |
| 5887 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 72bcf15 | 2019-02-28 00:01:05 +0000 | [diff] [blame] | 5888 | LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, |
| 5889 | LLT MoreTy) { |
| 5890 | assert(TypeIdx == 0 && "Expecting only Idx 0"); |
| 5891 | |
| 5892 | Observer.changingInstr(MI); |
| 5893 | for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { |
| 5894 | MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); |
| 5895 | MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); |
| 5896 | moreElementsVectorSrc(MI, MoreTy, I); |
| 5897 | } |
| 5898 | |
| 5899 | MachineBasicBlock &MBB = *MI.getParent(); |
Amara Emerson | 9d64721 | 2019-09-16 23:46:03 +0000 | [diff] [blame] | 5900 | MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); |
Matt Arsenault | 72bcf15 | 2019-02-28 00:01:05 +0000 | [diff] [blame] | 5901 | moreElementsVectorDst(MI, MoreTy, 0); |
| 5902 | Observer.changedInstr(MI); |
| 5903 | return Legalized; |
| 5904 | } |
| 5905 | |
Dhruv Chawla (work) | 2c9b6c1 | 2024-02-27 15:57:46 +0530 | [diff] [blame] | 5906 | MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce( |
| 5907 | unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) { |
| 5908 | assert(Ty.isScalar() && "Expected scalar type to make neutral element for"); |
| 5909 | |
| 5910 | switch (Opcode) { |
| 5911 | default: |
| 5912 | llvm_unreachable( |
| 5913 | "getNeutralElementForVecReduce called with invalid opcode!"); |
| 5914 | case TargetOpcode::G_VECREDUCE_ADD: |
| 5915 | case TargetOpcode::G_VECREDUCE_OR: |
| 5916 | case TargetOpcode::G_VECREDUCE_XOR: |
| 5917 | case TargetOpcode::G_VECREDUCE_UMAX: |
| 5918 | return MIRBuilder.buildConstant(Ty, 0); |
| 5919 | case TargetOpcode::G_VECREDUCE_MUL: |
| 5920 | return MIRBuilder.buildConstant(Ty, 1); |
| 5921 | case TargetOpcode::G_VECREDUCE_AND: |
| 5922 | case TargetOpcode::G_VECREDUCE_UMIN: |
| 5923 | return MIRBuilder.buildConstant( |
| 5924 | Ty, APInt::getAllOnes(Ty.getScalarSizeInBits())); |
| 5925 | case TargetOpcode::G_VECREDUCE_SMAX: |
| 5926 | return MIRBuilder.buildConstant( |
| 5927 | Ty, APInt::getSignedMinValue(Ty.getSizeInBits())); |
| 5928 | case TargetOpcode::G_VECREDUCE_SMIN: |
| 5929 | return MIRBuilder.buildConstant( |
| 5930 | Ty, APInt::getSignedMaxValue(Ty.getSizeInBits())); |
| 5931 | case TargetOpcode::G_VECREDUCE_FADD: |
| 5932 | return MIRBuilder.buildFConstant(Ty, -0.0); |
| 5933 | case TargetOpcode::G_VECREDUCE_FMUL: |
| 5934 | return MIRBuilder.buildFConstant(Ty, 1.0); |
| 5935 | case TargetOpcode::G_VECREDUCE_FMINIMUM: |
| 5936 | case TargetOpcode::G_VECREDUCE_FMAXIMUM: |
| 5937 | assert(false && "getNeutralElementForVecReduce unimplemented for " |
Nikita Popov | f2f1845 | 2024-06-21 08:33:40 +0200 | [diff] [blame] | 5938 | "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!"); |
Dhruv Chawla (work) | 2c9b6c1 | 2024-02-27 15:57:46 +0530 | [diff] [blame] | 5939 | } |
| 5940 | llvm_unreachable("switch expected to return!"); |
| 5941 | } |
| 5942 | |
Matt Arsenault | 72bcf15 | 2019-02-28 00:01:05 +0000 | [diff] [blame] | 5943 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 5944 | LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, |
| 5945 | LLT MoreTy) { |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 5946 | unsigned Opc = MI.getOpcode(); |
| 5947 | switch (Opc) { |
Matt Arsenault | 7bedceb | 2019-08-01 01:44:22 +0000 | [diff] [blame] | 5948 | case TargetOpcode::G_IMPLICIT_DEF: |
| 5949 | case TargetOpcode::G_LOAD: { |
| 5950 | if (TypeIdx != 0) |
| 5951 | return UnableToLegalize; |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 5952 | Observer.changingInstr(MI); |
| 5953 | moreElementsVectorDst(MI, MoreTy, 0); |
| 5954 | Observer.changedInstr(MI); |
| 5955 | return Legalized; |
| 5956 | } |
Matt Arsenault | 7bedceb | 2019-08-01 01:44:22 +0000 | [diff] [blame] | 5957 | case TargetOpcode::G_STORE: |
| 5958 | if (TypeIdx != 0) |
| 5959 | return UnableToLegalize; |
| 5960 | Observer.changingInstr(MI); |
| 5961 | moreElementsVectorSrc(MI, MoreTy, 0); |
| 5962 | Observer.changedInstr(MI); |
| 5963 | return Legalized; |
Matt Arsenault | 26b7e85 | 2019-02-19 16:30:19 +0000 | [diff] [blame] | 5964 | case TargetOpcode::G_AND: |
| 5965 | case TargetOpcode::G_OR: |
Matt Arsenault | 0f3ba44 | 2019-05-23 17:58:48 +0000 | [diff] [blame] | 5966 | case TargetOpcode::G_XOR: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5967 | case TargetOpcode::G_ADD: |
| 5968 | case TargetOpcode::G_SUB: |
| 5969 | case TargetOpcode::G_MUL: |
| 5970 | case TargetOpcode::G_FADD: |
David Green | ef0b8cf | 2023-08-23 09:51:06 +0100 | [diff] [blame] | 5971 | case TargetOpcode::G_FSUB: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5972 | case TargetOpcode::G_FMUL: |
David Green | 58a2f83 | 2023-08-30 22:09:53 +0100 | [diff] [blame] | 5973 | case TargetOpcode::G_FDIV: |
David Green | 3a77522 | 2024-02-17 10:19:27 +0000 | [diff] [blame] | 5974 | case TargetOpcode::G_FCOPYSIGN: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 5975 | case TargetOpcode::G_UADDSAT: |
| 5976 | case TargetOpcode::G_USUBSAT: |
| 5977 | case TargetOpcode::G_SADDSAT: |
| 5978 | case TargetOpcode::G_SSUBSAT: |
Matt Arsenault | 0f3ba44 | 2019-05-23 17:58:48 +0000 | [diff] [blame] | 5979 | case TargetOpcode::G_SMIN: |
| 5980 | case TargetOpcode::G_SMAX: |
| 5981 | case TargetOpcode::G_UMIN: |
Matt Arsenault | 9fd31fd | 2019-07-27 17:47:08 -0400 | [diff] [blame] | 5982 | case TargetOpcode::G_UMAX: |
| 5983 | case TargetOpcode::G_FMINNUM: |
| 5984 | case TargetOpcode::G_FMAXNUM: |
| 5985 | case TargetOpcode::G_FMINNUM_IEEE: |
| 5986 | case TargetOpcode::G_FMAXNUM_IEEE: |
| 5987 | case TargetOpcode::G_FMINIMUM: |
Matt Arsenault | 08ec15e | 2022-11-17 22:14:35 -0800 | [diff] [blame] | 5988 | case TargetOpcode::G_FMAXIMUM: |
| 5989 | case TargetOpcode::G_STRICT_FADD: |
| 5990 | case TargetOpcode::G_STRICT_FSUB: |
chuongg3 | bfef161 | 2024-01-22 14:08:26 +0000 | [diff] [blame] | 5991 | case TargetOpcode::G_STRICT_FMUL: |
| 5992 | case TargetOpcode::G_SHL: |
| 5993 | case TargetOpcode::G_ASHR: |
| 5994 | case TargetOpcode::G_LSHR: { |
Matt Arsenault | 26b7e85 | 2019-02-19 16:30:19 +0000 | [diff] [blame] | 5995 | Observer.changingInstr(MI); |
| 5996 | moreElementsVectorSrc(MI, MoreTy, 1); |
| 5997 | moreElementsVectorSrc(MI, MoreTy, 2); |
| 5998 | moreElementsVectorDst(MI, MoreTy, 0); |
| 5999 | Observer.changedInstr(MI); |
| 6000 | return Legalized; |
| 6001 | } |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6002 | case TargetOpcode::G_FMA: |
Matt Arsenault | fe5b9a6 | 2020-05-31 13:23:20 -0400 | [diff] [blame] | 6003 | case TargetOpcode::G_STRICT_FMA: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6004 | case TargetOpcode::G_FSHR: |
| 6005 | case TargetOpcode::G_FSHL: { |
| 6006 | Observer.changingInstr(MI); |
| 6007 | moreElementsVectorSrc(MI, MoreTy, 1); |
| 6008 | moreElementsVectorSrc(MI, MoreTy, 2); |
| 6009 | moreElementsVectorSrc(MI, MoreTy, 3); |
| 6010 | moreElementsVectorDst(MI, MoreTy, 0); |
| 6011 | Observer.changedInstr(MI); |
| 6012 | return Legalized; |
| 6013 | } |
Mateja Marjanovic | cf76074 | 2023-05-03 17:32:22 +0200 | [diff] [blame] | 6014 | case TargetOpcode::G_EXTRACT_VECTOR_ELT: |
Matt Arsenault | 4d88427 | 2019-02-19 16:44:22 +0000 | [diff] [blame] | 6015 | case TargetOpcode::G_EXTRACT: |
| 6016 | if (TypeIdx != 1) |
| 6017 | return UnableToLegalize; |
| 6018 | Observer.changingInstr(MI); |
| 6019 | moreElementsVectorSrc(MI, MoreTy, 1); |
| 6020 | Observer.changedInstr(MI); |
| 6021 | return Legalized; |
Matt Arsenault | c4d0755 | 2019-02-20 16:11:22 +0000 | [diff] [blame] | 6022 | case TargetOpcode::G_INSERT: |
Mateja Marjanovic | cf76074 | 2023-05-03 17:32:22 +0200 | [diff] [blame] | 6023 | case TargetOpcode::G_INSERT_VECTOR_ELT: |
Dominik Montada | 55e3a7c | 2020-04-14 11:25:05 +0200 | [diff] [blame] | 6024 | case TargetOpcode::G_FREEZE: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6025 | case TargetOpcode::G_FNEG: |
| 6026 | case TargetOpcode::G_FABS: |
David Green | acd17ea | 2023-08-11 10:16:45 +0100 | [diff] [blame] | 6027 | case TargetOpcode::G_FSQRT: |
David Green | cf65afb | 2023-08-17 16:25:32 +0100 | [diff] [blame] | 6028 | case TargetOpcode::G_FCEIL: |
| 6029 | case TargetOpcode::G_FFLOOR: |
| 6030 | case TargetOpcode::G_FNEARBYINT: |
| 6031 | case TargetOpcode::G_FRINT: |
| 6032 | case TargetOpcode::G_INTRINSIC_ROUND: |
| 6033 | case TargetOpcode::G_INTRINSIC_ROUNDEVEN: |
| 6034 | case TargetOpcode::G_INTRINSIC_TRUNC: |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6035 | case TargetOpcode::G_BSWAP: |
| 6036 | case TargetOpcode::G_FCANONICALIZE: |
| 6037 | case TargetOpcode::G_SEXT_INREG: |
chuongg3 | 2c552d3 | 2024-01-28 20:21:38 +0000 | [diff] [blame] | 6038 | case TargetOpcode::G_ABS: |
Matt Arsenault | c4d0755 | 2019-02-20 16:11:22 +0000 | [diff] [blame] | 6039 | if (TypeIdx != 0) |
| 6040 | return UnableToLegalize; |
| 6041 | Observer.changingInstr(MI); |
| 6042 | moreElementsVectorSrc(MI, MoreTy, 1); |
| 6043 | moreElementsVectorDst(MI, MoreTy, 0); |
| 6044 | Observer.changedInstr(MI); |
| 6045 | return Legalized; |
Matt Arsenault | 3754f60 | 2022-04-11 21:31:15 -0400 | [diff] [blame] | 6046 | case TargetOpcode::G_SELECT: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6047 | auto [DstReg, DstTy, CondReg, CondTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 3754f60 | 2022-04-11 21:31:15 -0400 | [diff] [blame] | 6048 | if (TypeIdx == 1) { |
| 6049 | if (!CondTy.isScalar() || |
| 6050 | DstTy.getElementCount() != MoreTy.getElementCount()) |
| 6051 | return UnableToLegalize; |
| 6052 | |
| 6053 | // This is turning a scalar select of vectors into a vector |
| 6054 | // select. Broadcast the select condition. |
| 6055 | auto ShufSplat = MIRBuilder.buildShuffleSplat(MoreTy, CondReg); |
| 6056 | Observer.changingInstr(MI); |
| 6057 | MI.getOperand(1).setReg(ShufSplat.getReg(0)); |
| 6058 | Observer.changedInstr(MI); |
| 6059 | return Legalized; |
| 6060 | } |
| 6061 | |
| 6062 | if (CondTy.isVector()) |
Matt Arsenault | b4c95b3 | 2019-02-19 17:03:09 +0000 | [diff] [blame] | 6063 | return UnableToLegalize; |
| 6064 | |
| 6065 | Observer.changingInstr(MI); |
| 6066 | moreElementsVectorSrc(MI, MoreTy, 2); |
| 6067 | moreElementsVectorSrc(MI, MoreTy, 3); |
| 6068 | moreElementsVectorDst(MI, MoreTy, 0); |
| 6069 | Observer.changedInstr(MI); |
| 6070 | return Legalized; |
Matt Arsenault | 3754f60 | 2022-04-11 21:31:15 -0400 | [diff] [blame] | 6071 | } |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6072 | case TargetOpcode::G_UNMERGE_VALUES: |
| 6073 | return UnableToLegalize; |
Matt Arsenault | 72bcf15 | 2019-02-28 00:01:05 +0000 | [diff] [blame] | 6074 | case TargetOpcode::G_PHI: |
| 6075 | return moreElementsVectorPhi(MI, TypeIdx, MoreTy); |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6076 | case TargetOpcode::G_SHUFFLE_VECTOR: |
| 6077 | return moreElementsVectorShuffle(MI, TypeIdx, MoreTy); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 6078 | case TargetOpcode::G_BUILD_VECTOR: { |
| 6079 | SmallVector<SrcOp, 8> Elts; |
| 6080 | for (auto Op : MI.uses()) { |
| 6081 | Elts.push_back(Op.getReg()); |
| 6082 | } |
| 6083 | |
| 6084 | for (unsigned i = Elts.size(); i < MoreTy.getNumElements(); ++i) { |
| 6085 | Elts.push_back(MIRBuilder.buildUndef(MoreTy.getScalarType())); |
| 6086 | } |
| 6087 | |
| 6088 | MIRBuilder.buildDeleteTrailingVectorElements( |
| 6089 | MI.getOperand(0).getReg(), MIRBuilder.buildInstr(Opc, {MoreTy}, Elts)); |
| 6090 | MI.eraseFromParent(); |
| 6091 | return Legalized; |
| 6092 | } |
Dhruv Chawla | 843a978 | 2024-03-18 07:46:17 +0530 | [diff] [blame] | 6093 | case TargetOpcode::G_SEXT: |
| 6094 | case TargetOpcode::G_ZEXT: |
| 6095 | case TargetOpcode::G_ANYEXT: |
chuongg3 | d88d983 | 2023-10-11 16:05:25 +0100 | [diff] [blame] | 6096 | case TargetOpcode::G_TRUNC: |
David Green | 6edc9a7 | 2023-07-23 16:58:13 +0100 | [diff] [blame] | 6097 | case TargetOpcode::G_FPTRUNC: |
David Green | 54574d3 | 2023-11-04 11:47:05 +0000 | [diff] [blame] | 6098 | case TargetOpcode::G_FPEXT: |
| 6099 | case TargetOpcode::G_FPTOSI: |
David Green | 10ce319 | 2023-11-10 13:41:13 +0000 | [diff] [blame] | 6100 | case TargetOpcode::G_FPTOUI: |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 6101 | case TargetOpcode::G_FPTOSI_SAT: |
| 6102 | case TargetOpcode::G_FPTOUI_SAT: |
David Green | 10ce319 | 2023-11-10 13:41:13 +0000 | [diff] [blame] | 6103 | case TargetOpcode::G_SITOFP: |
| 6104 | case TargetOpcode::G_UITOFP: { |
David Green | 74c0bdf | 2023-07-18 18:52:19 +0100 | [diff] [blame] | 6105 | Observer.changingInstr(MI); |
David Green | fbc2473 | 2024-03-26 09:48:06 +0000 | [diff] [blame] | 6106 | LLT SrcExtTy; |
| 6107 | LLT DstExtTy; |
| 6108 | if (TypeIdx == 0) { |
| 6109 | DstExtTy = MoreTy; |
| 6110 | SrcExtTy = LLT::fixed_vector( |
| 6111 | MoreTy.getNumElements(), |
| 6112 | MRI.getType(MI.getOperand(1).getReg()).getElementType()); |
| 6113 | } else { |
| 6114 | DstExtTy = LLT::fixed_vector( |
| 6115 | MoreTy.getNumElements(), |
| 6116 | MRI.getType(MI.getOperand(0).getReg()).getElementType()); |
| 6117 | SrcExtTy = MoreTy; |
| 6118 | } |
| 6119 | moreElementsVectorSrc(MI, SrcExtTy, 1); |
| 6120 | moreElementsVectorDst(MI, DstExtTy, 0); |
David Green | 74c0bdf | 2023-07-18 18:52:19 +0100 | [diff] [blame] | 6121 | Observer.changedInstr(MI); |
| 6122 | return Legalized; |
| 6123 | } |
David Green | f297d0b | 2024-01-28 15:42:36 +0000 | [diff] [blame] | 6124 | case TargetOpcode::G_ICMP: |
| 6125 | case TargetOpcode::G_FCMP: { |
| 6126 | if (TypeIdx != 1) |
| 6127 | return UnableToLegalize; |
| 6128 | |
Thorsten Schütt | 67dc6e9 | 2024-01-17 22:23:51 +0100 | [diff] [blame] | 6129 | Observer.changingInstr(MI); |
| 6130 | moreElementsVectorSrc(MI, MoreTy, 2); |
| 6131 | moreElementsVectorSrc(MI, MoreTy, 3); |
David Green | f297d0b | 2024-01-28 15:42:36 +0000 | [diff] [blame] | 6132 | LLT CondTy = LLT::fixed_vector( |
| 6133 | MoreTy.getNumElements(), |
| 6134 | MRI.getType(MI.getOperand(0).getReg()).getElementType()); |
| 6135 | moreElementsVectorDst(MI, CondTy, 0); |
Thorsten Schütt | 67dc6e9 | 2024-01-17 22:23:51 +0100 | [diff] [blame] | 6136 | Observer.changedInstr(MI); |
| 6137 | return Legalized; |
| 6138 | } |
chuongg3 | 0fb3d42 | 2024-02-21 13:24:45 +0000 | [diff] [blame] | 6139 | case TargetOpcode::G_BITCAST: { |
| 6140 | if (TypeIdx != 0) |
| 6141 | return UnableToLegalize; |
| 6142 | |
| 6143 | LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); |
| 6144 | LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); |
| 6145 | |
| 6146 | unsigned coefficient = SrcTy.getNumElements() * MoreTy.getNumElements(); |
| 6147 | if (coefficient % DstTy.getNumElements() != 0) |
| 6148 | return UnableToLegalize; |
| 6149 | |
| 6150 | coefficient = coefficient / DstTy.getNumElements(); |
| 6151 | |
| 6152 | LLT NewTy = SrcTy.changeElementCount( |
| 6153 | ElementCount::get(coefficient, MoreTy.isScalable())); |
| 6154 | Observer.changingInstr(MI); |
| 6155 | moreElementsVectorSrc(MI, NewTy, 1); |
| 6156 | moreElementsVectorDst(MI, MoreTy, 0); |
| 6157 | Observer.changedInstr(MI); |
| 6158 | return Legalized; |
| 6159 | } |
Dhruv Chawla (work) | 2c9b6c1 | 2024-02-27 15:57:46 +0530 | [diff] [blame] | 6160 | case TargetOpcode::G_VECREDUCE_FADD: |
| 6161 | case TargetOpcode::G_VECREDUCE_FMUL: |
| 6162 | case TargetOpcode::G_VECREDUCE_ADD: |
| 6163 | case TargetOpcode::G_VECREDUCE_MUL: |
| 6164 | case TargetOpcode::G_VECREDUCE_AND: |
| 6165 | case TargetOpcode::G_VECREDUCE_OR: |
| 6166 | case TargetOpcode::G_VECREDUCE_XOR: |
| 6167 | case TargetOpcode::G_VECREDUCE_SMAX: |
| 6168 | case TargetOpcode::G_VECREDUCE_SMIN: |
| 6169 | case TargetOpcode::G_VECREDUCE_UMAX: |
| 6170 | case TargetOpcode::G_VECREDUCE_UMIN: { |
| 6171 | LLT OrigTy = MRI.getType(MI.getOperand(1).getReg()); |
| 6172 | MachineOperand &MO = MI.getOperand(1); |
| 6173 | auto NewVec = MIRBuilder.buildPadVectorWithUndefElements(MoreTy, MO); |
| 6174 | auto NeutralElement = getNeutralElementForVecReduce( |
| 6175 | MI.getOpcode(), MIRBuilder, MoreTy.getElementType()); |
| 6176 | |
| 6177 | LLT IdxTy(TLI.getVectorIdxTy(MIRBuilder.getDataLayout())); |
| 6178 | for (size_t i = OrigTy.getNumElements(), e = MoreTy.getNumElements(); |
| 6179 | i != e; i++) { |
| 6180 | auto Idx = MIRBuilder.buildConstant(IdxTy, i); |
| 6181 | NewVec = MIRBuilder.buildInsertVectorElement(MoreTy, NewVec, |
| 6182 | NeutralElement, Idx); |
| 6183 | } |
| 6184 | |
| 6185 | Observer.changingInstr(MI); |
| 6186 | MO.setReg(NewVec.getReg(0)); |
| 6187 | Observer.changedInstr(MI); |
| 6188 | return Legalized; |
| 6189 | } |
| 6190 | |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 6191 | default: |
| 6192 | return UnableToLegalize; |
| 6193 | } |
| 6194 | } |
| 6195 | |
Vladislav Dzhidzhoev | 3a51eed | 2023-02-07 21:32:50 +0100 | [diff] [blame] | 6196 | LegalizerHelper::LegalizeResult |
| 6197 | LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6198 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Kevin Athey | ec7cffc | 2022-12-15 11:19:24 -0800 | [diff] [blame] | 6199 | ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); |
| 6200 | unsigned MaskNumElts = Mask.size(); |
| 6201 | unsigned SrcNumElts = SrcTy.getNumElements(); |
Kevin Athey | ec7cffc | 2022-12-15 11:19:24 -0800 | [diff] [blame] | 6202 | LLT DestEltTy = DstTy.getElementType(); |
| 6203 | |
Vladislav Dzhidzhoev | 3a51eed | 2023-02-07 21:32:50 +0100 | [diff] [blame] | 6204 | if (MaskNumElts == SrcNumElts) |
| 6205 | return Legalized; |
| 6206 | |
| 6207 | if (MaskNumElts < SrcNumElts) { |
| 6208 | // Extend mask to match new destination vector size with |
| 6209 | // undef values. |
Craig Topper | 5797ed6 | 2024-12-10 22:18:46 -0800 | [diff] [blame] | 6210 | SmallVector<int, 16> NewMask(SrcNumElts, -1); |
| 6211 | llvm::copy(Mask, NewMask.begin()); |
Vladislav Dzhidzhoev | 3a51eed | 2023-02-07 21:32:50 +0100 | [diff] [blame] | 6212 | |
| 6213 | moreElementsVectorDst(MI, SrcTy, 0); |
| 6214 | MIRBuilder.setInstrAndDebugLoc(MI); |
| 6215 | MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(), |
| 6216 | MI.getOperand(1).getReg(), |
| 6217 | MI.getOperand(2).getReg(), NewMask); |
| 6218 | MI.eraseFromParent(); |
| 6219 | |
| 6220 | return Legalized; |
Kevin Athey | ec7cffc | 2022-12-15 11:19:24 -0800 | [diff] [blame] | 6221 | } |
| 6222 | |
| 6223 | unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts); |
| 6224 | unsigned NumConcat = PaddedMaskNumElts / SrcNumElts; |
| 6225 | LLT PaddedTy = LLT::fixed_vector(PaddedMaskNumElts, DestEltTy); |
| 6226 | |
| 6227 | // Create new source vectors by concatenating the initial |
| 6228 | // source vectors with undefined vectors of the same size. |
| 6229 | auto Undef = MIRBuilder.buildUndef(SrcTy); |
| 6230 | SmallVector<Register, 8> MOps1(NumConcat, Undef.getReg(0)); |
| 6231 | SmallVector<Register, 8> MOps2(NumConcat, Undef.getReg(0)); |
| 6232 | MOps1[0] = MI.getOperand(1).getReg(); |
| 6233 | MOps2[0] = MI.getOperand(2).getReg(); |
| 6234 | |
| 6235 | auto Src1 = MIRBuilder.buildConcatVectors(PaddedTy, MOps1); |
| 6236 | auto Src2 = MIRBuilder.buildConcatVectors(PaddedTy, MOps2); |
| 6237 | |
| 6238 | // Readjust mask for new input vector length. |
| 6239 | SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1); |
| 6240 | for (unsigned I = 0; I != MaskNumElts; ++I) { |
| 6241 | int Idx = Mask[I]; |
| 6242 | if (Idx >= static_cast<int>(SrcNumElts)) |
| 6243 | Idx += PaddedMaskNumElts - SrcNumElts; |
| 6244 | MappedOps[I] = Idx; |
| 6245 | } |
| 6246 | |
| 6247 | // If we got more elements than required, extract subvector. |
| 6248 | if (MaskNumElts != PaddedMaskNumElts) { |
| 6249 | auto Shuffle = |
| 6250 | MIRBuilder.buildShuffleVector(PaddedTy, Src1, Src2, MappedOps); |
| 6251 | |
| 6252 | SmallVector<Register, 16> Elts(MaskNumElts); |
| 6253 | for (unsigned I = 0; I < MaskNumElts; ++I) { |
| 6254 | Elts[I] = |
| 6255 | MIRBuilder.buildExtractVectorElementConstant(DestEltTy, Shuffle, I) |
| 6256 | .getReg(0); |
| 6257 | } |
| 6258 | MIRBuilder.buildBuildVector(DstReg, Elts); |
| 6259 | } else { |
| 6260 | MIRBuilder.buildShuffleVector(DstReg, Src1, Src2, MappedOps); |
| 6261 | } |
| 6262 | |
| 6263 | MI.eraseFromParent(); |
| 6264 | return LegalizerHelper::LegalizeResult::Legalized; |
| 6265 | } |
| 6266 | |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6267 | LegalizerHelper::LegalizeResult |
| 6268 | LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI, |
| 6269 | unsigned int TypeIdx, LLT MoreTy) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6270 | auto [DstTy, Src1Ty, Src2Ty] = MI.getFirst3LLTs(); |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6271 | ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6272 | unsigned NumElts = DstTy.getNumElements(); |
| 6273 | unsigned WidenNumElts = MoreTy.getNumElements(); |
| 6274 | |
Kevin Athey | ec7cffc | 2022-12-15 11:19:24 -0800 | [diff] [blame] | 6275 | if (DstTy.isVector() && Src1Ty.isVector() && |
Vladislav Dzhidzhoev | 3a51eed | 2023-02-07 21:32:50 +0100 | [diff] [blame] | 6276 | DstTy.getNumElements() != Src1Ty.getNumElements()) { |
| 6277 | return equalizeVectorShuffleLengths(MI); |
Kevin Athey | ec7cffc | 2022-12-15 11:19:24 -0800 | [diff] [blame] | 6278 | } |
| 6279 | |
| 6280 | if (TypeIdx != 0) |
| 6281 | return UnableToLegalize; |
| 6282 | |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6283 | // Expect a canonicalized shuffle. |
| 6284 | if (DstTy != Src1Ty || DstTy != Src2Ty) |
| 6285 | return UnableToLegalize; |
| 6286 | |
| 6287 | moreElementsVectorSrc(MI, MoreTy, 1); |
| 6288 | moreElementsVectorSrc(MI, MoreTy, 2); |
| 6289 | |
| 6290 | // Adjust mask based on new input vector length. |
Craig Topper | 5797ed6 | 2024-12-10 22:18:46 -0800 | [diff] [blame] | 6291 | SmallVector<int, 16> NewMask(WidenNumElts, -1); |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6292 | for (unsigned I = 0; I != NumElts; ++I) { |
| 6293 | int Idx = Mask[I]; |
| 6294 | if (Idx < static_cast<int>(NumElts)) |
Craig Topper | 5797ed6 | 2024-12-10 22:18:46 -0800 | [diff] [blame] | 6295 | NewMask[I] = Idx; |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6296 | else |
Craig Topper | 5797ed6 | 2024-12-10 22:18:46 -0800 | [diff] [blame] | 6297 | NewMask[I] = Idx - NumElts + WidenNumElts; |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6298 | } |
Amara Emerson | 97c4263 | 2021-07-09 23:11:22 -0700 | [diff] [blame] | 6299 | moreElementsVectorDst(MI, MoreTy, 0); |
| 6300 | MIRBuilder.setInstrAndDebugLoc(MI); |
| 6301 | MIRBuilder.buildShuffleVector(MI.getOperand(0).getReg(), |
| 6302 | MI.getOperand(1).getReg(), |
| 6303 | MI.getOperand(2).getReg(), NewMask); |
| 6304 | MI.eraseFromParent(); |
| 6305 | return Legalized; |
| 6306 | } |
| 6307 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 6308 | void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs, |
| 6309 | ArrayRef<Register> Src1Regs, |
| 6310 | ArrayRef<Register> Src2Regs, |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6311 | LLT NarrowTy) { |
| 6312 | MachineIRBuilder &B = MIRBuilder; |
| 6313 | unsigned SrcParts = Src1Regs.size(); |
| 6314 | unsigned DstParts = DstRegs.size(); |
| 6315 | |
| 6316 | unsigned DstIdx = 0; // Low bits of the result. |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6317 | Register FactorSum = |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6318 | B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); |
| 6319 | DstRegs[DstIdx] = FactorSum; |
| 6320 | |
| 6321 | unsigned CarrySumPrevDstIdx; |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 6322 | SmallVector<Register, 4> Factors; |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6323 | |
| 6324 | for (DstIdx = 1; DstIdx < DstParts; DstIdx++) { |
| 6325 | // Collect low parts of muls for DstIdx. |
| 6326 | for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1; |
| 6327 | i <= std::min(DstIdx, SrcParts - 1); ++i) { |
| 6328 | MachineInstrBuilder Mul = |
| 6329 | B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); |
| 6330 | Factors.push_back(Mul.getReg(0)); |
| 6331 | } |
| 6332 | // Collect high parts of muls from previous DstIdx. |
| 6333 | for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts; |
| 6334 | i <= std::min(DstIdx - 1, SrcParts - 1); ++i) { |
| 6335 | MachineInstrBuilder Umulh = |
| 6336 | B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); |
| 6337 | Factors.push_back(Umulh.getReg(0)); |
| 6338 | } |
Greg Bedwell | b1c4b4d | 2019-10-28 14:28:00 +0000 | [diff] [blame] | 6339 | // Add CarrySum from additions calculated for previous DstIdx. |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6340 | if (DstIdx != 1) { |
| 6341 | Factors.push_back(CarrySumPrevDstIdx); |
| 6342 | } |
| 6343 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6344 | Register CarrySum; |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6345 | // Add all factors and accumulate all carries into CarrySum. |
| 6346 | if (DstIdx != DstParts - 1) { |
| 6347 | MachineInstrBuilder Uaddo = |
Jay Foad | 24688f8 | 2021-10-04 20:25:42 +0100 | [diff] [blame] | 6348 | B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6349 | FactorSum = Uaddo.getReg(0); |
| 6350 | CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); |
| 6351 | for (unsigned i = 2; i < Factors.size(); ++i) { |
| 6352 | MachineInstrBuilder Uaddo = |
Jay Foad | 24688f8 | 2021-10-04 20:25:42 +0100 | [diff] [blame] | 6353 | B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6354 | FactorSum = Uaddo.getReg(0); |
| 6355 | MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); |
| 6356 | CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); |
| 6357 | } |
| 6358 | } else { |
| 6359 | // Since value for the next index is not calculated, neither is CarrySum. |
| 6360 | FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); |
| 6361 | for (unsigned i = 2; i < Factors.size(); ++i) |
| 6362 | FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); |
| 6363 | } |
| 6364 | |
| 6365 | CarrySumPrevDstIdx = CarrySum; |
| 6366 | DstRegs[DstIdx] = FactorSum; |
| 6367 | Factors.clear(); |
| 6368 | } |
| 6369 | } |
| 6370 | |
Matt Arsenault | 18ec382 | 2019-02-11 22:00:39 +0000 | [diff] [blame] | 6371 | LegalizerHelper::LegalizeResult |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6372 | LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, |
| 6373 | LLT NarrowTy) { |
| 6374 | if (TypeIdx != 0) |
| 6375 | return UnableToLegalize; |
| 6376 | |
Cassie Jones | 97a1cdb | 2021-02-14 14:42:46 -0500 | [diff] [blame] | 6377 | Register DstReg = MI.getOperand(0).getReg(); |
| 6378 | LLT DstType = MRI.getType(DstReg); |
| 6379 | // FIXME: add support for vector types |
| 6380 | if (DstType.isVector()) |
| 6381 | return UnableToLegalize; |
| 6382 | |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6383 | unsigned Opcode = MI.getOpcode(); |
| 6384 | unsigned OpO, OpE, OpF; |
| 6385 | switch (Opcode) { |
| 6386 | case TargetOpcode::G_SADDO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6387 | case TargetOpcode::G_SADDE: |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6388 | case TargetOpcode::G_UADDO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6389 | case TargetOpcode::G_UADDE: |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6390 | case TargetOpcode::G_ADD: |
| 6391 | OpO = TargetOpcode::G_UADDO; |
| 6392 | OpE = TargetOpcode::G_UADDE; |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6393 | OpF = TargetOpcode::G_UADDE; |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6394 | if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE) |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6395 | OpF = TargetOpcode::G_SADDE; |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6396 | break; |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6397 | case TargetOpcode::G_SSUBO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6398 | case TargetOpcode::G_SSUBE: |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6399 | case TargetOpcode::G_USUBO: |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6400 | case TargetOpcode::G_USUBE: |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6401 | case TargetOpcode::G_SUB: |
| 6402 | OpO = TargetOpcode::G_USUBO; |
| 6403 | OpE = TargetOpcode::G_USUBE; |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6404 | OpF = TargetOpcode::G_USUBE; |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6405 | if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE) |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6406 | OpF = TargetOpcode::G_SSUBE; |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6407 | break; |
| 6408 | default: |
| 6409 | llvm_unreachable("Unexpected add/sub opcode!"); |
| 6410 | } |
| 6411 | |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6412 | // 1 for a plain add/sub, 2 if this is an operation with a carry-out. |
| 6413 | unsigned NumDefs = MI.getNumExplicitDefs(); |
| 6414 | Register Src1 = MI.getOperand(NumDefs).getReg(); |
| 6415 | Register Src2 = MI.getOperand(NumDefs + 1).getReg(); |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6416 | Register CarryDst, CarryIn; |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6417 | if (NumDefs == 2) |
| 6418 | CarryDst = MI.getOperand(1).getReg(); |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6419 | if (MI.getNumOperands() == NumDefs + 3) |
| 6420 | CarryIn = MI.getOperand(NumDefs + 2).getReg(); |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6421 | |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6422 | LLT RegTy = MRI.getType(MI.getOperand(0).getReg()); |
| 6423 | LLT LeftoverTy, DummyTy; |
| 6424 | SmallVector<Register, 2> Src1Regs, Src2Regs, Src1Left, Src2Left, DstRegs; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6425 | extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left, |
| 6426 | MIRBuilder, MRI); |
| 6427 | extractParts(Src2, RegTy, NarrowTy, DummyTy, Src2Regs, Src2Left, MIRBuilder, |
| 6428 | MRI); |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6429 | |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6430 | int NarrowParts = Src1Regs.size(); |
Craig Topper | e3284d8 | 2024-12-10 07:18:20 -0800 | [diff] [blame] | 6431 | Src1Regs.append(Src1Left); |
| 6432 | Src2Regs.append(Src2Left); |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6433 | DstRegs.reserve(Src1Regs.size()); |
| 6434 | |
| 6435 | for (int i = 0, e = Src1Regs.size(); i != e; ++i) { |
| 6436 | Register DstReg = |
| 6437 | MRI.createGenericVirtualRegister(MRI.getType(Src1Regs[i])); |
Craig Topper | 7c12418 | 2024-12-09 20:23:24 -0800 | [diff] [blame] | 6438 | Register CarryOut; |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6439 | // Forward the final carry-out to the destination register |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6440 | if (i == e - 1 && CarryDst) |
Cassie Jones | c63b33b | 2021-02-22 17:10:58 -0500 | [diff] [blame] | 6441 | CarryOut = CarryDst; |
Craig Topper | 7c12418 | 2024-12-09 20:23:24 -0800 | [diff] [blame] | 6442 | else |
| 6443 | CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6444 | |
Cassie Jones | 8f956a5 | 2021-02-22 17:11:35 -0500 | [diff] [blame] | 6445 | if (!CarryIn) { |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6446 | MIRBuilder.buildInstr(OpO, {DstReg, CarryOut}, |
| 6447 | {Src1Regs[i], Src2Regs[i]}); |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6448 | } else if (i == e - 1) { |
Cassie Jones | e153264 | 2021-02-22 17:11:23 -0500 | [diff] [blame] | 6449 | MIRBuilder.buildInstr(OpF, {DstReg, CarryOut}, |
| 6450 | {Src1Regs[i], Src2Regs[i], CarryIn}); |
| 6451 | } else { |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6452 | MIRBuilder.buildInstr(OpE, {DstReg, CarryOut}, |
| 6453 | {Src1Regs[i], Src2Regs[i], CarryIn}); |
| 6454 | } |
| 6455 | |
| 6456 | DstRegs.push_back(DstReg); |
| 6457 | CarryIn = CarryOut; |
| 6458 | } |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6459 | insertParts(MI.getOperand(0).getReg(), RegTy, NarrowTy, |
serge-sans-paille | 38818b6 | 2023-01-04 08:28:45 +0100 | [diff] [blame] | 6460 | ArrayRef(DstRegs).take_front(NarrowParts), LeftoverTy, |
| 6461 | ArrayRef(DstRegs).drop_front(NarrowParts)); |
Justin Bogner | 4271e1d | 2021-03-02 14:46:03 -0800 | [diff] [blame] | 6462 | |
Cassie Jones | 36246388 | 2021-02-14 14:37:55 -0500 | [diff] [blame] | 6463 | MI.eraseFromParent(); |
| 6464 | return Legalized; |
| 6465 | } |
| 6466 | |
| 6467 | LegalizerHelper::LegalizeResult |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6468 | LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6469 | auto [DstReg, Src1, Src2] = MI.getFirst3Regs(); |
Petar Avramovic | 0b17e59 | 2019-03-11 10:00:17 +0000 | [diff] [blame] | 6470 | |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6471 | LLT Ty = MRI.getType(DstReg); |
Jay Foad | 24688f8 | 2021-10-04 20:25:42 +0100 | [diff] [blame] | 6472 | if (Ty.isVector()) |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6473 | return UnableToLegalize; |
| 6474 | |
Jay Foad | 0a031f5 | 2021-10-05 10:47:54 +0100 | [diff] [blame] | 6475 | unsigned Size = Ty.getSizeInBits(); |
Jay Foad | 24688f8 | 2021-10-04 20:25:42 +0100 | [diff] [blame] | 6476 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
Jay Foad | 0a031f5 | 2021-10-05 10:47:54 +0100 | [diff] [blame] | 6477 | if (Size % NarrowSize != 0) |
Jay Foad | 24688f8 | 2021-10-04 20:25:42 +0100 | [diff] [blame] | 6478 | return UnableToLegalize; |
| 6479 | |
Jay Foad | 0a031f5 | 2021-10-05 10:47:54 +0100 | [diff] [blame] | 6480 | unsigned NumParts = Size / NarrowSize; |
Petar Avramovic | 5229f47 | 2019-03-11 10:08:44 +0000 | [diff] [blame] | 6481 | bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; |
Jay Foad | 0a031f5 | 2021-10-05 10:47:54 +0100 | [diff] [blame] | 6482 | unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1); |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6483 | |
Matt Arsenault | de8451f | 2020-02-04 10:34:22 -0500 | [diff] [blame] | 6484 | SmallVector<Register, 2> Src1Parts, Src2Parts; |
| 6485 | SmallVector<Register, 2> DstTmpRegs(DstTmpParts); |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6486 | extractParts(Src1, NarrowTy, NumParts, Src1Parts, MIRBuilder, MRI); |
| 6487 | extractParts(Src2, NarrowTy, NumParts, Src2Parts, MIRBuilder, MRI); |
Petar Avramovic | 5229f47 | 2019-03-11 10:08:44 +0000 | [diff] [blame] | 6488 | multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6489 | |
Petar Avramovic | 5229f47 | 2019-03-11 10:08:44 +0000 | [diff] [blame] | 6490 | // Take only high half of registers if this is high mul. |
Jay Foad | 0a031f5 | 2021-10-05 10:47:54 +0100 | [diff] [blame] | 6491 | ArrayRef<Register> DstRegs(&DstTmpRegs[DstTmpParts - NumParts], NumParts); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 6492 | MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6493 | MI.eraseFromParent(); |
| 6494 | return Legalized; |
| 6495 | } |
| 6496 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6497 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 83a25a1 | 2021-03-26 17:29:36 -0400 | [diff] [blame] | 6498 | LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, |
| 6499 | LLT NarrowTy) { |
| 6500 | if (TypeIdx != 0) |
| 6501 | return UnableToLegalize; |
| 6502 | |
| 6503 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI; |
| 6504 | |
| 6505 | Register Src = MI.getOperand(1).getReg(); |
| 6506 | LLT SrcTy = MRI.getType(Src); |
| 6507 | |
| 6508 | // If all finite floats fit into the narrowed integer type, we can just swap |
| 6509 | // out the result type. This is practically only useful for conversions from |
| 6510 | // half to at least 16-bits, so just handle the one case. |
| 6511 | if (SrcTy.getScalarType() != LLT::scalar(16) || |
Simon Pilgrim | bc98076 | 2021-04-20 17:19:15 +0100 | [diff] [blame] | 6512 | NarrowTy.getScalarSizeInBits() < (IsSigned ? 17u : 16u)) |
Matt Arsenault | 83a25a1 | 2021-03-26 17:29:36 -0400 | [diff] [blame] | 6513 | return UnableToLegalize; |
| 6514 | |
| 6515 | Observer.changingInstr(MI); |
| 6516 | narrowScalarDst(MI, NarrowTy, 0, |
| 6517 | IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT); |
| 6518 | Observer.changedInstr(MI); |
| 6519 | return Legalized; |
| 6520 | } |
| 6521 | |
| 6522 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6523 | LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, |
| 6524 | LLT NarrowTy) { |
| 6525 | if (TypeIdx != 1) |
| 6526 | return UnableToLegalize; |
| 6527 | |
| 6528 | uint64_t NarrowSize = NarrowTy.getSizeInBits(); |
| 6529 | |
| 6530 | int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); |
| 6531 | // FIXME: add support for when SizeOp1 isn't an exact multiple of |
| 6532 | // NarrowSize. |
| 6533 | if (SizeOp1 % NarrowSize != 0) |
| 6534 | return UnableToLegalize; |
| 6535 | int NumParts = SizeOp1 / NarrowSize; |
| 6536 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 6537 | SmallVector<Register, 2> SrcRegs, DstRegs; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6538 | SmallVector<uint64_t, 2> Indexes; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6539 | extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs, |
| 6540 | MIRBuilder, MRI); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6541 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6542 | Register OpReg = MI.getOperand(0).getReg(); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6543 | uint64_t OpStart = MI.getOperand(2).getImm(); |
| 6544 | uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); |
| 6545 | for (int i = 0; i < NumParts; ++i) { |
| 6546 | unsigned SrcStart = i * NarrowSize; |
| 6547 | |
| 6548 | if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { |
| 6549 | // No part of the extract uses this subregister, ignore it. |
| 6550 | continue; |
| 6551 | } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { |
| 6552 | // The entire subregister is extracted, forward the value. |
| 6553 | DstRegs.push_back(SrcRegs[i]); |
| 6554 | continue; |
| 6555 | } |
| 6556 | |
| 6557 | // OpSegStart is where this destination segment would start in OpReg if it |
| 6558 | // extended infinitely in both directions. |
| 6559 | int64_t ExtractOffset; |
| 6560 | uint64_t SegSize; |
| 6561 | if (OpStart < SrcStart) { |
| 6562 | ExtractOffset = 0; |
| 6563 | SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); |
| 6564 | } else { |
| 6565 | ExtractOffset = OpStart - SrcStart; |
| 6566 | SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); |
| 6567 | } |
| 6568 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6569 | Register SegReg = SrcRegs[i]; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6570 | if (ExtractOffset != 0 || SegSize != NarrowSize) { |
| 6571 | // A genuine extract is needed. |
| 6572 | SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); |
| 6573 | MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); |
| 6574 | } |
| 6575 | |
| 6576 | DstRegs.push_back(SegReg); |
| 6577 | } |
| 6578 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6579 | Register DstReg = MI.getOperand(0).getReg(); |
Dominik Montada | 6b96623 | 2020-03-12 09:03:08 +0100 | [diff] [blame] | 6580 | if (MRI.getType(DstReg).isVector()) |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6581 | MIRBuilder.buildBuildVector(DstReg, DstRegs); |
Dominik Montada | 6b96623 | 2020-03-12 09:03:08 +0100 | [diff] [blame] | 6582 | else if (DstRegs.size() > 1) |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 6583 | MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); |
Dominik Montada | 6b96623 | 2020-03-12 09:03:08 +0100 | [diff] [blame] | 6584 | else |
| 6585 | MIRBuilder.buildCopy(DstReg, DstRegs[0]); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6586 | MI.eraseFromParent(); |
| 6587 | return Legalized; |
| 6588 | } |
| 6589 | |
| 6590 | LegalizerHelper::LegalizeResult |
| 6591 | LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, |
| 6592 | LLT NarrowTy) { |
| 6593 | // FIXME: Don't know how to handle secondary types yet. |
| 6594 | if (TypeIdx != 0) |
| 6595 | return UnableToLegalize; |
| 6596 | |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6597 | SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6598 | SmallVector<uint64_t, 2> Indexes; |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6599 | LLT RegTy = MRI.getType(MI.getOperand(0).getReg()); |
| 6600 | LLT LeftoverTy; |
| 6601 | extractParts(MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6602 | LeftoverRegs, MIRBuilder, MRI); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6603 | |
Craig Topper | e3284d8 | 2024-12-10 07:18:20 -0800 | [diff] [blame] | 6604 | SrcRegs.append(LeftoverRegs); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6605 | |
| 6606 | uint64_t NarrowSize = NarrowTy.getSizeInBits(); |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6607 | Register OpReg = MI.getOperand(2).getReg(); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6608 | uint64_t OpStart = MI.getOperand(3).getImm(); |
| 6609 | uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6610 | for (int I = 0, E = SrcRegs.size(); I != E; ++I) { |
| 6611 | unsigned DstStart = I * NarrowSize; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6612 | |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6613 | if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6614 | // The entire subregister is defined by this insert, forward the new |
| 6615 | // value. |
| 6616 | DstRegs.push_back(OpReg); |
| 6617 | continue; |
| 6618 | } |
| 6619 | |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6620 | Register SrcReg = SrcRegs[I]; |
| 6621 | if (MRI.getType(SrcRegs[I]) == LeftoverTy) { |
| 6622 | // The leftover reg is smaller than NarrowTy, so we need to extend it. |
| 6623 | SrcReg = MRI.createGenericVirtualRegister(NarrowTy); |
| 6624 | MIRBuilder.buildAnyExt(SrcReg, SrcRegs[I]); |
| 6625 | } |
| 6626 | |
| 6627 | if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { |
| 6628 | // No part of the insert affects this subregister, forward the original. |
| 6629 | DstRegs.push_back(SrcReg); |
| 6630 | continue; |
| 6631 | } |
| 6632 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6633 | // OpSegStart is where this destination segment would start in OpReg if it |
| 6634 | // extended infinitely in both directions. |
| 6635 | int64_t ExtractOffset, InsertOffset; |
| 6636 | uint64_t SegSize; |
| 6637 | if (OpStart < DstStart) { |
| 6638 | InsertOffset = 0; |
| 6639 | ExtractOffset = DstStart - OpStart; |
| 6640 | SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); |
| 6641 | } else { |
| 6642 | InsertOffset = OpStart - DstStart; |
| 6643 | ExtractOffset = 0; |
| 6644 | SegSize = |
| 6645 | std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); |
| 6646 | } |
| 6647 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6648 | Register SegReg = OpReg; |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6649 | if (ExtractOffset != 0 || SegSize != OpSize) { |
| 6650 | // A genuine extract is needed. |
| 6651 | SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); |
| 6652 | MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); |
| 6653 | } |
| 6654 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6655 | Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6656 | MIRBuilder.buildInsert(DstReg, SrcReg, SegReg, InsertOffset); |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6657 | DstRegs.push_back(DstReg); |
| 6658 | } |
| 6659 | |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6660 | uint64_t WideSize = DstRegs.size() * NarrowSize; |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6661 | Register DstReg = MI.getOperand(0).getReg(); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6662 | if (WideSize > RegTy.getSizeInBits()) { |
| 6663 | Register MergeReg = MRI.createGenericVirtualRegister(LLT::scalar(WideSize)); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 6664 | MIRBuilder.buildMergeLikeInstr(MergeReg, DstRegs); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6665 | MIRBuilder.buildTrunc(DstReg, MergeReg); |
| 6666 | } else |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 6667 | MIRBuilder.buildMergeLikeInstr(DstReg, DstRegs); |
Justin Bogner | 2a7e759 | 2021-03-02 09:49:15 -0800 | [diff] [blame] | 6668 | |
Matt Arsenault | 1cf71366 | 2019-02-12 14:54:52 +0000 | [diff] [blame] | 6669 | MI.eraseFromParent(); |
| 6670 | return Legalized; |
| 6671 | } |
| 6672 | |
Matt Arsenault | 211e89d | 2019-01-27 00:52:51 +0000 | [diff] [blame] | 6673 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6674 | LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, |
| 6675 | LLT NarrowTy) { |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6676 | Register DstReg = MI.getOperand(0).getReg(); |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6677 | LLT DstTy = MRI.getType(DstReg); |
| 6678 | |
| 6679 | assert(MI.getNumOperands() == 3 && TypeIdx == 0); |
| 6680 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 6681 | SmallVector<Register, 4> DstRegs, DstLeftoverRegs; |
| 6682 | SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs; |
| 6683 | SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6684 | LLT LeftoverTy; |
| 6685 | if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6686 | Src0Regs, Src0LeftoverRegs, MIRBuilder, MRI)) |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6687 | return UnableToLegalize; |
| 6688 | |
| 6689 | LLT Unused; |
| 6690 | if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6691 | Src1Regs, Src1LeftoverRegs, MIRBuilder, MRI)) |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6692 | llvm_unreachable("inconsistent extractParts result"); |
| 6693 | |
| 6694 | for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { |
| 6695 | auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, |
| 6696 | {Src0Regs[I], Src1Regs[I]}); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6697 | DstRegs.push_back(Inst.getReg(0)); |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6698 | } |
| 6699 | |
| 6700 | for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { |
| 6701 | auto Inst = MIRBuilder.buildInstr( |
| 6702 | MI.getOpcode(), |
| 6703 | {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]}); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6704 | DstLeftoverRegs.push_back(Inst.getReg(0)); |
Matt Arsenault | 9e0eeba | 2019-04-10 17:07:56 +0000 | [diff] [blame] | 6705 | } |
| 6706 | |
| 6707 | insertParts(DstReg, DstTy, NarrowTy, DstRegs, |
| 6708 | LeftoverTy, DstLeftoverRegs); |
| 6709 | |
| 6710 | MI.eraseFromParent(); |
| 6711 | return Legalized; |
| 6712 | } |
| 6713 | |
| 6714 | LegalizerHelper::LegalizeResult |
Matt Arsenault | be31a7b | 2020-01-10 11:02:18 -0500 | [diff] [blame] | 6715 | LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, |
| 6716 | LLT NarrowTy) { |
| 6717 | if (TypeIdx != 0) |
| 6718 | return UnableToLegalize; |
| 6719 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6720 | auto [DstReg, SrcReg] = MI.getFirst2Regs(); |
Matt Arsenault | be31a7b | 2020-01-10 11:02:18 -0500 | [diff] [blame] | 6721 | |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 6722 | LLT DstTy = MRI.getType(DstReg); |
| 6723 | if (DstTy.isVector()) |
Matt Arsenault | be31a7b | 2020-01-10 11:02:18 -0500 | [diff] [blame] | 6724 | return UnableToLegalize; |
| 6725 | |
Matt Arsenault | a66d281 | 2020-01-10 10:41:29 -0500 | [diff] [blame] | 6726 | SmallVector<Register, 8> Parts; |
| 6727 | LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg); |
Matt Arsenault | cd7650c | 2020-01-11 19:05:06 -0500 | [diff] [blame] | 6728 | LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode()); |
| 6729 | buildWidenedRemergeToDst(DstReg, LCMTy, Parts); |
| 6730 | |
Matt Arsenault | be31a7b | 2020-01-10 11:02:18 -0500 | [diff] [blame] | 6731 | MI.eraseFromParent(); |
| 6732 | return Legalized; |
| 6733 | } |
| 6734 | |
| 6735 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6736 | LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, |
| 6737 | LLT NarrowTy) { |
| 6738 | if (TypeIdx != 0) |
| 6739 | return UnableToLegalize; |
| 6740 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6741 | Register CondReg = MI.getOperand(1).getReg(); |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6742 | LLT CondTy = MRI.getType(CondReg); |
| 6743 | if (CondTy.isVector()) // TODO: Handle vselect |
| 6744 | return UnableToLegalize; |
| 6745 | |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6746 | Register DstReg = MI.getOperand(0).getReg(); |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6747 | LLT DstTy = MRI.getType(DstReg); |
| 6748 | |
Matt Arsenault | e3a676e | 2019-06-24 15:50:29 +0000 | [diff] [blame] | 6749 | SmallVector<Register, 4> DstRegs, DstLeftoverRegs; |
| 6750 | SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs; |
| 6751 | SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs; |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6752 | LLT LeftoverTy; |
| 6753 | if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6754 | Src1Regs, Src1LeftoverRegs, MIRBuilder, MRI)) |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6755 | return UnableToLegalize; |
| 6756 | |
| 6757 | LLT Unused; |
| 6758 | if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 6759 | Src2Regs, Src2LeftoverRegs, MIRBuilder, MRI)) |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6760 | llvm_unreachable("inconsistent extractParts result"); |
| 6761 | |
| 6762 | for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { |
| 6763 | auto Select = MIRBuilder.buildSelect(NarrowTy, |
| 6764 | CondReg, Src1Regs[I], Src2Regs[I]); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6765 | DstRegs.push_back(Select.getReg(0)); |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6766 | } |
| 6767 | |
| 6768 | for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { |
| 6769 | auto Select = MIRBuilder.buildSelect( |
| 6770 | LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6771 | DstLeftoverRegs.push_back(Select.getReg(0)); |
Matt Arsenault | 81511e5 | 2019-02-05 00:13:44 +0000 | [diff] [blame] | 6772 | } |
| 6773 | |
| 6774 | insertParts(DstReg, DstTy, NarrowTy, DstRegs, |
| 6775 | LeftoverTy, DstLeftoverRegs); |
| 6776 | |
| 6777 | MI.eraseFromParent(); |
| 6778 | return Legalized; |
| 6779 | } |
| 6780 | |
| 6781 | LegalizerHelper::LegalizeResult |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 6782 | LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, |
| 6783 | LLT NarrowTy) { |
| 6784 | if (TypeIdx != 1) |
| 6785 | return UnableToLegalize; |
| 6786 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6787 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 6788 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
| 6789 | |
| 6790 | if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 6791 | const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF; |
| 6792 | |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 6793 | MachineIRBuilder &B = MIRBuilder; |
Matt Arsenault | 6135f5e | 2020-02-07 11:55:39 -0500 | [diff] [blame] | 6794 | auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 6795 | // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi) |
| 6796 | auto C_0 = B.buildConstant(NarrowTy, 0); |
| 6797 | auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), |
| 6798 | UnmergeSrc.getReg(1), C_0); |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 6799 | auto LoCTLZ = IsUndef ? |
| 6800 | B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) : |
| 6801 | B.buildCTLZ(DstTy, UnmergeSrc.getReg(0)); |
Matt Arsenault | 6135f5e | 2020-02-07 11:55:39 -0500 | [diff] [blame] | 6802 | auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); |
| 6803 | auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize); |
| 6804 | auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)); |
| 6805 | B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ); |
Petar Avramovic | 2b66d32 | 2020-01-27 09:43:38 +0100 | [diff] [blame] | 6806 | |
| 6807 | MI.eraseFromParent(); |
| 6808 | return Legalized; |
| 6809 | } |
| 6810 | |
| 6811 | return UnableToLegalize; |
| 6812 | } |
| 6813 | |
| 6814 | LegalizerHelper::LegalizeResult |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 6815 | LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, |
| 6816 | LLT NarrowTy) { |
| 6817 | if (TypeIdx != 1) |
| 6818 | return UnableToLegalize; |
| 6819 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6820 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 6821 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
| 6822 | |
| 6823 | if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 6824 | const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF; |
| 6825 | |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 6826 | MachineIRBuilder &B = MIRBuilder; |
Matt Arsenault | 6135f5e | 2020-02-07 11:55:39 -0500 | [diff] [blame] | 6827 | auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg); |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 6828 | // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo) |
| 6829 | auto C_0 = B.buildConstant(NarrowTy, 0); |
| 6830 | auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), |
| 6831 | UnmergeSrc.getReg(0), C_0); |
Matt Arsenault | 312a9d1 | 2020-02-07 12:24:15 -0500 | [diff] [blame] | 6832 | auto HiCTTZ = IsUndef ? |
| 6833 | B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) : |
| 6834 | B.buildCTTZ(DstTy, UnmergeSrc.getReg(1)); |
Matt Arsenault | 6135f5e | 2020-02-07 11:55:39 -0500 | [diff] [blame] | 6835 | auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize); |
| 6836 | auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize); |
| 6837 | auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)); |
| 6838 | B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ); |
Petar Avramovic | 8bc7ba5 | 2020-01-27 09:51:06 +0100 | [diff] [blame] | 6839 | |
| 6840 | MI.eraseFromParent(); |
| 6841 | return Legalized; |
| 6842 | } |
| 6843 | |
| 6844 | return UnableToLegalize; |
| 6845 | } |
| 6846 | |
| 6847 | LegalizerHelper::LegalizeResult |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 6848 | LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, |
| 6849 | LLT NarrowTy) { |
| 6850 | if (TypeIdx != 1) |
| 6851 | return UnableToLegalize; |
| 6852 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6853 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 6854 | unsigned NarrowSize = NarrowTy.getSizeInBits(); |
| 6855 | |
| 6856 | if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) { |
| 6857 | auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1)); |
| 6858 | |
Matt Arsenault | 3b19851 | 2020-02-06 22:29:23 -0500 | [diff] [blame] | 6859 | auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0)); |
| 6860 | auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1)); |
Jon Roelofs | f2e8e46 | 2021-07-26 16:42:20 -0700 | [diff] [blame] | 6861 | MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 6862 | |
| 6863 | MI.eraseFromParent(); |
| 6864 | return Legalized; |
| 6865 | } |
| 6866 | |
| 6867 | return UnableToLegalize; |
| 6868 | } |
| 6869 | |
| 6870 | LegalizerHelper::LegalizeResult |
Matt Arsenault | eece6ba | 2023-04-26 22:02:42 -0400 | [diff] [blame] | 6871 | LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, |
| 6872 | LLT NarrowTy) { |
| 6873 | if (TypeIdx != 1) |
| 6874 | return UnableToLegalize; |
| 6875 | |
| 6876 | MachineIRBuilder &B = MIRBuilder; |
| 6877 | Register ExpReg = MI.getOperand(2).getReg(); |
| 6878 | LLT ExpTy = MRI.getType(ExpReg); |
| 6879 | |
| 6880 | unsigned ClampSize = NarrowTy.getScalarSizeInBits(); |
| 6881 | |
| 6882 | // Clamp the exponent to the range of the target type. |
| 6883 | auto MinExp = B.buildConstant(ExpTy, minIntN(ClampSize)); |
| 6884 | auto ClampMin = B.buildSMax(ExpTy, ExpReg, MinExp); |
| 6885 | auto MaxExp = B.buildConstant(ExpTy, maxIntN(ClampSize)); |
| 6886 | auto Clamp = B.buildSMin(ExpTy, ClampMin, MaxExp); |
| 6887 | |
| 6888 | auto Trunc = B.buildTrunc(NarrowTy, Clamp); |
| 6889 | Observer.changingInstr(MI); |
| 6890 | MI.getOperand(2).setReg(Trunc.getReg(0)); |
| 6891 | Observer.changedInstr(MI); |
| 6892 | return Legalized; |
| 6893 | } |
| 6894 | |
| 6895 | LegalizerHelper::LegalizeResult |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 6896 | LegalizerHelper::lowerBitCount(MachineInstr &MI) { |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6897 | unsigned Opc = MI.getOpcode(); |
Matt Arsenault | a679f27 | 2020-07-19 12:29:48 -0400 | [diff] [blame] | 6898 | const auto &TII = MIRBuilder.getTII(); |
Diana Picus | 0528e2c | 2018-11-26 11:07:02 +0000 | [diff] [blame] | 6899 | auto isSupported = [this](const LegalityQuery &Q) { |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6900 | auto QAction = LI.getAction(Q).Action; |
Diana Picus | 0528e2c | 2018-11-26 11:07:02 +0000 | [diff] [blame] | 6901 | return QAction == Legal || QAction == Libcall || QAction == Custom; |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6902 | }; |
| 6903 | switch (Opc) { |
| 6904 | default: |
| 6905 | return UnableToLegalize; |
| 6906 | case TargetOpcode::G_CTLZ_ZERO_UNDEF: { |
| 6907 | // This trivially expands to CTLZ. |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 6908 | Observer.changingInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6909 | MI.setDesc(TII.get(TargetOpcode::G_CTLZ)); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 6910 | Observer.changedInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6911 | return Legalized; |
| 6912 | } |
| 6913 | case TargetOpcode::G_CTLZ: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6914 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6915 | unsigned Len = SrcTy.getSizeInBits(); |
| 6916 | |
| 6917 | if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) { |
Diana Picus | 0528e2c | 2018-11-26 11:07:02 +0000 | [diff] [blame] | 6918 | // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero. |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6919 | auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg); |
| 6920 | auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0); |
| 6921 | auto ICmp = MIRBuilder.buildICmp( |
| 6922 | CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc); |
| 6923 | auto LenConst = MIRBuilder.buildConstant(DstTy, Len); |
| 6924 | MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6925 | MI.eraseFromParent(); |
| 6926 | return Legalized; |
| 6927 | } |
| 6928 | // for now, we do this: |
| 6929 | // NewLen = NextPowerOf2(Len); |
| 6930 | // x = x | (x >> 1); |
| 6931 | // x = x | (x >> 2); |
| 6932 | // ... |
| 6933 | // x = x | (x >>16); |
| 6934 | // x = x | (x >>32); // for 64-bit input |
| 6935 | // Upto NewLen/2 |
| 6936 | // return Len - popcount(x); |
| 6937 | // |
| 6938 | // Ref: "Hacker's Delight" by Henry Warren |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 6939 | Register Op = SrcReg; |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6940 | unsigned NewLen = PowerOf2Ceil(Len); |
| 6941 | for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) { |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6942 | auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i); |
| 6943 | auto MIBOp = MIRBuilder.buildOr( |
| 6944 | SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt)); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6945 | Op = MIBOp.getReg(0); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6946 | } |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6947 | auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op); |
| 6948 | MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len), |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 6949 | MIBPop); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6950 | MI.eraseFromParent(); |
| 6951 | return Legalized; |
| 6952 | } |
| 6953 | case TargetOpcode::G_CTTZ_ZERO_UNDEF: { |
| 6954 | // This trivially expands to CTTZ. |
Daniel Sanders | d001e0e | 2018-12-12 23:48:13 +0000 | [diff] [blame] | 6955 | Observer.changingInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6956 | MI.setDesc(TII.get(TargetOpcode::G_CTTZ)); |
Aditya Nandakumar | f75d4f3 | 2018-12-05 20:14:52 +0000 | [diff] [blame] | 6957 | Observer.changedInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6958 | return Legalized; |
| 6959 | } |
| 6960 | case TargetOpcode::G_CTTZ: { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 6961 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6962 | |
| 6963 | unsigned Len = SrcTy.getSizeInBits(); |
| 6964 | if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) { |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6965 | // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with |
| 6966 | // zero. |
Matt Arsenault | 8de2dad | 2020-02-06 21:11:52 -0500 | [diff] [blame] | 6967 | auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg); |
| 6968 | auto Zero = MIRBuilder.buildConstant(SrcTy, 0); |
| 6969 | auto ICmp = MIRBuilder.buildICmp( |
| 6970 | CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero); |
| 6971 | auto LenConst = MIRBuilder.buildConstant(DstTy, Len); |
| 6972 | MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6973 | MI.eraseFromParent(); |
| 6974 | return Legalized; |
| 6975 | } |
| 6976 | // for now, we use: { return popcount(~x & (x - 1)); } |
| 6977 | // unless the target has ctlz but not ctpop, in which case we use: |
| 6978 | // { return 32 - nlz(~x & (x-1)); } |
| 6979 | // Ref: "Hacker's Delight" by Henry Warren |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 6980 | auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1); |
| 6981 | auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1); |
Jay Foad | 28bb43b | 2020-01-16 12:09:48 +0000 | [diff] [blame] | 6982 | auto MIBTmp = MIRBuilder.buildAnd( |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 6983 | SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1)); |
| 6984 | if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) && |
| 6985 | isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) { |
| 6986 | auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len); |
Jay Foad | 63f7354 | 2020-01-16 12:37:00 +0000 | [diff] [blame] | 6987 | MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen, |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 6988 | MIRBuilder.buildCTLZ(SrcTy, MIBTmp)); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6989 | MI.eraseFromParent(); |
| 6990 | return Legalized; |
| 6991 | } |
Craig Topper | 44e8bea | 2023-11-12 19:36:24 -0800 | [diff] [blame] | 6992 | Observer.changingInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6993 | MI.setDesc(TII.get(TargetOpcode::G_CTPOP)); |
Jay Foad | b482e1b | 2020-01-23 11:51:35 +0000 | [diff] [blame] | 6994 | MI.getOperand(1).setReg(MIBTmp.getReg(0)); |
Craig Topper | 44e8bea | 2023-11-12 19:36:24 -0800 | [diff] [blame] | 6995 | Observer.changedInstr(MI); |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 6996 | return Legalized; |
| 6997 | } |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 6998 | case TargetOpcode::G_CTPOP: { |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 6999 | Register SrcReg = MI.getOperand(1).getReg(); |
| 7000 | LLT Ty = MRI.getType(SrcReg); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7001 | unsigned Size = Ty.getSizeInBits(); |
| 7002 | MachineIRBuilder &B = MIRBuilder; |
| 7003 | |
| 7004 | // Count set bits in blocks of 2 bits. Default approach would be |
| 7005 | // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 } |
| 7006 | // We use following formula instead: |
| 7007 | // B2Count = val - { (val >> 1) & 0x55555555 } |
| 7008 | // since it gives same result in blocks of 2 with one instruction less. |
| 7009 | auto C_1 = B.buildConstant(Ty, 1); |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7010 | auto B2Set1LoTo1Hi = B.buildLShr(Ty, SrcReg, C_1); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7011 | APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55)); |
| 7012 | auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0); |
| 7013 | auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0); |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7014 | auto B2Count = B.buildSub(Ty, SrcReg, B2Count1Hi); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7015 | |
| 7016 | // In order to get count in blocks of 4 add values from adjacent block of 2. |
| 7017 | // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 } |
| 7018 | auto C_2 = B.buildConstant(Ty, 2); |
| 7019 | auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2); |
| 7020 | APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33)); |
| 7021 | auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0); |
| 7022 | auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0); |
| 7023 | auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0); |
| 7024 | auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count); |
| 7025 | |
| 7026 | // For count in blocks of 8 bits we don't have to mask high 4 bits before |
| 7027 | // addition since count value sits in range {0,...,8} and 4 bits are enough |
| 7028 | // to hold such binary values. After addition high 4 bits still hold count |
| 7029 | // of set bits in high 4 bit block, set them to zero and get 8 bit result. |
| 7030 | // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F |
| 7031 | auto C_4 = B.buildConstant(Ty, 4); |
| 7032 | auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4); |
| 7033 | auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count); |
| 7034 | APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F)); |
| 7035 | auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0); |
| 7036 | auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0); |
| 7037 | |
| 7038 | assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm"); |
| 7039 | // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this |
| 7040 | // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks. |
| 7041 | auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01))); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7042 | |
| 7043 | // Shift count result from 8 high bits to low bits. |
| 7044 | auto C_SizeM8 = B.buildConstant(Ty, Size - 8); |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7045 | |
Wang Pengcheng | 610b9e2 | 2024-03-29 15:38:39 +0800 | [diff] [blame] | 7046 | auto IsMulSupported = [this](const LLT Ty) { |
| 7047 | auto Action = LI.getAction({TargetOpcode::G_MUL, {Ty}}).Action; |
| 7048 | return Action == Legal || Action == WidenScalar || Action == Custom; |
| 7049 | }; |
| 7050 | if (IsMulSupported(Ty)) { |
| 7051 | auto ResTmp = B.buildMul(Ty, B8Count, MulMask); |
| 7052 | B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8); |
| 7053 | } else { |
| 7054 | auto ResTmp = B8Count; |
| 7055 | for (unsigned Shift = 8; Shift < Size; Shift *= 2) { |
| 7056 | auto ShiftC = B.buildConstant(Ty, Shift); |
| 7057 | auto Shl = B.buildShl(Ty, ResTmp, ShiftC); |
| 7058 | ResTmp = B.buildAdd(Ty, ResTmp, Shl); |
| 7059 | } |
| 7060 | B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8); |
| 7061 | } |
Petar Avramovic | cbf03aee | 2020-01-27 09:59:50 +0100 | [diff] [blame] | 7062 | MI.eraseFromParent(); |
| 7063 | return Legalized; |
| 7064 | } |
Aditya Nandakumar | c0333f7 | 2018-08-21 17:30:31 +0000 | [diff] [blame] | 7065 | } |
| 7066 | } |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7067 | |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7068 | // Check that (every element of) Reg is undef or not an exact multiple of BW. |
| 7069 | static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, |
| 7070 | Register Reg, unsigned BW) { |
| 7071 | return matchUnaryPredicate( |
| 7072 | MRI, Reg, |
| 7073 | [=](const Constant *C) { |
| 7074 | // Null constant here means an undef. |
| 7075 | const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(C); |
| 7076 | return !CI || CI->getValue().urem(BW) != 0; |
| 7077 | }, |
| 7078 | /*AllowUndefs*/ true); |
| 7079 | } |
| 7080 | |
| 7081 | LegalizerHelper::LegalizeResult |
| 7082 | LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7083 | auto [Dst, X, Y, Z] = MI.getFirst4Regs(); |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7084 | LLT Ty = MRI.getType(Dst); |
| 7085 | LLT ShTy = MRI.getType(Z); |
| 7086 | |
| 7087 | unsigned BW = Ty.getScalarSizeInBits(); |
Matt Arsenault | 14b03b4 | 2021-03-29 17:26:49 -0400 | [diff] [blame] | 7088 | |
| 7089 | if (!isPowerOf2_32(BW)) |
| 7090 | return UnableToLegalize; |
| 7091 | |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7092 | const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; |
| 7093 | unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL; |
| 7094 | |
| 7095 | if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) { |
| 7096 | // fshl X, Y, Z -> fshr X, Y, -Z |
| 7097 | // fshr X, Y, Z -> fshl X, Y, -Z |
| 7098 | auto Zero = MIRBuilder.buildConstant(ShTy, 0); |
| 7099 | Z = MIRBuilder.buildSub(Ty, Zero, Z).getReg(0); |
| 7100 | } else { |
| 7101 | // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z |
| 7102 | // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z |
| 7103 | auto One = MIRBuilder.buildConstant(ShTy, 1); |
| 7104 | if (IsFSHL) { |
| 7105 | Y = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0); |
| 7106 | X = MIRBuilder.buildLShr(Ty, X, One).getReg(0); |
| 7107 | } else { |
| 7108 | X = MIRBuilder.buildInstr(RevOpcode, {Ty}, {X, Y, One}).getReg(0); |
| 7109 | Y = MIRBuilder.buildShl(Ty, Y, One).getReg(0); |
| 7110 | } |
| 7111 | |
| 7112 | Z = MIRBuilder.buildNot(ShTy, Z).getReg(0); |
| 7113 | } |
| 7114 | |
| 7115 | MIRBuilder.buildInstr(RevOpcode, {Dst}, {X, Y, Z}); |
| 7116 | MI.eraseFromParent(); |
| 7117 | return Legalized; |
| 7118 | } |
| 7119 | |
| 7120 | LegalizerHelper::LegalizeResult |
| 7121 | LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7122 | auto [Dst, X, Y, Z] = MI.getFirst4Regs(); |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7123 | LLT Ty = MRI.getType(Dst); |
| 7124 | LLT ShTy = MRI.getType(Z); |
| 7125 | |
| 7126 | const unsigned BW = Ty.getScalarSizeInBits(); |
| 7127 | const bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; |
| 7128 | |
| 7129 | Register ShX, ShY; |
| 7130 | Register ShAmt, InvShAmt; |
| 7131 | |
| 7132 | // FIXME: Emit optimized urem by constant instead of letting it expand later. |
| 7133 | if (isNonZeroModBitWidthOrUndef(MRI, Z, BW)) { |
| 7134 | // fshl: X << C | Y >> (BW - C) |
| 7135 | // fshr: X << (BW - C) | Y >> C |
| 7136 | // where C = Z % BW is not zero |
| 7137 | auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW); |
| 7138 | ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0); |
| 7139 | InvShAmt = MIRBuilder.buildSub(ShTy, BitWidthC, ShAmt).getReg(0); |
| 7140 | ShX = MIRBuilder.buildShl(Ty, X, IsFSHL ? ShAmt : InvShAmt).getReg(0); |
| 7141 | ShY = MIRBuilder.buildLShr(Ty, Y, IsFSHL ? InvShAmt : ShAmt).getReg(0); |
| 7142 | } else { |
| 7143 | // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW)) |
| 7144 | // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW) |
| 7145 | auto Mask = MIRBuilder.buildConstant(ShTy, BW - 1); |
| 7146 | if (isPowerOf2_32(BW)) { |
| 7147 | // Z % BW -> Z & (BW - 1) |
| 7148 | ShAmt = MIRBuilder.buildAnd(ShTy, Z, Mask).getReg(0); |
| 7149 | // (BW - 1) - (Z % BW) -> ~Z & (BW - 1) |
| 7150 | auto NotZ = MIRBuilder.buildNot(ShTy, Z); |
| 7151 | InvShAmt = MIRBuilder.buildAnd(ShTy, NotZ, Mask).getReg(0); |
| 7152 | } else { |
| 7153 | auto BitWidthC = MIRBuilder.buildConstant(ShTy, BW); |
| 7154 | ShAmt = MIRBuilder.buildURem(ShTy, Z, BitWidthC).getReg(0); |
| 7155 | InvShAmt = MIRBuilder.buildSub(ShTy, Mask, ShAmt).getReg(0); |
| 7156 | } |
| 7157 | |
| 7158 | auto One = MIRBuilder.buildConstant(ShTy, 1); |
| 7159 | if (IsFSHL) { |
| 7160 | ShX = MIRBuilder.buildShl(Ty, X, ShAmt).getReg(0); |
| 7161 | auto ShY1 = MIRBuilder.buildLShr(Ty, Y, One); |
| 7162 | ShY = MIRBuilder.buildLShr(Ty, ShY1, InvShAmt).getReg(0); |
| 7163 | } else { |
| 7164 | auto ShX1 = MIRBuilder.buildShl(Ty, X, One); |
| 7165 | ShX = MIRBuilder.buildShl(Ty, ShX1, InvShAmt).getReg(0); |
| 7166 | ShY = MIRBuilder.buildLShr(Ty, Y, ShAmt).getReg(0); |
| 7167 | } |
| 7168 | } |
| 7169 | |
| 7170 | MIRBuilder.buildOr(Dst, ShX, ShY); |
| 7171 | MI.eraseFromParent(); |
| 7172 | return Legalized; |
| 7173 | } |
| 7174 | |
| 7175 | LegalizerHelper::LegalizeResult |
| 7176 | LegalizerHelper::lowerFunnelShift(MachineInstr &MI) { |
| 7177 | // These operations approximately do the following (while avoiding undefined |
| 7178 | // shifts by BW): |
| 7179 | // G_FSHL: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) |
| 7180 | // G_FSHR: (X << (BW - (Z % BW))) | (Y >> (Z % BW)) |
| 7181 | Register Dst = MI.getOperand(0).getReg(); |
| 7182 | LLT Ty = MRI.getType(Dst); |
| 7183 | LLT ShTy = MRI.getType(MI.getOperand(3).getReg()); |
| 7184 | |
| 7185 | bool IsFSHL = MI.getOpcode() == TargetOpcode::G_FSHL; |
| 7186 | unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL; |
Matt Arsenault | 14b03b4 | 2021-03-29 17:26:49 -0400 | [diff] [blame] | 7187 | |
| 7188 | // TODO: Use smarter heuristic that accounts for vector legalization. |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7189 | if (LI.getAction({RevOpcode, {Ty, ShTy}}).Action == Lower) |
| 7190 | return lowerFunnelShiftAsShifts(MI); |
Matt Arsenault | 14b03b4 | 2021-03-29 17:26:49 -0400 | [diff] [blame] | 7191 | |
| 7192 | // This only works for powers of 2, fallback to shifts if it fails. |
| 7193 | LegalizerHelper::LegalizeResult Result = lowerFunnelShiftWithInverse(MI); |
| 7194 | if (Result == UnableToLegalize) |
| 7195 | return lowerFunnelShiftAsShifts(MI); |
| 7196 | return Result; |
Matt Arsenault | b24436a | 2020-03-19 22:48:13 -0400 | [diff] [blame] | 7197 | } |
| 7198 | |
Tuan Chuong Goh | a40c984 | 2023-08-17 16:31:54 +0100 | [diff] [blame] | 7199 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) { |
| 7200 | auto [Dst, Src] = MI.getFirst2Regs(); |
| 7201 | LLT DstTy = MRI.getType(Dst); |
| 7202 | LLT SrcTy = MRI.getType(Src); |
| 7203 | |
| 7204 | uint32_t DstTySize = DstTy.getSizeInBits(); |
| 7205 | uint32_t DstTyScalarSize = DstTy.getScalarSizeInBits(); |
| 7206 | uint32_t SrcTyScalarSize = SrcTy.getScalarSizeInBits(); |
| 7207 | |
| 7208 | if (!isPowerOf2_32(DstTySize) || !isPowerOf2_32(DstTyScalarSize) || |
| 7209 | !isPowerOf2_32(SrcTyScalarSize)) |
| 7210 | return UnableToLegalize; |
| 7211 | |
| 7212 | // The step between extend is too large, split it by creating an intermediate |
| 7213 | // extend instruction |
| 7214 | if (SrcTyScalarSize * 2 < DstTyScalarSize) { |
| 7215 | LLT MidTy = SrcTy.changeElementSize(SrcTyScalarSize * 2); |
| 7216 | // If the destination type is illegal, split it into multiple statements |
| 7217 | // zext x -> zext(merge(zext(unmerge), zext(unmerge))) |
| 7218 | auto NewExt = MIRBuilder.buildInstr(MI.getOpcode(), {MidTy}, {Src}); |
| 7219 | // Unmerge the vector |
| 7220 | LLT EltTy = MidTy.changeElementCount( |
| 7221 | MidTy.getElementCount().divideCoefficientBy(2)); |
| 7222 | auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, NewExt); |
| 7223 | |
| 7224 | // ZExt the vectors |
| 7225 | LLT ZExtResTy = DstTy.changeElementCount( |
| 7226 | DstTy.getElementCount().divideCoefficientBy(2)); |
| 7227 | auto ZExtRes1 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, |
| 7228 | {UnmergeSrc.getReg(0)}); |
| 7229 | auto ZExtRes2 = MIRBuilder.buildInstr(MI.getOpcode(), {ZExtResTy}, |
| 7230 | {UnmergeSrc.getReg(1)}); |
| 7231 | |
| 7232 | // Merge the ending vectors |
| 7233 | MIRBuilder.buildMergeLikeInstr(Dst, {ZExtRes1, ZExtRes2}); |
| 7234 | |
| 7235 | MI.eraseFromParent(); |
| 7236 | return Legalized; |
| 7237 | } |
| 7238 | return UnableToLegalize; |
| 7239 | } |
| 7240 | |
chuongg3 | d88d983 | 2023-10-11 16:05:25 +0100 | [diff] [blame] | 7241 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) { |
| 7242 | // MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; |
| 7243 | MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); |
| 7244 | // Similar to how operand splitting is done in SelectiondDAG, we can handle |
| 7245 | // %res(v8s8) = G_TRUNC %in(v8s32) by generating: |
| 7246 | // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>) |
| 7247 | // %lo16(<4 x s16>) = G_TRUNC %inlo |
| 7248 | // %hi16(<4 x s16>) = G_TRUNC %inhi |
| 7249 | // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16 |
| 7250 | // %res(<8 x s8>) = G_TRUNC %in16 |
| 7251 | |
| 7252 | assert(MI.getOpcode() == TargetOpcode::G_TRUNC); |
| 7253 | |
| 7254 | Register DstReg = MI.getOperand(0).getReg(); |
| 7255 | Register SrcReg = MI.getOperand(1).getReg(); |
| 7256 | LLT DstTy = MRI.getType(DstReg); |
| 7257 | LLT SrcTy = MRI.getType(SrcReg); |
| 7258 | |
| 7259 | if (DstTy.isVector() && isPowerOf2_32(DstTy.getNumElements()) && |
| 7260 | isPowerOf2_32(DstTy.getScalarSizeInBits()) && |
| 7261 | isPowerOf2_32(SrcTy.getNumElements()) && |
| 7262 | isPowerOf2_32(SrcTy.getScalarSizeInBits())) { |
| 7263 | // Split input type. |
| 7264 | LLT SplitSrcTy = SrcTy.changeElementCount( |
| 7265 | SrcTy.getElementCount().divideCoefficientBy(2)); |
| 7266 | |
| 7267 | // First, split the source into two smaller vectors. |
| 7268 | SmallVector<Register, 2> SplitSrcs; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 7269 | extractParts(SrcReg, SplitSrcTy, 2, SplitSrcs, MIRBuilder, MRI); |
chuongg3 | d88d983 | 2023-10-11 16:05:25 +0100 | [diff] [blame] | 7270 | |
| 7271 | // Truncate the splits into intermediate narrower elements. |
| 7272 | LLT InterTy; |
| 7273 | if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits()) |
| 7274 | InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2); |
| 7275 | else |
| 7276 | InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits()); |
| 7277 | for (unsigned I = 0; I < SplitSrcs.size(); ++I) { |
| 7278 | SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0); |
| 7279 | } |
| 7280 | |
| 7281 | // Combine the new truncates into one vector |
| 7282 | auto Merge = MIRBuilder.buildMergeLikeInstr( |
| 7283 | DstTy.changeElementSize(InterTy.getScalarSizeInBits()), SplitSrcs); |
| 7284 | |
| 7285 | // Truncate the new vector to the final result type |
| 7286 | if (DstTy.getScalarSizeInBits() * 2 < SrcTy.getScalarSizeInBits()) |
| 7287 | MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), Merge.getReg(0)); |
| 7288 | else |
| 7289 | MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Merge.getReg(0)); |
| 7290 | |
| 7291 | MI.eraseFromParent(); |
| 7292 | |
| 7293 | return Legalized; |
| 7294 | } |
| 7295 | return UnableToLegalize; |
| 7296 | } |
| 7297 | |
Amara Emerson | f5e9be6 | 2021-03-26 15:27:15 -0700 | [diff] [blame] | 7298 | LegalizerHelper::LegalizeResult |
| 7299 | LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7300 | auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs(); |
Amara Emerson | f5e9be6 | 2021-03-26 15:27:15 -0700 | [diff] [blame] | 7301 | auto Zero = MIRBuilder.buildConstant(AmtTy, 0); |
| 7302 | bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL; |
| 7303 | unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL; |
| 7304 | auto Neg = MIRBuilder.buildSub(AmtTy, Zero, Amt); |
| 7305 | MIRBuilder.buildInstr(RevRot, {Dst}, {Src, Neg}); |
| 7306 | MI.eraseFromParent(); |
| 7307 | return Legalized; |
| 7308 | } |
| 7309 | |
| 7310 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7311 | auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] = MI.getFirst3RegLLTs(); |
Amara Emerson | f5e9be6 | 2021-03-26 15:27:15 -0700 | [diff] [blame] | 7312 | |
| 7313 | unsigned EltSizeInBits = DstTy.getScalarSizeInBits(); |
| 7314 | bool IsLeft = MI.getOpcode() == TargetOpcode::G_ROTL; |
| 7315 | |
| 7316 | MIRBuilder.setInstrAndDebugLoc(MI); |
| 7317 | |
| 7318 | // If a rotate in the other direction is supported, use it. |
| 7319 | unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL; |
| 7320 | if (LI.isLegalOrCustom({RevRot, {DstTy, SrcTy}}) && |
| 7321 | isPowerOf2_32(EltSizeInBits)) |
| 7322 | return lowerRotateWithReverseRotate(MI); |
| 7323 | |
Mirko Brkusanin | 5263bf5 | 2021-09-07 16:18:19 +0200 | [diff] [blame] | 7324 | // If a funnel shift is supported, use it. |
| 7325 | unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR; |
| 7326 | unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR; |
| 7327 | bool IsFShLegal = false; |
| 7328 | if ((IsFShLegal = LI.isLegalOrCustom({FShOpc, {DstTy, AmtTy}})) || |
| 7329 | LI.isLegalOrCustom({RevFsh, {DstTy, AmtTy}})) { |
| 7330 | auto buildFunnelShift = [&](unsigned Opc, Register R1, Register R2, |
| 7331 | Register R3) { |
| 7332 | MIRBuilder.buildInstr(Opc, {R1}, {R2, R2, R3}); |
| 7333 | MI.eraseFromParent(); |
| 7334 | return Legalized; |
| 7335 | }; |
| 7336 | // If a funnel shift in the other direction is supported, use it. |
| 7337 | if (IsFShLegal) { |
| 7338 | return buildFunnelShift(FShOpc, Dst, Src, Amt); |
| 7339 | } else if (isPowerOf2_32(EltSizeInBits)) { |
| 7340 | Amt = MIRBuilder.buildNeg(DstTy, Amt).getReg(0); |
| 7341 | return buildFunnelShift(RevFsh, Dst, Src, Amt); |
| 7342 | } |
| 7343 | } |
| 7344 | |
Amara Emerson | f5e9be6 | 2021-03-26 15:27:15 -0700 | [diff] [blame] | 7345 | auto Zero = MIRBuilder.buildConstant(AmtTy, 0); |
| 7346 | unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR; |
| 7347 | unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL; |
| 7348 | auto BitWidthMinusOneC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits - 1); |
| 7349 | Register ShVal; |
| 7350 | Register RevShiftVal; |
| 7351 | if (isPowerOf2_32(EltSizeInBits)) { |
| 7352 | // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1)) |
| 7353 | // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1)) |
| 7354 | auto NegAmt = MIRBuilder.buildSub(AmtTy, Zero, Amt); |
| 7355 | auto ShAmt = MIRBuilder.buildAnd(AmtTy, Amt, BitWidthMinusOneC); |
| 7356 | ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0); |
| 7357 | auto RevAmt = MIRBuilder.buildAnd(AmtTy, NegAmt, BitWidthMinusOneC); |
| 7358 | RevShiftVal = |
| 7359 | MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, RevAmt}).getReg(0); |
| 7360 | } else { |
| 7361 | // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w)) |
| 7362 | // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w)) |
| 7363 | auto BitWidthC = MIRBuilder.buildConstant(AmtTy, EltSizeInBits); |
| 7364 | auto ShAmt = MIRBuilder.buildURem(AmtTy, Amt, BitWidthC); |
| 7365 | ShVal = MIRBuilder.buildInstr(ShOpc, {DstTy}, {Src, ShAmt}).getReg(0); |
| 7366 | auto RevAmt = MIRBuilder.buildSub(AmtTy, BitWidthMinusOneC, ShAmt); |
| 7367 | auto One = MIRBuilder.buildConstant(AmtTy, 1); |
| 7368 | auto Inner = MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Src, One}); |
| 7369 | RevShiftVal = |
| 7370 | MIRBuilder.buildInstr(RevShiftOpc, {DstTy}, {Inner, RevAmt}).getReg(0); |
| 7371 | } |
| 7372 | MIRBuilder.buildOr(Dst, ShVal, RevShiftVal); |
| 7373 | MI.eraseFromParent(); |
| 7374 | return Legalized; |
| 7375 | } |
| 7376 | |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7377 | // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float |
| 7378 | // representation. |
| 7379 | LegalizerHelper::LegalizeResult |
| 7380 | LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7381 | auto [Dst, Src] = MI.getFirst2Regs(); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7382 | const LLT S64 = LLT::scalar(64); |
| 7383 | const LLT S32 = LLT::scalar(32); |
| 7384 | const LLT S1 = LLT::scalar(1); |
| 7385 | |
| 7386 | assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); |
| 7387 | |
| 7388 | // unsigned cul2f(ulong u) { |
| 7389 | // uint lz = clz(u); |
| 7390 | // uint e = (u != 0) ? 127U + 63U - lz : 0; |
| 7391 | // u = (u << lz) & 0x7fffffffffffffffUL; |
| 7392 | // ulong t = u & 0xffffffffffUL; |
| 7393 | // uint v = (e << 23) | (uint)(u >> 40); |
| 7394 | // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U); |
| 7395 | // return as_float(v + r); |
| 7396 | // } |
| 7397 | |
| 7398 | auto Zero32 = MIRBuilder.buildConstant(S32, 0); |
| 7399 | auto Zero64 = MIRBuilder.buildConstant(S64, 0); |
| 7400 | |
| 7401 | auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); |
| 7402 | |
| 7403 | auto K = MIRBuilder.buildConstant(S32, 127U + 63U); |
| 7404 | auto Sub = MIRBuilder.buildSub(S32, K, LZ); |
| 7405 | |
| 7406 | auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); |
| 7407 | auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); |
| 7408 | |
| 7409 | auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); |
| 7410 | auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); |
| 7411 | |
| 7412 | auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); |
| 7413 | |
| 7414 | auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); |
| 7415 | auto T = MIRBuilder.buildAnd(S64, U, Mask1); |
| 7416 | |
| 7417 | auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); |
| 7418 | auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); |
| 7419 | auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); |
| 7420 | |
| 7421 | auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); |
| 7422 | auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); |
| 7423 | auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); |
| 7424 | auto One = MIRBuilder.buildConstant(S32, 1); |
| 7425 | |
| 7426 | auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); |
| 7427 | auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); |
| 7428 | auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); |
| 7429 | MIRBuilder.buildAdd(Dst, V, R); |
| 7430 | |
Matt Arsenault | 350ee7fb | 2020-06-12 10:20:07 -0400 | [diff] [blame] | 7431 | MI.eraseFromParent(); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7432 | return Legalized; |
| 7433 | } |
| 7434 | |
Evgenii Kudriashov | e9cb440 | 2024-09-25 17:15:36 +0300 | [diff] [blame] | 7435 | // Expand s32 = G_UITOFP s64 to an IEEE float representation using bit |
| 7436 | // operations and G_SITOFP |
| 7437 | LegalizerHelper::LegalizeResult |
| 7438 | LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) { |
| 7439 | auto [Dst, Src] = MI.getFirst2Regs(); |
| 7440 | const LLT S64 = LLT::scalar(64); |
| 7441 | const LLT S32 = LLT::scalar(32); |
| 7442 | const LLT S1 = LLT::scalar(1); |
| 7443 | |
| 7444 | assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); |
| 7445 | |
| 7446 | // For i64 < INT_MAX we simply reuse SITOFP. |
| 7447 | // Otherwise, divide i64 by 2, round result by ORing with the lowest bit |
| 7448 | // saved before division, convert to float by SITOFP, multiply the result |
| 7449 | // by 2. |
| 7450 | auto One = MIRBuilder.buildConstant(S64, 1); |
| 7451 | auto Zero = MIRBuilder.buildConstant(S64, 0); |
| 7452 | // Result if Src < INT_MAX |
| 7453 | auto SmallResult = MIRBuilder.buildSITOFP(S32, Src); |
| 7454 | // Result if Src >= INT_MAX |
| 7455 | auto Halved = MIRBuilder.buildLShr(S64, Src, One); |
| 7456 | auto LowerBit = MIRBuilder.buildAnd(S64, Src, One); |
| 7457 | auto RoundedHalved = MIRBuilder.buildOr(S64, Halved, LowerBit); |
| 7458 | auto HalvedFP = MIRBuilder.buildSITOFP(S32, RoundedHalved); |
| 7459 | auto LargeResult = MIRBuilder.buildFAdd(S32, HalvedFP, HalvedFP); |
| 7460 | // Check if the original value is larger than INT_MAX by comparing with |
| 7461 | // zero to pick one of the two conversions. |
| 7462 | auto IsLarge = |
| 7463 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_SLT, S1, Src, Zero); |
| 7464 | MIRBuilder.buildSelect(Dst, IsLarge, LargeResult, SmallResult); |
| 7465 | |
| 7466 | MI.eraseFromParent(); |
| 7467 | return Legalized; |
| 7468 | } |
| 7469 | |
| 7470 | // Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an |
| 7471 | // IEEE double representation. |
| 7472 | LegalizerHelper::LegalizeResult |
| 7473 | LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) { |
| 7474 | auto [Dst, Src] = MI.getFirst2Regs(); |
| 7475 | const LLT S64 = LLT::scalar(64); |
| 7476 | const LLT S32 = LLT::scalar(32); |
| 7477 | |
| 7478 | assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64); |
| 7479 | |
| 7480 | // We create double value from 32 bit parts with 32 exponent difference. |
| 7481 | // Note that + and - are float operations that adjust the implicit leading |
| 7482 | // one, the bases 2^52 and 2^84 are for illustrative purposes. |
| 7483 | // |
| 7484 | // X = 2^52 * 1.0...LowBits |
| 7485 | // Y = 2^84 * 1.0...HighBits |
| 7486 | // Scratch = 2^84 * 1.0...HighBits - 2^84 * 1.0 - 2^52 * 1.0 |
| 7487 | // = - 2^52 * 1.0...HighBits |
| 7488 | // Result = - 2^52 * 1.0...HighBits + 2^52 * 1.0...LowBits |
| 7489 | auto TwoP52 = MIRBuilder.buildConstant(S64, UINT64_C(0x4330000000000000)); |
| 7490 | auto TwoP84 = MIRBuilder.buildConstant(S64, UINT64_C(0x4530000000000000)); |
| 7491 | auto TwoP52P84 = llvm::bit_cast<double>(UINT64_C(0x4530000000100000)); |
| 7492 | auto TwoP52P84FP = MIRBuilder.buildFConstant(S64, TwoP52P84); |
| 7493 | auto HalfWidth = MIRBuilder.buildConstant(S64, 32); |
| 7494 | |
| 7495 | auto LowBits = MIRBuilder.buildTrunc(S32, Src); |
| 7496 | LowBits = MIRBuilder.buildZExt(S64, LowBits); |
| 7497 | auto LowBitsFP = MIRBuilder.buildOr(S64, TwoP52, LowBits); |
| 7498 | auto HighBits = MIRBuilder.buildLShr(S64, Src, HalfWidth); |
| 7499 | auto HighBitsFP = MIRBuilder.buildOr(S64, TwoP84, HighBits); |
| 7500 | auto Scratch = MIRBuilder.buildFSub(S64, HighBitsFP, TwoP52P84FP); |
| 7501 | MIRBuilder.buildFAdd(Dst, Scratch, LowBitsFP); |
| 7502 | |
| 7503 | MI.eraseFromParent(); |
| 7504 | return Legalized; |
| 7505 | } |
| 7506 | |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7507 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7508 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7509 | |
Matt Arsenault | bc276c6 | 2019-11-15 11:59:12 +0530 | [diff] [blame] | 7510 | if (SrcTy == LLT::scalar(1)) { |
| 7511 | auto True = MIRBuilder.buildFConstant(DstTy, 1.0); |
| 7512 | auto False = MIRBuilder.buildFConstant(DstTy, 0.0); |
| 7513 | MIRBuilder.buildSelect(Dst, Src, True, False); |
| 7514 | MI.eraseFromParent(); |
| 7515 | return Legalized; |
| 7516 | } |
| 7517 | |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7518 | if (SrcTy != LLT::scalar(64)) |
| 7519 | return UnableToLegalize; |
| 7520 | |
Evgenii Kudriashov | e9cb440 | 2024-09-25 17:15:36 +0300 | [diff] [blame] | 7521 | if (DstTy == LLT::scalar(32)) |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7522 | // TODO: SelectionDAG has several alternative expansions to port which may |
Evgenii Kudriashov | e9cb440 | 2024-09-25 17:15:36 +0300 | [diff] [blame] | 7523 | // be more reasonable depending on the available instructions. We also need |
| 7524 | // a more advanced mechanism to choose an optimal version depending on |
| 7525 | // target features such as sitofp or CTLZ availability. |
| 7526 | return lowerU64ToF32WithSITOFP(MI); |
| 7527 | |
| 7528 | if (DstTy == LLT::scalar(64)) |
| 7529 | return lowerU64ToF64BitFloatOps(MI); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7530 | |
| 7531 | return UnableToLegalize; |
| 7532 | } |
| 7533 | |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7534 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7535 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7536 | |
| 7537 | const LLT S64 = LLT::scalar(64); |
| 7538 | const LLT S32 = LLT::scalar(32); |
| 7539 | const LLT S1 = LLT::scalar(1); |
| 7540 | |
Matt Arsenault | bc276c6 | 2019-11-15 11:59:12 +0530 | [diff] [blame] | 7541 | if (SrcTy == S1) { |
| 7542 | auto True = MIRBuilder.buildFConstant(DstTy, -1.0); |
| 7543 | auto False = MIRBuilder.buildFConstant(DstTy, 0.0); |
| 7544 | MIRBuilder.buildSelect(Dst, Src, True, False); |
| 7545 | MI.eraseFromParent(); |
| 7546 | return Legalized; |
| 7547 | } |
| 7548 | |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7549 | if (SrcTy != S64) |
| 7550 | return UnableToLegalize; |
| 7551 | |
| 7552 | if (DstTy == S32) { |
| 7553 | // signed cl2f(long l) { |
| 7554 | // long s = l >> 63; |
| 7555 | // float r = cul2f((l + s) ^ s); |
| 7556 | // return s ? -r : r; |
| 7557 | // } |
Matt Arsenault | faeaedf | 2019-06-24 16:16:12 +0000 | [diff] [blame] | 7558 | Register L = Src; |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7559 | auto SignBit = MIRBuilder.buildConstant(S64, 63); |
| 7560 | auto S = MIRBuilder.buildAShr(S64, L, SignBit); |
| 7561 | |
| 7562 | auto LPlusS = MIRBuilder.buildAdd(S64, L, S); |
| 7563 | auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); |
| 7564 | auto R = MIRBuilder.buildUITOFP(S32, Xor); |
| 7565 | |
| 7566 | auto RNeg = MIRBuilder.buildFNeg(S32, R); |
| 7567 | auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, |
| 7568 | MIRBuilder.buildConstant(S64, 0)); |
| 7569 | MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); |
Matt Arsenault | 350ee7fb | 2020-06-12 10:20:07 -0400 | [diff] [blame] | 7570 | MI.eraseFromParent(); |
Matt Arsenault | 02b5ca8 | 2019-05-17 23:05:13 +0000 | [diff] [blame] | 7571 | return Legalized; |
| 7572 | } |
| 7573 | |
| 7574 | return UnableToLegalize; |
| 7575 | } |
Matt Arsenault | 6f74f55 | 2019-07-01 17:18:03 +0000 | [diff] [blame] | 7576 | |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7577 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7578 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
Petar Avramovic | 6412b56 | 2019-08-30 05:44:02 +0000 | [diff] [blame] | 7579 | const LLT S64 = LLT::scalar(64); |
| 7580 | const LLT S32 = LLT::scalar(32); |
| 7581 | |
| 7582 | if (SrcTy != S64 && SrcTy != S32) |
| 7583 | return UnableToLegalize; |
| 7584 | if (DstTy != S32 && DstTy != S64) |
| 7585 | return UnableToLegalize; |
| 7586 | |
| 7587 | // FPTOSI gives same result as FPTOUI for positive signed integers. |
| 7588 | // FPTOUI needs to deal with fp values that convert to unsigned integers |
| 7589 | // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp. |
| 7590 | |
| 7591 | APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits()); |
| 7592 | APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle() |
| 7593 | : APFloat::IEEEdouble(), |
Chris Lattner | 735f467 | 2021-09-08 22:13:13 -0700 | [diff] [blame] | 7594 | APInt::getZero(SrcTy.getSizeInBits())); |
Petar Avramovic | 6412b56 | 2019-08-30 05:44:02 +0000 | [diff] [blame] | 7595 | TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven); |
| 7596 | |
| 7597 | MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src); |
| 7598 | |
| 7599 | MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP); |
| 7600 | // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on |
| 7601 | // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1. |
| 7602 | MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold); |
| 7603 | MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub); |
| 7604 | MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt); |
| 7605 | MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit); |
| 7606 | |
Matt Arsenault | 1060b9e | 2020-01-04 17:06:47 -0500 | [diff] [blame] | 7607 | const LLT S1 = LLT::scalar(1); |
| 7608 | |
Petar Avramovic | 6412b56 | 2019-08-30 05:44:02 +0000 | [diff] [blame] | 7609 | MachineInstrBuilder FCMP = |
Matt Arsenault | 1060b9e | 2020-01-04 17:06:47 -0500 | [diff] [blame] | 7610 | MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold); |
Petar Avramovic | 6412b56 | 2019-08-30 05:44:02 +0000 | [diff] [blame] | 7611 | MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res); |
| 7612 | |
| 7613 | MI.eraseFromParent(); |
| 7614 | return Legalized; |
| 7615 | } |
| 7616 | |
Matt Arsenault | ea95668 | 2020-01-04 17:09:48 -0500 | [diff] [blame] | 7617 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7618 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | ea95668 | 2020-01-04 17:09:48 -0500 | [diff] [blame] | 7619 | const LLT S64 = LLT::scalar(64); |
| 7620 | const LLT S32 = LLT::scalar(32); |
| 7621 | |
| 7622 | // FIXME: Only f32 to i64 conversions are supported. |
| 7623 | if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64) |
| 7624 | return UnableToLegalize; |
| 7625 | |
| 7626 | // Expand f32 -> i64 conversion |
| 7627 | // This algorithm comes from compiler-rt's implementation of fixsfdi: |
xgupta | 94fac81 | 2021-02-01 12:54:21 +0530 | [diff] [blame] | 7628 | // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c |
Matt Arsenault | ea95668 | 2020-01-04 17:09:48 -0500 | [diff] [blame] | 7629 | |
| 7630 | unsigned SrcEltBits = SrcTy.getScalarSizeInBits(); |
| 7631 | |
| 7632 | auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000); |
| 7633 | auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23); |
| 7634 | |
| 7635 | auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask); |
| 7636 | auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit); |
| 7637 | |
| 7638 | auto SignMask = MIRBuilder.buildConstant(SrcTy, |
| 7639 | APInt::getSignMask(SrcEltBits)); |
| 7640 | auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask); |
| 7641 | auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1); |
| 7642 | auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit); |
| 7643 | Sign = MIRBuilder.buildSExt(DstTy, Sign); |
| 7644 | |
| 7645 | auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF); |
| 7646 | auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask); |
| 7647 | auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000); |
| 7648 | |
| 7649 | auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K); |
| 7650 | R = MIRBuilder.buildZExt(DstTy, R); |
| 7651 | |
| 7652 | auto Bias = MIRBuilder.buildConstant(SrcTy, 127); |
| 7653 | auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias); |
| 7654 | auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit); |
| 7655 | auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent); |
| 7656 | |
| 7657 | auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent); |
| 7658 | auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub); |
| 7659 | |
| 7660 | const LLT S1 = LLT::scalar(1); |
| 7661 | auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, |
| 7662 | S1, Exponent, ExponentLoBit); |
| 7663 | |
| 7664 | R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl); |
| 7665 | |
| 7666 | auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign); |
| 7667 | auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign); |
| 7668 | |
| 7669 | auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0); |
| 7670 | |
| 7671 | auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, |
| 7672 | S1, Exponent, ZeroSrcTy); |
| 7673 | |
| 7674 | auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0); |
| 7675 | MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret); |
| 7676 | |
| 7677 | MI.eraseFromParent(); |
| 7678 | return Legalized; |
| 7679 | } |
| 7680 | |
David Green | feac761 | 2024-09-16 10:33:59 +0100 | [diff] [blame] | 7681 | LegalizerHelper::LegalizeResult |
| 7682 | LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) { |
| 7683 | auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs(); |
| 7684 | |
| 7685 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT; |
| 7686 | unsigned SatWidth = DstTy.getScalarSizeInBits(); |
| 7687 | |
| 7688 | // Determine minimum and maximum integer values and their corresponding |
| 7689 | // floating-point values. |
| 7690 | APInt MinInt, MaxInt; |
| 7691 | if (IsSigned) { |
| 7692 | MinInt = APInt::getSignedMinValue(SatWidth); |
| 7693 | MaxInt = APInt::getSignedMaxValue(SatWidth); |
| 7694 | } else { |
| 7695 | MinInt = APInt::getMinValue(SatWidth); |
| 7696 | MaxInt = APInt::getMaxValue(SatWidth); |
| 7697 | } |
| 7698 | |
| 7699 | const fltSemantics &Semantics = getFltSemanticForLLT(SrcTy.getScalarType()); |
| 7700 | APFloat MinFloat(Semantics); |
| 7701 | APFloat MaxFloat(Semantics); |
| 7702 | |
| 7703 | APFloat::opStatus MinStatus = |
| 7704 | MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero); |
| 7705 | APFloat::opStatus MaxStatus = |
| 7706 | MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero); |
| 7707 | bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) && |
| 7708 | !(MaxStatus & APFloat::opStatus::opInexact); |
| 7709 | |
| 7710 | // If the integer bounds are exactly representable as floats, emit a |
| 7711 | // min+max+fptoi sequence. Otherwise we have to use a sequence of comparisons |
| 7712 | // and selects. |
| 7713 | if (AreExactFloatBounds) { |
| 7714 | // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat. |
| 7715 | auto MaxC = MIRBuilder.buildFConstant(SrcTy, MinFloat); |
| 7716 | auto MaxP = MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, |
| 7717 | SrcTy.changeElementSize(1), Src, MaxC); |
| 7718 | auto Max = MIRBuilder.buildSelect(SrcTy, MaxP, Src, MaxC); |
| 7719 | // Clamp by MaxFloat from above. NaN cannot occur. |
| 7720 | auto MinC = MIRBuilder.buildFConstant(SrcTy, MaxFloat); |
| 7721 | auto MinP = |
| 7722 | MIRBuilder.buildFCmp(CmpInst::FCMP_OGT, SrcTy.changeElementSize(1), Max, |
| 7723 | MinC, MachineInstr::FmNoNans); |
| 7724 | auto Min = |
| 7725 | MIRBuilder.buildSelect(SrcTy, MinP, Max, MinC, MachineInstr::FmNoNans); |
| 7726 | // Convert clamped value to integer. In the unsigned case we're done, |
| 7727 | // because we mapped NaN to MinFloat, which will cast to zero. |
| 7728 | if (!IsSigned) { |
| 7729 | MIRBuilder.buildFPTOUI(Dst, Min); |
| 7730 | MI.eraseFromParent(); |
| 7731 | return Legalized; |
| 7732 | } |
| 7733 | |
| 7734 | // Otherwise, select 0 if Src is NaN. |
| 7735 | auto FpToInt = MIRBuilder.buildFPTOSI(DstTy, Min); |
| 7736 | auto IsZero = MIRBuilder.buildFCmp(CmpInst::FCMP_UNO, |
| 7737 | DstTy.changeElementSize(1), Src, Src); |
| 7738 | MIRBuilder.buildSelect(Dst, IsZero, MIRBuilder.buildConstant(DstTy, 0), |
| 7739 | FpToInt); |
| 7740 | MI.eraseFromParent(); |
| 7741 | return Legalized; |
| 7742 | } |
| 7743 | |
| 7744 | // Result of direct conversion. The assumption here is that the operation is |
| 7745 | // non-trapping and it's fine to apply it to an out-of-range value if we |
| 7746 | // select it away later. |
| 7747 | auto FpToInt = IsSigned ? MIRBuilder.buildFPTOSI(DstTy, Src) |
| 7748 | : MIRBuilder.buildFPTOUI(DstTy, Src); |
| 7749 | |
| 7750 | // If Src ULT MinFloat, select MinInt. In particular, this also selects |
| 7751 | // MinInt if Src is NaN. |
| 7752 | auto ULT = |
| 7753 | MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, SrcTy.changeElementSize(1), Src, |
| 7754 | MIRBuilder.buildFConstant(SrcTy, MinFloat)); |
| 7755 | auto Max = MIRBuilder.buildSelect( |
| 7756 | DstTy, ULT, MIRBuilder.buildConstant(DstTy, MinInt), FpToInt); |
| 7757 | // If Src OGT MaxFloat, select MaxInt. |
| 7758 | auto OGT = |
| 7759 | MIRBuilder.buildFCmp(CmpInst::FCMP_OGT, SrcTy.changeElementSize(1), Src, |
| 7760 | MIRBuilder.buildFConstant(SrcTy, MaxFloat)); |
| 7761 | |
| 7762 | // In the unsigned case we are done, because we mapped NaN to MinInt, which |
| 7763 | // is already zero. |
| 7764 | if (!IsSigned) { |
| 7765 | MIRBuilder.buildSelect(Dst, OGT, MIRBuilder.buildConstant(DstTy, MaxInt), |
| 7766 | Max); |
| 7767 | MI.eraseFromParent(); |
| 7768 | return Legalized; |
| 7769 | } |
| 7770 | |
| 7771 | // Otherwise, select 0 if Src is NaN. |
| 7772 | auto Min = MIRBuilder.buildSelect( |
| 7773 | DstTy, OGT, MIRBuilder.buildConstant(DstTy, MaxInt), Max); |
| 7774 | auto IsZero = MIRBuilder.buildFCmp(CmpInst::FCMP_UNO, |
| 7775 | DstTy.changeElementSize(1), Src, Src); |
| 7776 | MIRBuilder.buildSelect(Dst, IsZero, MIRBuilder.buildConstant(DstTy, 0), Min); |
| 7777 | MI.eraseFromParent(); |
| 7778 | return Legalized; |
| 7779 | } |
| 7780 | |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7781 | // f64 -> f16 conversion using round-to-nearest-even rounding mode. |
| 7782 | LegalizerHelper::LegalizeResult |
| 7783 | LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) { |
Ivan Kosarev | 15e7749 | 2023-07-12 11:19:36 +0100 | [diff] [blame] | 7784 | const LLT S1 = LLT::scalar(1); |
Ivan Kosarev | 15e7749 | 2023-07-12 11:19:36 +0100 | [diff] [blame] | 7785 | const LLT S32 = LLT::scalar(32); |
Ivan Kosarev | 15e7749 | 2023-07-12 11:19:36 +0100 | [diff] [blame] | 7786 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7787 | auto [Dst, Src] = MI.getFirst2Regs(); |
Ivan Kosarev | e705b2b | 2023-07-12 14:35:42 +0100 | [diff] [blame] | 7788 | assert(MRI.getType(Dst).getScalarType() == LLT::scalar(16) && |
| 7789 | MRI.getType(Src).getScalarType() == LLT::scalar(64)); |
Ivan Kosarev | 15e7749 | 2023-07-12 11:19:36 +0100 | [diff] [blame] | 7790 | |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7791 | if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly. |
| 7792 | return UnableToLegalize; |
| 7793 | |
Ivan Kosarev | 15e7749 | 2023-07-12 11:19:36 +0100 | [diff] [blame] | 7794 | if (MIRBuilder.getMF().getTarget().Options.UnsafeFPMath) { |
| 7795 | unsigned Flags = MI.getFlags(); |
| 7796 | auto Src32 = MIRBuilder.buildFPTrunc(S32, Src, Flags); |
| 7797 | MIRBuilder.buildFPTrunc(Dst, Src32, Flags); |
| 7798 | MI.eraseFromParent(); |
| 7799 | return Legalized; |
| 7800 | } |
| 7801 | |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7802 | const unsigned ExpMask = 0x7ff; |
| 7803 | const unsigned ExpBiasf64 = 1023; |
| 7804 | const unsigned ExpBiasf16 = 15; |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7805 | |
| 7806 | auto Unmerge = MIRBuilder.buildUnmerge(S32, Src); |
| 7807 | Register U = Unmerge.getReg(0); |
| 7808 | Register UH = Unmerge.getReg(1); |
| 7809 | |
| 7810 | auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20)); |
Petar Avramovic | bd3d951 | 2020-06-11 17:55:59 +0200 | [diff] [blame] | 7811 | E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask)); |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7812 | |
| 7813 | // Subtract the fp64 exponent bias (1023) to get the real exponent and |
| 7814 | // add the f16 bias (15) to get the biased exponent for the f16 format. |
| 7815 | E = MIRBuilder.buildAdd( |
| 7816 | S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16)); |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7817 | |
| 7818 | auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8)); |
| 7819 | M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe)); |
| 7820 | |
| 7821 | auto MaskedSig = MIRBuilder.buildAnd(S32, UH, |
| 7822 | MIRBuilder.buildConstant(S32, 0x1ff)); |
| 7823 | MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U); |
| 7824 | |
| 7825 | auto Zero = MIRBuilder.buildConstant(S32, 0); |
| 7826 | auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero); |
| 7827 | auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0); |
| 7828 | M = MIRBuilder.buildOr(S32, M, Lo40Set); |
| 7829 | |
| 7830 | // (M != 0 ? 0x0200 : 0) | 0x7c00; |
| 7831 | auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200); |
| 7832 | auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero); |
| 7833 | auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero); |
| 7834 | |
| 7835 | auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00); |
| 7836 | auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00); |
| 7837 | |
| 7838 | // N = M | (E << 12); |
| 7839 | auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12)); |
| 7840 | auto N = MIRBuilder.buildOr(S32, M, EShl12); |
| 7841 | |
| 7842 | // B = clamp(1-E, 0, 13); |
| 7843 | auto One = MIRBuilder.buildConstant(S32, 1); |
| 7844 | auto OneSubExp = MIRBuilder.buildSub(S32, One, E); |
| 7845 | auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero); |
| 7846 | B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13)); |
| 7847 | |
| 7848 | auto SigSetHigh = MIRBuilder.buildOr(S32, M, |
| 7849 | MIRBuilder.buildConstant(S32, 0x1000)); |
| 7850 | |
| 7851 | auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B); |
| 7852 | auto D0 = MIRBuilder.buildShl(S32, D, B); |
| 7853 | |
| 7854 | auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, |
| 7855 | D0, SigSetHigh); |
| 7856 | auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh); |
| 7857 | D = MIRBuilder.buildOr(S32, D, D1); |
| 7858 | |
| 7859 | auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One); |
| 7860 | auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N); |
| 7861 | |
| 7862 | auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7)); |
| 7863 | V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2)); |
| 7864 | |
| 7865 | auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3, |
| 7866 | MIRBuilder.buildConstant(S32, 3)); |
| 7867 | auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3); |
| 7868 | |
| 7869 | auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3, |
| 7870 | MIRBuilder.buildConstant(S32, 5)); |
| 7871 | auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5); |
| 7872 | |
| 7873 | V1 = MIRBuilder.buildOr(S32, V0, V1); |
| 7874 | V = MIRBuilder.buildAdd(S32, V, V1); |
| 7875 | |
| 7876 | auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, |
| 7877 | E, MIRBuilder.buildConstant(S32, 30)); |
| 7878 | V = MIRBuilder.buildSelect(S32, CmpEGt30, |
| 7879 | MIRBuilder.buildConstant(S32, 0x7c00), V); |
| 7880 | |
| 7881 | auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, |
| 7882 | E, MIRBuilder.buildConstant(S32, 1039)); |
| 7883 | V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V); |
| 7884 | |
| 7885 | // Extract the sign bit. |
| 7886 | auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16)); |
| 7887 | Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000)); |
| 7888 | |
| 7889 | // Insert the sign bit |
| 7890 | V = MIRBuilder.buildOr(S32, Sign, V); |
| 7891 | |
| 7892 | MIRBuilder.buildTrunc(Dst, V); |
| 7893 | MI.eraseFromParent(); |
| 7894 | return Legalized; |
| 7895 | } |
| 7896 | |
| 7897 | LegalizerHelper::LegalizeResult |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7898 | LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7899 | auto [DstTy, SrcTy] = MI.getFirst2LLTs(); |
Matt Arsenault | bfbfa18 | 2020-01-18 10:08:11 -0500 | [diff] [blame] | 7900 | const LLT S64 = LLT::scalar(64); |
| 7901 | const LLT S16 = LLT::scalar(16); |
| 7902 | |
| 7903 | if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64) |
| 7904 | return lowerFPTRUNC_F64_TO_F16(MI); |
| 7905 | |
| 7906 | return UnableToLegalize; |
| 7907 | } |
| 7908 | |
Matt Arsenault | 7cd8a02 | 2020-07-17 11:01:15 -0400 | [diff] [blame] | 7909 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7910 | auto [Dst, Src0, Src1] = MI.getFirst3Regs(); |
Matt Arsenault | 7cd8a02 | 2020-07-17 11:01:15 -0400 | [diff] [blame] | 7911 | LLT Ty = MRI.getType(Dst); |
| 7912 | |
| 7913 | auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1); |
| 7914 | MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags()); |
| 7915 | MI.eraseFromParent(); |
| 7916 | return Legalized; |
| 7917 | } |
| 7918 | |
Matt Arsenault | 6f74f55 | 2019-07-01 17:18:03 +0000 | [diff] [blame] | 7919 | static CmpInst::Predicate minMaxToCompare(unsigned Opc) { |
| 7920 | switch (Opc) { |
| 7921 | case TargetOpcode::G_SMIN: |
| 7922 | return CmpInst::ICMP_SLT; |
| 7923 | case TargetOpcode::G_SMAX: |
| 7924 | return CmpInst::ICMP_SGT; |
| 7925 | case TargetOpcode::G_UMIN: |
| 7926 | return CmpInst::ICMP_ULT; |
| 7927 | case TargetOpcode::G_UMAX: |
| 7928 | return CmpInst::ICMP_UGT; |
| 7929 | default: |
| 7930 | llvm_unreachable("not in integer min/max"); |
| 7931 | } |
| 7932 | } |
| 7933 | |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7934 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7935 | auto [Dst, Src0, Src1] = MI.getFirst3Regs(); |
Matt Arsenault | 6f74f55 | 2019-07-01 17:18:03 +0000 | [diff] [blame] | 7936 | |
| 7937 | const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); |
| 7938 | LLT CmpType = MRI.getType(Dst).changeElementSize(1); |
| 7939 | |
| 7940 | auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); |
| 7941 | MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); |
| 7942 | |
| 7943 | MI.eraseFromParent(); |
| 7944 | return Legalized; |
| 7945 | } |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 7946 | |
| 7947 | LegalizerHelper::LegalizeResult |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 7948 | LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) { |
| 7949 | GSUCmp *Cmp = cast<GSUCmp>(&MI); |
| 7950 | |
| 7951 | Register Dst = Cmp->getReg(0); |
| 7952 | LLT DstTy = MRI.getType(Dst); |
Craig Topper | de1a423 | 2024-12-15 20:47:17 -0800 | [diff] [blame^] | 7953 | LLT SrcTy = MRI.getType(Cmp->getReg(1)); |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 7954 | LLT CmpTy = DstTy.changeElementSize(1); |
| 7955 | |
| 7956 | CmpInst::Predicate LTPredicate = Cmp->isSigned() |
| 7957 | ? CmpInst::Predicate::ICMP_SLT |
| 7958 | : CmpInst::Predicate::ICMP_ULT; |
| 7959 | CmpInst::Predicate GTPredicate = Cmp->isSigned() |
| 7960 | ? CmpInst::Predicate::ICMP_SGT |
| 7961 | : CmpInst::Predicate::ICMP_UGT; |
| 7962 | |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 7963 | auto Zero = MIRBuilder.buildConstant(DstTy, 0); |
| 7964 | auto IsGT = MIRBuilder.buildICmp(GTPredicate, CmpTy, Cmp->getLHSReg(), |
| 7965 | Cmp->getRHSReg()); |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 7966 | auto IsLT = MIRBuilder.buildICmp(LTPredicate, CmpTy, Cmp->getLHSReg(), |
| 7967 | Cmp->getRHSReg()); |
Craig Topper | de1a423 | 2024-12-15 20:47:17 -0800 | [diff] [blame^] | 7968 | |
| 7969 | auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); |
| 7970 | auto BC = TLI.getBooleanContents(DstTy.isVector(), /*isFP=*/false); |
| 7971 | if (TLI.shouldExpandCmpUsingSelects(getApproximateEVTForLLT(SrcTy, Ctx)) || |
| 7972 | BC == TargetLowering::UndefinedBooleanContent) { |
| 7973 | auto One = MIRBuilder.buildConstant(DstTy, 1); |
| 7974 | auto SelectZeroOrOne = MIRBuilder.buildSelect(DstTy, IsGT, One, Zero); |
| 7975 | |
| 7976 | auto MinusOne = MIRBuilder.buildConstant(DstTy, -1); |
| 7977 | MIRBuilder.buildSelect(Dst, IsLT, MinusOne, SelectZeroOrOne); |
| 7978 | } else { |
| 7979 | if (BC == TargetLowering::ZeroOrNegativeOneBooleanContent) |
| 7980 | std::swap(IsGT, IsLT); |
| 7981 | // Extend boolean results to DstTy, which is at least i2, before subtracting |
| 7982 | // them. |
| 7983 | unsigned BoolExtOp = |
| 7984 | MIRBuilder.getBoolExtOp(DstTy.isVector(), /*isFP=*/false); |
| 7985 | IsGT = MIRBuilder.buildInstr(BoolExtOp, {DstTy}, {IsGT}); |
| 7986 | IsLT = MIRBuilder.buildInstr(BoolExtOp, {DstTy}, {IsLT}); |
| 7987 | MIRBuilder.buildSub(Dst, IsGT, IsLT); |
| 7988 | } |
Thorsten Schütt | 2d2d685 | 2024-07-23 10:12:28 +0200 | [diff] [blame] | 7989 | |
| 7990 | MI.eraseFromParent(); |
| 7991 | return Legalized; |
| 7992 | } |
| 7993 | |
| 7994 | LegalizerHelper::LegalizeResult |
Matt Arsenault | a128292 | 2020-07-15 11:10:54 -0400 | [diff] [blame] | 7995 | LegalizerHelper::lowerFCopySign(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 7996 | auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] = MI.getFirst3RegLLTs(); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 7997 | const int Src0Size = Src0Ty.getScalarSizeInBits(); |
| 7998 | const int Src1Size = Src1Ty.getScalarSizeInBits(); |
| 7999 | |
| 8000 | auto SignBitMask = MIRBuilder.buildConstant( |
| 8001 | Src0Ty, APInt::getSignMask(Src0Size)); |
| 8002 | |
| 8003 | auto NotSignBitMask = MIRBuilder.buildConstant( |
| 8004 | Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1)); |
| 8005 | |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8006 | Register And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask).getReg(0); |
| 8007 | Register And1; |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 8008 | if (Src0Ty == Src1Ty) { |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8009 | And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask).getReg(0); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 8010 | } else if (Src0Size > Src1Size) { |
| 8011 | auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); |
| 8012 | auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); |
| 8013 | auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8014 | And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask).getReg(0); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 8015 | } else { |
| 8016 | auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); |
| 8017 | auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); |
| 8018 | auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8019 | And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask).getReg(0); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 8020 | } |
| 8021 | |
| 8022 | // Be careful about setting nsz/nnan/ninf on every instruction, since the |
| 8023 | // constants are a nan and -0.0, but the final result should preserve |
| 8024 | // everything. |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8025 | unsigned Flags = MI.getFlags(); |
Matt Arsenault | 2df2373 | 2024-06-28 23:03:39 +0200 | [diff] [blame] | 8026 | |
| 8027 | // We masked the sign bit and the not-sign bit, so these are disjoint. |
| 8028 | Flags |= MachineInstr::Disjoint; |
| 8029 | |
Jay Foad | 5cf6412 | 2021-01-29 14:41:58 +0000 | [diff] [blame] | 8030 | MIRBuilder.buildOr(Dst, And0, And1, Flags); |
Matt Arsenault | b1843e1 | 2019-07-09 23:34:29 +0000 | [diff] [blame] | 8031 | |
| 8032 | MI.eraseFromParent(); |
| 8033 | return Legalized; |
| 8034 | } |
Matt Arsenault | 6ce1b4f | 2019-07-10 16:31:19 +0000 | [diff] [blame] | 8035 | |
| 8036 | LegalizerHelper::LegalizeResult |
| 8037 | LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { |
| 8038 | unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? |
| 8039 | TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; |
| 8040 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8041 | auto [Dst, Src0, Src1] = MI.getFirst3Regs(); |
Matt Arsenault | 6ce1b4f | 2019-07-10 16:31:19 +0000 | [diff] [blame] | 8042 | LLT Ty = MRI.getType(Dst); |
| 8043 | |
| 8044 | if (!MI.getFlag(MachineInstr::FmNoNans)) { |
| 8045 | // Insert canonicalizes if it's possible we need to quiet to get correct |
| 8046 | // sNaN behavior. |
| 8047 | |
| 8048 | // Note this must be done here, and not as an optimization combine in the |
| 8049 | // absence of a dedicate quiet-snan instruction as we're using an |
| 8050 | // omni-purpose G_FCANONICALIZE. |
| 8051 | if (!isKnownNeverSNaN(Src0, MRI)) |
| 8052 | Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); |
| 8053 | |
| 8054 | if (!isKnownNeverSNaN(Src1, MRI)) |
| 8055 | Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); |
| 8056 | } |
| 8057 | |
| 8058 | // If there are no nans, it's safe to simply replace this with the non-IEEE |
| 8059 | // version. |
| 8060 | MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); |
| 8061 | MI.eraseFromParent(); |
| 8062 | return Legalized; |
| 8063 | } |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8064 | |
Matt Arsenault | 4d33918 | 2019-09-13 00:44:35 +0000 | [diff] [blame] | 8065 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) { |
| 8066 | // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c |
| 8067 | Register DstReg = MI.getOperand(0).getReg(); |
| 8068 | LLT Ty = MRI.getType(DstReg); |
| 8069 | unsigned Flags = MI.getFlags(); |
| 8070 | |
| 8071 | auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2), |
| 8072 | Flags); |
| 8073 | MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags); |
| 8074 | MI.eraseFromParent(); |
| 8075 | return Legalized; |
| 8076 | } |
| 8077 | |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8078 | LegalizerHelper::LegalizeResult |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 8079 | LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8080 | auto [DstReg, X] = MI.getFirst2Regs(); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8081 | const unsigned Flags = MI.getFlags(); |
| 8082 | const LLT Ty = MRI.getType(DstReg); |
| 8083 | const LLT CondTy = Ty.changeElementSize(1); |
| 8084 | |
| 8085 | // round(x) => |
| 8086 | // t = trunc(x); |
| 8087 | // d = fabs(x - t); |
Matt Arsenault | 1328a85 | 2023-09-19 09:14:17 +0300 | [diff] [blame] | 8088 | // o = copysign(d >= 0.5 ? 1.0 : 0.0, x); |
| 8089 | // return t + o; |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8090 | |
| 8091 | auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags); |
| 8092 | |
| 8093 | auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags); |
| 8094 | auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags); |
Matt Arsenault | 1328a85 | 2023-09-19 09:14:17 +0300 | [diff] [blame] | 8095 | |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8096 | auto Half = MIRBuilder.buildFConstant(Ty, 0.5); |
Matt Arsenault | 1328a85 | 2023-09-19 09:14:17 +0300 | [diff] [blame] | 8097 | auto Cmp = |
| 8098 | MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half, Flags); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8099 | |
Matt Arsenault | 1328a85 | 2023-09-19 09:14:17 +0300 | [diff] [blame] | 8100 | // Could emit G_UITOFP instead |
| 8101 | auto One = MIRBuilder.buildFConstant(Ty, 1.0); |
| 8102 | auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); |
| 8103 | auto BoolFP = MIRBuilder.buildSelect(Ty, Cmp, One, Zero); |
| 8104 | auto SignedOffset = MIRBuilder.buildFCopysign(Ty, BoolFP, X); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8105 | |
Matt Arsenault | 1328a85 | 2023-09-19 09:14:17 +0300 | [diff] [blame] | 8106 | MIRBuilder.buildFAdd(DstReg, T, SignedOffset, Flags); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8107 | |
| 8108 | MI.eraseFromParent(); |
| 8109 | return Legalized; |
| 8110 | } |
| 8111 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8112 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) { |
| 8113 | auto [DstReg, SrcReg] = MI.getFirst2Regs(); |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 8114 | unsigned Flags = MI.getFlags(); |
| 8115 | LLT Ty = MRI.getType(DstReg); |
| 8116 | const LLT CondTy = Ty.changeElementSize(1); |
| 8117 | |
| 8118 | // result = trunc(src); |
| 8119 | // if (src < 0.0 && src != result) |
| 8120 | // result += -1.0. |
| 8121 | |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 8122 | auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags); |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8123 | auto Zero = MIRBuilder.buildFConstant(Ty, 0.0); |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 8124 | |
| 8125 | auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy, |
| 8126 | SrcReg, Zero, Flags); |
| 8127 | auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy, |
| 8128 | SrcReg, Trunc, Flags); |
| 8129 | auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc); |
| 8130 | auto AddVal = MIRBuilder.buildSITOFP(Ty, And); |
| 8131 | |
Matt Arsenault | 19a0350 | 2020-03-14 14:52:48 -0400 | [diff] [blame] | 8132 | MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags); |
Matt Arsenault | f3de8ab | 2019-12-24 14:49:31 -0500 | [diff] [blame] | 8133 | MI.eraseFromParent(); |
| 8134 | return Legalized; |
| 8135 | } |
| 8136 | |
| 8137 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 6999960 | 2020-03-29 15:51:54 -0400 | [diff] [blame] | 8138 | LegalizerHelper::lowerMergeValues(MachineInstr &MI) { |
| 8139 | const unsigned NumOps = MI.getNumOperands(); |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8140 | auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs(); |
| 8141 | unsigned PartSize = Src0Ty.getSizeInBits(); |
Matt Arsenault | 6999960 | 2020-03-29 15:51:54 -0400 | [diff] [blame] | 8142 | |
| 8143 | LLT WideTy = LLT::scalar(DstTy.getSizeInBits()); |
| 8144 | Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0); |
| 8145 | |
| 8146 | for (unsigned I = 2; I != NumOps; ++I) { |
| 8147 | const unsigned Offset = (I - 1) * PartSize; |
| 8148 | |
| 8149 | Register SrcReg = MI.getOperand(I).getReg(); |
| 8150 | auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); |
| 8151 | |
| 8152 | Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg : |
| 8153 | MRI.createGenericVirtualRegister(WideTy); |
| 8154 | |
| 8155 | auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); |
| 8156 | auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); |
| 8157 | MIRBuilder.buildOr(NextResult, ResultReg, Shl); |
| 8158 | ResultReg = NextResult; |
| 8159 | } |
| 8160 | |
| 8161 | if (DstTy.isPointer()) { |
| 8162 | if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace( |
| 8163 | DstTy.getAddressSpace())) { |
| 8164 | LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n"); |
| 8165 | return UnableToLegalize; |
| 8166 | } |
| 8167 | |
| 8168 | MIRBuilder.buildIntToPtr(DstReg, ResultReg); |
| 8169 | } |
| 8170 | |
| 8171 | MI.eraseFromParent(); |
| 8172 | return Legalized; |
| 8173 | } |
| 8174 | |
| 8175 | LegalizerHelper::LegalizeResult |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8176 | LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) { |
| 8177 | const unsigned NumDst = MI.getNumOperands() - 1; |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8178 | Register SrcReg = MI.getOperand(NumDst).getReg(); |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8179 | Register Dst0Reg = MI.getOperand(0).getReg(); |
| 8180 | LLT DstTy = MRI.getType(Dst0Reg); |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8181 | if (DstTy.isPointer()) |
| 8182 | return UnableToLegalize; // TODO |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8183 | |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8184 | SrcReg = coerceToScalar(SrcReg); |
| 8185 | if (!SrcReg) |
| 8186 | return UnableToLegalize; |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8187 | |
| 8188 | // Expand scalarizing unmerge as bitcast to integer and shift. |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8189 | LLT IntTy = MRI.getType(SrcReg); |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8190 | |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8191 | MIRBuilder.buildTrunc(Dst0Reg, SrcReg); |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8192 | |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8193 | const unsigned DstSize = DstTy.getSizeInBits(); |
| 8194 | unsigned Offset = DstSize; |
| 8195 | for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) { |
| 8196 | auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset); |
| 8197 | auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt); |
| 8198 | MIRBuilder.buildTrunc(MI.getOperand(I), Shift); |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8199 | } |
| 8200 | |
Matt Arsenault | 3af85fa | 2020-03-29 18:04:53 -0400 | [diff] [blame] | 8201 | MI.eraseFromParent(); |
| 8202 | return Legalized; |
Matt Arsenault | d9d30a4 | 2019-08-01 19:10:05 +0000 | [diff] [blame] | 8203 | } |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8204 | |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8205 | /// Lower a vector extract or insert by writing the vector to a stack temporary |
| 8206 | /// and reloading the element or vector. |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8207 | /// |
| 8208 | /// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx |
| 8209 | /// => |
| 8210 | /// %stack_temp = G_FRAME_INDEX |
| 8211 | /// G_STORE %vec, %stack_temp |
| 8212 | /// %idx = clamp(%idx, %vec.getNumElements()) |
| 8213 | /// %element_ptr = G_PTR_ADD %stack_temp, %idx |
| 8214 | /// %dst = G_LOAD %element_ptr |
| 8215 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8216 | LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) { |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8217 | Register DstReg = MI.getOperand(0).getReg(); |
| 8218 | Register SrcVec = MI.getOperand(1).getReg(); |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8219 | Register InsertVal; |
| 8220 | if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) |
| 8221 | InsertVal = MI.getOperand(2).getReg(); |
| 8222 | |
| 8223 | Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg(); |
| 8224 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8225 | LLT VecTy = MRI.getType(SrcVec); |
| 8226 | LLT EltTy = VecTy.getElementType(); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8227 | unsigned NumElts = VecTy.getNumElements(); |
| 8228 | |
| 8229 | int64_t IdxVal; |
| 8230 | if (mi_match(Idx, MRI, m_ICst(IdxVal)) && IdxVal <= NumElts) { |
| 8231 | SmallVector<Register, 8> SrcRegs; |
chuongg3 | fcfe1b6 | 2024-01-15 16:40:39 +0000 | [diff] [blame] | 8232 | extractParts(SrcVec, EltTy, NumElts, SrcRegs, MIRBuilder, MRI); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8233 | |
| 8234 | if (InsertVal) { |
| 8235 | SrcRegs[IdxVal] = MI.getOperand(2).getReg(); |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 8236 | MIRBuilder.buildMergeLikeInstr(DstReg, SrcRegs); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8237 | } else { |
| 8238 | MIRBuilder.buildCopy(DstReg, SrcRegs[IdxVal]); |
| 8239 | } |
| 8240 | |
| 8241 | MI.eraseFromParent(); |
| 8242 | return Legalized; |
| 8243 | } |
| 8244 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8245 | if (!EltTy.isByteSized()) { // Not implemented. |
| 8246 | LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n"); |
| 8247 | return UnableToLegalize; |
| 8248 | } |
| 8249 | |
| 8250 | unsigned EltBytes = EltTy.getSizeInBytes(); |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8251 | Align VecAlign = getStackTemporaryAlignment(VecTy); |
| 8252 | Align EltAlign; |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8253 | |
| 8254 | MachinePointerInfo PtrInfo; |
Sander de Smalen | 81b7f11 | 2023-11-22 08:52:53 +0000 | [diff] [blame] | 8255 | auto StackTemp = createStackTemporary( |
| 8256 | TypeSize::getFixed(VecTy.getSizeInBytes()), VecAlign, PtrInfo); |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8257 | MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign); |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8258 | |
| 8259 | // Get the pointer to the element, and be sure not to hit undefined behavior |
| 8260 | // if the index is out of bounds. |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8261 | Register EltPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx); |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8262 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8263 | if (mi_match(Idx, MRI, m_ICst(IdxVal))) { |
| 8264 | int64_t Offset = IdxVal * EltBytes; |
| 8265 | PtrInfo = PtrInfo.getWithOffset(Offset); |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8266 | EltAlign = commonAlignment(VecAlign, Offset); |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8267 | } else { |
| 8268 | // We lose information with a variable offset. |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8269 | EltAlign = getStackTemporaryAlignment(EltTy); |
| 8270 | PtrInfo = MachinePointerInfo(MRI.getType(EltPtr).getAddressSpace()); |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8271 | } |
| 8272 | |
Matt Arsenault | 1ad051dd | 2020-07-27 21:13:40 -0400 | [diff] [blame] | 8273 | if (InsertVal) { |
| 8274 | // Write the inserted element |
| 8275 | MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign); |
| 8276 | |
| 8277 | // Reload the whole vector. |
| 8278 | MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign); |
| 8279 | } else { |
| 8280 | MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign); |
| 8281 | } |
| 8282 | |
Matt Arsenault | 0b7de79 | 2020-07-26 21:25:10 -0400 | [diff] [blame] | 8283 | MI.eraseFromParent(); |
| 8284 | return Legalized; |
| 8285 | } |
| 8286 | |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8287 | LegalizerHelper::LegalizeResult |
| 8288 | LegalizerHelper::lowerShuffleVector(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8289 | auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] = |
| 8290 | MI.getFirst3RegLLTs(); |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8291 | LLT IdxTy = LLT::scalar(32); |
| 8292 | |
Eli Friedman | e68e4cb | 2020-01-13 15:32:45 -0800 | [diff] [blame] | 8293 | ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8294 | Register Undef; |
| 8295 | SmallVector<Register, 32> BuildVec; |
Jay Foad | 71ca53b | 2023-09-04 18:32:43 +0100 | [diff] [blame] | 8296 | LLT EltTy = DstTy.getScalarType(); |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8297 | |
| 8298 | for (int Idx : Mask) { |
| 8299 | if (Idx < 0) { |
| 8300 | if (!Undef.isValid()) |
| 8301 | Undef = MIRBuilder.buildUndef(EltTy).getReg(0); |
| 8302 | BuildVec.push_back(Undef); |
| 8303 | continue; |
| 8304 | } |
| 8305 | |
Aditya Nandakumar | 615eee6 | 2019-08-13 21:49:11 +0000 | [diff] [blame] | 8306 | if (Src0Ty.isScalar()) { |
| 8307 | BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg); |
| 8308 | } else { |
Aditya Nandakumar | c65ac86 | 2019-08-14 01:23:33 +0000 | [diff] [blame] | 8309 | int NumElts = Src0Ty.getNumElements(); |
Aditya Nandakumar | 615eee6 | 2019-08-13 21:49:11 +0000 | [diff] [blame] | 8310 | Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg; |
| 8311 | int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts; |
| 8312 | auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx); |
| 8313 | auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK); |
| 8314 | BuildVec.push_back(Extract.getReg(0)); |
| 8315 | } |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8316 | } |
| 8317 | |
Jay Foad | 71ca53b | 2023-09-04 18:32:43 +0100 | [diff] [blame] | 8318 | if (DstTy.isScalar()) |
| 8319 | MIRBuilder.buildCopy(DstReg, BuildVec[0]); |
| 8320 | else |
| 8321 | MIRBuilder.buildBuildVector(DstReg, BuildVec); |
Matt Arsenault | 690645b | 2019-08-13 16:09:07 +0000 | [diff] [blame] | 8322 | MI.eraseFromParent(); |
| 8323 | return Legalized; |
| 8324 | } |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8325 | |
Lawrence Benson | 177ce19 | 2024-07-17 14:24:24 +0200 | [diff] [blame] | 8326 | LegalizerHelper::LegalizeResult |
| 8327 | LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) { |
| 8328 | auto [Dst, DstTy, Vec, VecTy, Mask, MaskTy, Passthru, PassthruTy] = |
| 8329 | MI.getFirst4RegLLTs(); |
| 8330 | |
| 8331 | if (VecTy.isScalableVector()) |
| 8332 | report_fatal_error("Cannot expand masked_compress for scalable vectors."); |
| 8333 | |
| 8334 | Align VecAlign = getStackTemporaryAlignment(VecTy); |
| 8335 | MachinePointerInfo PtrInfo; |
| 8336 | Register StackPtr = |
| 8337 | createStackTemporary(TypeSize::getFixed(VecTy.getSizeInBytes()), VecAlign, |
| 8338 | PtrInfo) |
| 8339 | .getReg(0); |
| 8340 | MachinePointerInfo ValPtrInfo = |
| 8341 | MachinePointerInfo::getUnknownStack(*MI.getMF()); |
| 8342 | |
| 8343 | LLT IdxTy = LLT::scalar(32); |
| 8344 | LLT ValTy = VecTy.getElementType(); |
| 8345 | Align ValAlign = getStackTemporaryAlignment(ValTy); |
| 8346 | |
| 8347 | auto OutPos = MIRBuilder.buildConstant(IdxTy, 0); |
| 8348 | |
| 8349 | bool HasPassthru = |
| 8350 | MRI.getVRegDef(Passthru)->getOpcode() != TargetOpcode::G_IMPLICIT_DEF; |
| 8351 | |
| 8352 | if (HasPassthru) |
| 8353 | MIRBuilder.buildStore(Passthru, StackPtr, PtrInfo, VecAlign); |
| 8354 | |
| 8355 | Register LastWriteVal; |
| 8356 | std::optional<APInt> PassthruSplatVal = |
| 8357 | isConstantOrConstantSplatVector(*MRI.getVRegDef(Passthru), MRI); |
| 8358 | |
| 8359 | if (PassthruSplatVal.has_value()) { |
| 8360 | LastWriteVal = |
| 8361 | MIRBuilder.buildConstant(ValTy, PassthruSplatVal.value()).getReg(0); |
| 8362 | } else if (HasPassthru) { |
| 8363 | auto Popcount = MIRBuilder.buildZExt(MaskTy.changeElementSize(32), Mask); |
| 8364 | Popcount = MIRBuilder.buildInstr(TargetOpcode::G_VECREDUCE_ADD, |
| 8365 | {LLT::scalar(32)}, {Popcount}); |
| 8366 | |
| 8367 | Register LastElmtPtr = |
| 8368 | getVectorElementPointer(StackPtr, VecTy, Popcount.getReg(0)); |
| 8369 | LastWriteVal = |
| 8370 | MIRBuilder.buildLoad(ValTy, LastElmtPtr, ValPtrInfo, ValAlign) |
| 8371 | .getReg(0); |
| 8372 | } |
| 8373 | |
| 8374 | unsigned NumElmts = VecTy.getNumElements(); |
| 8375 | for (unsigned I = 0; I < NumElmts; ++I) { |
| 8376 | auto Idx = MIRBuilder.buildConstant(IdxTy, I); |
| 8377 | auto Val = MIRBuilder.buildExtractVectorElement(ValTy, Vec, Idx); |
| 8378 | Register ElmtPtr = |
| 8379 | getVectorElementPointer(StackPtr, VecTy, OutPos.getReg(0)); |
| 8380 | MIRBuilder.buildStore(Val, ElmtPtr, ValPtrInfo, ValAlign); |
| 8381 | |
| 8382 | LLT MaskITy = MaskTy.getElementType(); |
| 8383 | auto MaskI = MIRBuilder.buildExtractVectorElement(MaskITy, Mask, Idx); |
| 8384 | if (MaskITy.getSizeInBits() > 1) |
| 8385 | MaskI = MIRBuilder.buildTrunc(LLT::scalar(1), MaskI); |
| 8386 | |
| 8387 | MaskI = MIRBuilder.buildZExt(IdxTy, MaskI); |
| 8388 | OutPos = MIRBuilder.buildAdd(IdxTy, OutPos, MaskI); |
| 8389 | |
| 8390 | if (HasPassthru && I == NumElmts - 1) { |
| 8391 | auto EndOfVector = |
| 8392 | MIRBuilder.buildConstant(IdxTy, VecTy.getNumElements() - 1); |
| 8393 | auto AllLanesSelected = MIRBuilder.buildICmp( |
| 8394 | CmpInst::ICMP_UGT, LLT::scalar(1), OutPos, EndOfVector); |
| 8395 | OutPos = MIRBuilder.buildInstr(TargetOpcode::G_UMIN, {IdxTy}, |
| 8396 | {OutPos, EndOfVector}); |
| 8397 | ElmtPtr = getVectorElementPointer(StackPtr, VecTy, OutPos.getReg(0)); |
| 8398 | |
| 8399 | LastWriteVal = |
| 8400 | MIRBuilder.buildSelect(ValTy, AllLanesSelected, Val, LastWriteVal) |
| 8401 | .getReg(0); |
| 8402 | MIRBuilder.buildStore(LastWriteVal, ElmtPtr, ValPtrInfo, ValAlign); |
| 8403 | } |
| 8404 | } |
| 8405 | |
| 8406 | // TODO: Use StackPtr's FrameIndex alignment. |
| 8407 | MIRBuilder.buildLoad(Dst, StackPtr, PtrInfo, VecAlign); |
| 8408 | |
| 8409 | MI.eraseFromParent(); |
| 8410 | return Legalized; |
| 8411 | } |
| 8412 | |
Momchil Velikov | c1140d4 | 2023-12-04 09:44:02 +0000 | [diff] [blame] | 8413 | Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg, |
| 8414 | Register AllocSize, |
| 8415 | Align Alignment, |
| 8416 | LLT PtrTy) { |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8417 | LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); |
| 8418 | |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8419 | auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg); |
| 8420 | SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp); |
| 8421 | |
| 8422 | // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't |
| 8423 | // have to generate an extra instruction to negate the alloc and then use |
Daniel Sanders | e74c5b9 | 2019-11-01 13:18:00 -0700 | [diff] [blame] | 8424 | // G_PTR_ADD to add the negative offset. |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8425 | auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize); |
Guillaume Chatelet | 9f5c786 | 2020-04-03 08:10:59 +0000 | [diff] [blame] | 8426 | if (Alignment > Align(1)) { |
| 8427 | APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true); |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8428 | AlignMask.negate(); |
| 8429 | auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask); |
| 8430 | Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst); |
| 8431 | } |
| 8432 | |
Momchil Velikov | c1140d4 | 2023-12-04 09:44:02 +0000 | [diff] [blame] | 8433 | return MIRBuilder.buildCast(PtrTy, Alloc).getReg(0); |
| 8434 | } |
| 8435 | |
| 8436 | LegalizerHelper::LegalizeResult |
| 8437 | LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) { |
| 8438 | const auto &MF = *MI.getMF(); |
| 8439 | const auto &TFI = *MF.getSubtarget().getFrameLowering(); |
| 8440 | if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp) |
| 8441 | return UnableToLegalize; |
| 8442 | |
| 8443 | Register Dst = MI.getOperand(0).getReg(); |
| 8444 | Register AllocSize = MI.getOperand(1).getReg(); |
| 8445 | Align Alignment = assumeAligned(MI.getOperand(2).getImm()); |
| 8446 | |
| 8447 | LLT PtrTy = MRI.getType(Dst); |
| 8448 | Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
| 8449 | Register SPTmp = |
| 8450 | getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy); |
| 8451 | |
Amara Emerson | e20b91c | 2019-08-27 19:54:27 +0000 | [diff] [blame] | 8452 | MIRBuilder.buildCopy(SPReg, SPTmp); |
| 8453 | MIRBuilder.buildCopy(Dst, SPTmp); |
| 8454 | |
| 8455 | MI.eraseFromParent(); |
| 8456 | return Legalized; |
| 8457 | } |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8458 | |
| 8459 | LegalizerHelper::LegalizeResult |
Matt Arsenault | 1ca0808 | 2023-07-29 19:12:24 -0400 | [diff] [blame] | 8460 | LegalizerHelper::lowerStackSave(MachineInstr &MI) { |
| 8461 | Register StackPtr = TLI.getStackPointerRegisterToSaveRestore(); |
| 8462 | if (!StackPtr) |
| 8463 | return UnableToLegalize; |
| 8464 | |
| 8465 | MIRBuilder.buildCopy(MI.getOperand(0), StackPtr); |
| 8466 | MI.eraseFromParent(); |
| 8467 | return Legalized; |
| 8468 | } |
| 8469 | |
| 8470 | LegalizerHelper::LegalizeResult |
| 8471 | LegalizerHelper::lowerStackRestore(MachineInstr &MI) { |
| 8472 | Register StackPtr = TLI.getStackPointerRegisterToSaveRestore(); |
| 8473 | if (!StackPtr) |
| 8474 | return UnableToLegalize; |
| 8475 | |
| 8476 | MIRBuilder.buildCopy(StackPtr, MI.getOperand(0)); |
| 8477 | MI.eraseFromParent(); |
| 8478 | return Legalized; |
| 8479 | } |
| 8480 | |
| 8481 | LegalizerHelper::LegalizeResult |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8482 | LegalizerHelper::lowerExtract(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8483 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8484 | unsigned Offset = MI.getOperand(2).getImm(); |
| 8485 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8486 | // Extract sub-vector or one element |
| 8487 | if (SrcTy.isVector()) { |
| 8488 | unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits(); |
| 8489 | unsigned DstSize = DstTy.getSizeInBits(); |
| 8490 | |
| 8491 | if ((Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) && |
| 8492 | (Offset + DstSize <= SrcTy.getSizeInBits())) { |
| 8493 | // Unmerge and allow access to each Src element for the artifact combiner. |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8494 | auto Unmerge = MIRBuilder.buildUnmerge(SrcTy.getElementType(), SrcReg); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8495 | |
| 8496 | // Take element(s) we need to extract and copy it (merge them). |
| 8497 | SmallVector<Register, 8> SubVectorElts; |
| 8498 | for (unsigned Idx = Offset / SrcEltSize; |
| 8499 | Idx < (Offset + DstSize) / SrcEltSize; ++Idx) { |
| 8500 | SubVectorElts.push_back(Unmerge.getReg(Idx)); |
| 8501 | } |
| 8502 | if (SubVectorElts.size() == 1) |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8503 | MIRBuilder.buildCopy(DstReg, SubVectorElts[0]); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8504 | else |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8505 | MIRBuilder.buildMergeLikeInstr(DstReg, SubVectorElts); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8506 | |
| 8507 | MI.eraseFromParent(); |
| 8508 | return Legalized; |
| 8509 | } |
| 8510 | } |
| 8511 | |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8512 | if (DstTy.isScalar() && |
| 8513 | (SrcTy.isScalar() || |
| 8514 | (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) { |
| 8515 | LLT SrcIntTy = SrcTy; |
| 8516 | if (!SrcTy.isScalar()) { |
| 8517 | SrcIntTy = LLT::scalar(SrcTy.getSizeInBits()); |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8518 | SrcReg = MIRBuilder.buildBitcast(SrcIntTy, SrcReg).getReg(0); |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8519 | } |
| 8520 | |
| 8521 | if (Offset == 0) |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8522 | MIRBuilder.buildTrunc(DstReg, SrcReg); |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8523 | else { |
| 8524 | auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset); |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8525 | auto Shr = MIRBuilder.buildLShr(SrcIntTy, SrcReg, ShiftAmt); |
| 8526 | MIRBuilder.buildTrunc(DstReg, Shr); |
Matt Arsenault | a5b9c75 | 2019-10-06 01:37:35 +0000 | [diff] [blame] | 8527 | } |
| 8528 | |
| 8529 | MI.eraseFromParent(); |
| 8530 | return Legalized; |
| 8531 | } |
| 8532 | |
| 8533 | return UnableToLegalize; |
| 8534 | } |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 8535 | |
| 8536 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8537 | auto [Dst, Src, InsertSrc] = MI.getFirst3Regs(); |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 8538 | uint64_t Offset = MI.getOperand(3).getImm(); |
| 8539 | |
| 8540 | LLT DstTy = MRI.getType(Src); |
| 8541 | LLT InsertTy = MRI.getType(InsertSrc); |
| 8542 | |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8543 | // Insert sub-vector or one element |
| 8544 | if (DstTy.isVector() && !InsertTy.isPointer()) { |
| 8545 | LLT EltTy = DstTy.getElementType(); |
| 8546 | unsigned EltSize = EltTy.getSizeInBits(); |
| 8547 | unsigned InsertSize = InsertTy.getSizeInBits(); |
| 8548 | |
| 8549 | if ((Offset % EltSize == 0) && (InsertSize % EltSize == 0) && |
| 8550 | (Offset + InsertSize <= DstTy.getSizeInBits())) { |
| 8551 | auto UnmergeSrc = MIRBuilder.buildUnmerge(EltTy, Src); |
| 8552 | SmallVector<Register, 8> DstElts; |
| 8553 | unsigned Idx = 0; |
| 8554 | // Elements from Src before insert start Offset |
| 8555 | for (; Idx < Offset / EltSize; ++Idx) { |
| 8556 | DstElts.push_back(UnmergeSrc.getReg(Idx)); |
| 8557 | } |
| 8558 | |
| 8559 | // Replace elements in Src with elements from InsertSrc |
| 8560 | if (InsertTy.getSizeInBits() > EltSize) { |
| 8561 | auto UnmergeInsertSrc = MIRBuilder.buildUnmerge(EltTy, InsertSrc); |
| 8562 | for (unsigned i = 0; Idx < (Offset + InsertSize) / EltSize; |
| 8563 | ++Idx, ++i) { |
| 8564 | DstElts.push_back(UnmergeInsertSrc.getReg(i)); |
| 8565 | } |
| 8566 | } else { |
| 8567 | DstElts.push_back(InsertSrc); |
| 8568 | ++Idx; |
| 8569 | } |
| 8570 | |
| 8571 | // Remaining elements from Src after insert |
| 8572 | for (; Idx < DstTy.getNumElements(); ++Idx) { |
| 8573 | DstElts.push_back(UnmergeSrc.getReg(Idx)); |
| 8574 | } |
| 8575 | |
Diana Picus | f95a5fb | 2023-01-09 11:59:00 +0100 | [diff] [blame] | 8576 | MIRBuilder.buildMergeLikeInstr(Dst, DstElts); |
Petar Avramovic | 29f88b9 | 2021-12-23 14:09:51 +0100 | [diff] [blame] | 8577 | MI.eraseFromParent(); |
| 8578 | return Legalized; |
| 8579 | } |
| 8580 | } |
| 8581 | |
Dominik Montada | 8ff2dcb1 | 2020-03-11 12:18:59 +0100 | [diff] [blame] | 8582 | if (InsertTy.isVector() || |
| 8583 | (DstTy.isVector() && DstTy.getElementType() != InsertTy)) |
| 8584 | return UnableToLegalize; |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 8585 | |
Dominik Montada | 8ff2dcb1 | 2020-03-11 12:18:59 +0100 | [diff] [blame] | 8586 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 8587 | if ((DstTy.isPointer() && |
| 8588 | DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) || |
| 8589 | (InsertTy.isPointer() && |
| 8590 | DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) { |
| 8591 | LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n"); |
| 8592 | return UnableToLegalize; |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 8593 | } |
| 8594 | |
Dominik Montada | 8ff2dcb1 | 2020-03-11 12:18:59 +0100 | [diff] [blame] | 8595 | LLT IntDstTy = DstTy; |
| 8596 | |
| 8597 | if (!DstTy.isScalar()) { |
| 8598 | IntDstTy = LLT::scalar(DstTy.getSizeInBits()); |
| 8599 | Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0); |
| 8600 | } |
| 8601 | |
| 8602 | if (!InsertTy.isScalar()) { |
| 8603 | const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits()); |
| 8604 | InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0); |
| 8605 | } |
| 8606 | |
| 8607 | Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0); |
| 8608 | if (Offset != 0) { |
| 8609 | auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset); |
| 8610 | ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0); |
| 8611 | } |
| 8612 | |
| 8613 | APInt MaskVal = APInt::getBitsSetWithWrap( |
| 8614 | DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset); |
| 8615 | |
| 8616 | auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal); |
| 8617 | auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask); |
| 8618 | auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc); |
| 8619 | |
| 8620 | MIRBuilder.buildCast(Dst, Or); |
| 8621 | MI.eraseFromParent(); |
| 8622 | return Legalized; |
Matt Arsenault | 4bcdcad | 2019-10-07 19:13:27 +0000 | [diff] [blame] | 8623 | } |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8624 | |
| 8625 | LegalizerHelper::LegalizeResult |
| 8626 | LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8627 | auto [Dst0, Dst0Ty, Dst1, Dst1Ty, LHS, LHSTy, RHS, RHSTy] = |
| 8628 | MI.getFirst4RegLLTs(); |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8629 | const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO; |
| 8630 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8631 | LLT Ty = Dst0Ty; |
| 8632 | LLT BoolTy = Dst1Ty; |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8633 | |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8634 | Register NewDst0 = MRI.cloneVirtualRegister(Dst0); |
| 8635 | |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8636 | if (IsAdd) |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8637 | MIRBuilder.buildAdd(NewDst0, LHS, RHS); |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8638 | else |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8639 | MIRBuilder.buildSub(NewDst0, LHS, RHS); |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8640 | |
| 8641 | // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow. |
| 8642 | |
| 8643 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
| 8644 | |
| 8645 | // For an addition, the result should be less than one of the operands (LHS) |
| 8646 | // if and only if the other operand (RHS) is negative, otherwise there will |
| 8647 | // be overflow. |
| 8648 | // For a subtraction, the result should be less than one of the operands |
| 8649 | // (LHS) if and only if the other operand (RHS) is (non-zero) positive, |
| 8650 | // otherwise there will be overflow. |
| 8651 | auto ResultLowerThanLHS = |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8652 | MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, NewDst0, LHS); |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8653 | auto ConditionRHS = MIRBuilder.buildICmp( |
| 8654 | IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero); |
| 8655 | |
| 8656 | MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8657 | |
| 8658 | MIRBuilder.buildCopy(Dst0, NewDst0); |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8659 | MI.eraseFromParent(); |
Shilei Tian | 3a106e5 | 2024-03-29 15:59:50 -0400 | [diff] [blame] | 8660 | |
Matt Arsenault | 34ed76e | 2019-10-16 20:46:32 +0000 | [diff] [blame] | 8661 | return Legalized; |
| 8662 | } |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 8663 | |
| 8664 | LegalizerHelper::LegalizeResult |
Jay Foad | b35833b | 2020-07-12 14:18:45 -0400 | [diff] [blame] | 8665 | LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8666 | auto [Res, LHS, RHS] = MI.getFirst3Regs(); |
Jay Foad | b35833b | 2020-07-12 14:18:45 -0400 | [diff] [blame] | 8667 | LLT Ty = MRI.getType(Res); |
| 8668 | bool IsSigned; |
| 8669 | bool IsAdd; |
| 8670 | unsigned BaseOp; |
| 8671 | switch (MI.getOpcode()) { |
| 8672 | default: |
| 8673 | llvm_unreachable("unexpected addsat/subsat opcode"); |
| 8674 | case TargetOpcode::G_UADDSAT: |
| 8675 | IsSigned = false; |
| 8676 | IsAdd = true; |
| 8677 | BaseOp = TargetOpcode::G_ADD; |
| 8678 | break; |
| 8679 | case TargetOpcode::G_SADDSAT: |
| 8680 | IsSigned = true; |
| 8681 | IsAdd = true; |
| 8682 | BaseOp = TargetOpcode::G_ADD; |
| 8683 | break; |
| 8684 | case TargetOpcode::G_USUBSAT: |
| 8685 | IsSigned = false; |
| 8686 | IsAdd = false; |
| 8687 | BaseOp = TargetOpcode::G_SUB; |
| 8688 | break; |
| 8689 | case TargetOpcode::G_SSUBSAT: |
| 8690 | IsSigned = true; |
| 8691 | IsAdd = false; |
| 8692 | BaseOp = TargetOpcode::G_SUB; |
| 8693 | break; |
| 8694 | } |
| 8695 | |
| 8696 | if (IsSigned) { |
| 8697 | // sadd.sat(a, b) -> |
| 8698 | // hi = 0x7fffffff - smax(a, 0) |
| 8699 | // lo = 0x80000000 - smin(a, 0) |
| 8700 | // a + smin(smax(lo, b), hi) |
| 8701 | // ssub.sat(a, b) -> |
| 8702 | // lo = smax(a, -1) - 0x7fffffff |
| 8703 | // hi = smin(a, -1) - 0x80000000 |
| 8704 | // a - smin(smax(lo, b), hi) |
| 8705 | // TODO: AMDGPU can use a "median of 3" instruction here: |
| 8706 | // a +/- med3(lo, b, hi) |
| 8707 | uint64_t NumBits = Ty.getScalarSizeInBits(); |
| 8708 | auto MaxVal = |
| 8709 | MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits)); |
| 8710 | auto MinVal = |
| 8711 | MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); |
| 8712 | MachineInstrBuilder Hi, Lo; |
| 8713 | if (IsAdd) { |
| 8714 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
| 8715 | Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero)); |
| 8716 | Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero)); |
| 8717 | } else { |
| 8718 | auto NegOne = MIRBuilder.buildConstant(Ty, -1); |
| 8719 | Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne), |
| 8720 | MaxVal); |
| 8721 | Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne), |
| 8722 | MinVal); |
| 8723 | } |
| 8724 | auto RHSClamped = |
| 8725 | MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi); |
| 8726 | MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped}); |
| 8727 | } else { |
| 8728 | // uadd.sat(a, b) -> a + umin(~a, b) |
| 8729 | // usub.sat(a, b) -> a - umin(a, b) |
| 8730 | Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS; |
| 8731 | auto Min = MIRBuilder.buildUMin(Ty, Not, RHS); |
| 8732 | MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min}); |
| 8733 | } |
| 8734 | |
| 8735 | MI.eraseFromParent(); |
| 8736 | return Legalized; |
| 8737 | } |
| 8738 | |
| 8739 | LegalizerHelper::LegalizeResult |
| 8740 | LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8741 | auto [Res, LHS, RHS] = MI.getFirst3Regs(); |
Jay Foad | b35833b | 2020-07-12 14:18:45 -0400 | [diff] [blame] | 8742 | LLT Ty = MRI.getType(Res); |
| 8743 | LLT BoolTy = Ty.changeElementSize(1); |
| 8744 | bool IsSigned; |
| 8745 | bool IsAdd; |
| 8746 | unsigned OverflowOp; |
| 8747 | switch (MI.getOpcode()) { |
| 8748 | default: |
| 8749 | llvm_unreachable("unexpected addsat/subsat opcode"); |
| 8750 | case TargetOpcode::G_UADDSAT: |
| 8751 | IsSigned = false; |
| 8752 | IsAdd = true; |
| 8753 | OverflowOp = TargetOpcode::G_UADDO; |
| 8754 | break; |
| 8755 | case TargetOpcode::G_SADDSAT: |
| 8756 | IsSigned = true; |
| 8757 | IsAdd = true; |
| 8758 | OverflowOp = TargetOpcode::G_SADDO; |
| 8759 | break; |
| 8760 | case TargetOpcode::G_USUBSAT: |
| 8761 | IsSigned = false; |
| 8762 | IsAdd = false; |
| 8763 | OverflowOp = TargetOpcode::G_USUBO; |
| 8764 | break; |
| 8765 | case TargetOpcode::G_SSUBSAT: |
| 8766 | IsSigned = true; |
| 8767 | IsAdd = false; |
| 8768 | OverflowOp = TargetOpcode::G_SSUBO; |
| 8769 | break; |
| 8770 | } |
| 8771 | |
| 8772 | auto OverflowRes = |
| 8773 | MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS}); |
| 8774 | Register Tmp = OverflowRes.getReg(0); |
| 8775 | Register Ov = OverflowRes.getReg(1); |
| 8776 | MachineInstrBuilder Clamp; |
| 8777 | if (IsSigned) { |
| 8778 | // sadd.sat(a, b) -> |
| 8779 | // {tmp, ov} = saddo(a, b) |
| 8780 | // ov ? (tmp >>s 31) + 0x80000000 : r |
| 8781 | // ssub.sat(a, b) -> |
| 8782 | // {tmp, ov} = ssubo(a, b) |
| 8783 | // ov ? (tmp >>s 31) + 0x80000000 : r |
| 8784 | uint64_t NumBits = Ty.getScalarSizeInBits(); |
| 8785 | auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1); |
| 8786 | auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount); |
| 8787 | auto MinVal = |
| 8788 | MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits)); |
| 8789 | Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal); |
| 8790 | } else { |
| 8791 | // uadd.sat(a, b) -> |
| 8792 | // {tmp, ov} = uaddo(a, b) |
| 8793 | // ov ? 0xffffffff : tmp |
| 8794 | // usub.sat(a, b) -> |
| 8795 | // {tmp, ov} = usubo(a, b) |
| 8796 | // ov ? 0 : tmp |
| 8797 | Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0); |
| 8798 | } |
| 8799 | MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp); |
| 8800 | |
| 8801 | MI.eraseFromParent(); |
| 8802 | return Legalized; |
| 8803 | } |
| 8804 | |
| 8805 | LegalizerHelper::LegalizeResult |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 8806 | LegalizerHelper::lowerShlSat(MachineInstr &MI) { |
| 8807 | assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT || |
| 8808 | MI.getOpcode() == TargetOpcode::G_USHLSAT) && |
| 8809 | "Expected shlsat opcode!"); |
| 8810 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT; |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8811 | auto [Res, LHS, RHS] = MI.getFirst3Regs(); |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 8812 | LLT Ty = MRI.getType(Res); |
| 8813 | LLT BoolTy = Ty.changeElementSize(1); |
| 8814 | |
| 8815 | unsigned BW = Ty.getScalarSizeInBits(); |
| 8816 | auto Result = MIRBuilder.buildShl(Ty, LHS, RHS); |
| 8817 | auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS) |
| 8818 | : MIRBuilder.buildLShr(Ty, Result, RHS); |
| 8819 | |
| 8820 | MachineInstrBuilder SatVal; |
| 8821 | if (IsSigned) { |
| 8822 | auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW)); |
| 8823 | auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW)); |
| 8824 | auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS, |
| 8825 | MIRBuilder.buildConstant(Ty, 0)); |
| 8826 | SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax); |
| 8827 | } else { |
| 8828 | SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW)); |
| 8829 | } |
Mirko Brkusanin | 4cf6dd5 | 2020-11-16 17:43:15 +0100 | [diff] [blame] | 8830 | auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, BoolTy, LHS, Orig); |
Bevin Hansson | 5de6c56 | 2020-07-16 17:02:04 +0200 | [diff] [blame] | 8831 | MIRBuilder.buildSelect(Res, Ov, SatVal, Result); |
| 8832 | |
| 8833 | MI.eraseFromParent(); |
| 8834 | return Legalized; |
| 8835 | } |
| 8836 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8837 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) { |
| 8838 | auto [Dst, Src] = MI.getFirst2Regs(); |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 8839 | const LLT Ty = MRI.getType(Src); |
Matt Arsenault | 2e77362 | 2020-02-14 11:51:57 -0500 | [diff] [blame] | 8840 | unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8; |
Petar Avramovic | 94a24e7 | 2019-12-30 11:13:22 +0100 | [diff] [blame] | 8841 | unsigned BaseShiftAmt = (SizeInBytes - 1) * 8; |
| 8842 | |
| 8843 | // Swap most and least significant byte, set remaining bytes in Res to zero. |
| 8844 | auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt); |
| 8845 | auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt); |
| 8846 | auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); |
| 8847 | auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft); |
| 8848 | |
| 8849 | // Set i-th high/low byte in Res to i-th low/high byte from Src. |
| 8850 | for (unsigned i = 1; i < SizeInBytes / 2; ++i) { |
| 8851 | // AND with Mask leaves byte i unchanged and sets remaining bytes to 0. |
| 8852 | APInt APMask(SizeInBytes * 8, 0xFF << (i * 8)); |
| 8853 | auto Mask = MIRBuilder.buildConstant(Ty, APMask); |
| 8854 | auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i); |
| 8855 | // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt. |
| 8856 | auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask); |
| 8857 | auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt); |
| 8858 | Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft); |
| 8859 | // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask. |
| 8860 | auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt); |
| 8861 | auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask); |
| 8862 | Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight); |
| 8863 | } |
| 8864 | Res.getInstr()->getOperand(0).setReg(Dst); |
| 8865 | |
| 8866 | MI.eraseFromParent(); |
| 8867 | return Legalized; |
| 8868 | } |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8869 | |
| 8870 | //{ (Src & Mask) >> N } | { (Src << N) & Mask } |
| 8871 | static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, |
Yingwei Zheng | 6c1932f | 2024-03-23 14:57:35 +0800 | [diff] [blame] | 8872 | MachineInstrBuilder Src, const APInt &Mask) { |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8873 | const LLT Ty = Dst.getLLTTy(*B.getMRI()); |
| 8874 | MachineInstrBuilder C_N = B.buildConstant(Ty, N); |
| 8875 | MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask); |
| 8876 | auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N); |
| 8877 | auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0); |
| 8878 | return B.buildOr(Dst, LHS, RHS); |
| 8879 | } |
| 8880 | |
| 8881 | LegalizerHelper::LegalizeResult |
| 8882 | LegalizerHelper::lowerBitreverse(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8883 | auto [Dst, Src] = MI.getFirst2Regs(); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8884 | const LLT Ty = MRI.getType(Src); |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8885 | unsigned Size = Ty.getScalarSizeInBits(); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8886 | |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8887 | if (Size >= 8) { |
| 8888 | MachineInstrBuilder BSWAP = |
| 8889 | MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src}); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8890 | |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8891 | // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654 |
| 8892 | // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4] |
| 8893 | // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0] |
| 8894 | MachineInstrBuilder Swap4 = |
| 8895 | SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0))); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8896 | |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8897 | // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76 |
| 8898 | // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2] |
| 8899 | // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC] |
| 8900 | MachineInstrBuilder Swap2 = |
| 8901 | SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC))); |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8902 | |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8903 | // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 |
| 8904 | // 6|7 |
| 8905 | // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1] |
| 8906 | // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA] |
| 8907 | SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA))); |
| 8908 | } else { |
| 8909 | // Expand bitreverse for types smaller than 8 bits. |
| 8910 | MachineInstrBuilder Tmp; |
| 8911 | for (unsigned I = 0, J = Size - 1; I < Size; ++I, --J) { |
| 8912 | MachineInstrBuilder Tmp2; |
| 8913 | if (I < J) { |
| 8914 | auto ShAmt = MIRBuilder.buildConstant(Ty, J - I); |
| 8915 | Tmp2 = MIRBuilder.buildShl(Ty, Src, ShAmt); |
| 8916 | } else { |
| 8917 | auto ShAmt = MIRBuilder.buildConstant(Ty, I - J); |
| 8918 | Tmp2 = MIRBuilder.buildLShr(Ty, Src, ShAmt); |
| 8919 | } |
| 8920 | |
Simon Pilgrim | 4e251e7 | 2024-05-29 17:57:23 +0100 | [diff] [blame] | 8921 | auto Mask = MIRBuilder.buildConstant(Ty, 1ULL << J); |
Yingwei Zheng | 24ddce6 | 2024-05-29 21:42:08 +0800 | [diff] [blame] | 8922 | Tmp2 = MIRBuilder.buildAnd(Ty, Tmp2, Mask); |
| 8923 | if (I == 0) |
| 8924 | Tmp = Tmp2; |
| 8925 | else |
| 8926 | Tmp = MIRBuilder.buildOr(Ty, Tmp, Tmp2); |
| 8927 | } |
| 8928 | MIRBuilder.buildCopy(Dst, Tmp); |
| 8929 | } |
Petar Avramovic | 98f72a5 | 2019-12-30 18:06:29 +0100 | [diff] [blame] | 8930 | |
| 8931 | MI.eraseFromParent(); |
| 8932 | return Legalized; |
| 8933 | } |
Matt Arsenault | 0ea3c72 | 2019-12-27 19:26:51 -0500 | [diff] [blame] | 8934 | |
| 8935 | LegalizerHelper::LegalizeResult |
Matt Arsenault | c5c1bb3 | 2020-01-12 13:29:44 -0500 | [diff] [blame] | 8936 | LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) { |
Matt Arsenault | 0ea3c72 | 2019-12-27 19:26:51 -0500 | [diff] [blame] | 8937 | MachineFunction &MF = MIRBuilder.getMF(); |
Matt Arsenault | c5c1bb3 | 2020-01-12 13:29:44 -0500 | [diff] [blame] | 8938 | |
| 8939 | bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER; |
| 8940 | int NameOpIdx = IsRead ? 1 : 0; |
| 8941 | int ValRegIndex = IsRead ? 0 : 1; |
| 8942 | |
| 8943 | Register ValReg = MI.getOperand(ValRegIndex).getReg(); |
| 8944 | const LLT Ty = MRI.getType(ValReg); |
| 8945 | const MDString *RegStr = cast<MDString>( |
| 8946 | cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0)); |
| 8947 | |
Matt Arsenault | adbcc8e | 2020-07-31 11:41:05 -0400 | [diff] [blame] | 8948 | Register PhysReg = TLI.getRegisterByName(RegStr->getString().data(), Ty, MF); |
Matt Arsenault | c5c1bb3 | 2020-01-12 13:29:44 -0500 | [diff] [blame] | 8949 | if (!PhysReg.isValid()) |
Matt Arsenault | 0ea3c72 | 2019-12-27 19:26:51 -0500 | [diff] [blame] | 8950 | return UnableToLegalize; |
| 8951 | |
Matt Arsenault | c5c1bb3 | 2020-01-12 13:29:44 -0500 | [diff] [blame] | 8952 | if (IsRead) |
| 8953 | MIRBuilder.buildCopy(ValReg, PhysReg); |
| 8954 | else |
| 8955 | MIRBuilder.buildCopy(PhysReg, ValReg); |
| 8956 | |
Matt Arsenault | 0ea3c72 | 2019-12-27 19:26:51 -0500 | [diff] [blame] | 8957 | MI.eraseFromParent(); |
| 8958 | return Legalized; |
| 8959 | } |
Pushpinder Singh | 41d6669 | 2020-08-10 05:47:50 -0400 | [diff] [blame] | 8960 | |
| 8961 | LegalizerHelper::LegalizeResult |
| 8962 | LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) { |
| 8963 | bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH; |
| 8964 | unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; |
| 8965 | Register Result = MI.getOperand(0).getReg(); |
| 8966 | LLT OrigTy = MRI.getType(Result); |
| 8967 | auto SizeInBits = OrigTy.getScalarSizeInBits(); |
| 8968 | LLT WideTy = OrigTy.changeElementSize(SizeInBits * 2); |
| 8969 | |
| 8970 | auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)}); |
| 8971 | auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)}); |
| 8972 | auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS); |
| 8973 | unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR; |
| 8974 | |
| 8975 | auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits); |
| 8976 | auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt}); |
| 8977 | MIRBuilder.buildTrunc(Result, Shifted); |
| 8978 | |
| 8979 | MI.eraseFromParent(); |
| 8980 | return Legalized; |
| 8981 | } |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 8982 | |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 8983 | LegalizerHelper::LegalizeResult |
| 8984 | LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 8985 | auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs(); |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 8986 | FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(2).getImm()); |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 8987 | |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 8988 | if (Mask == fcNone) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 8989 | MIRBuilder.buildConstant(DstReg, 0); |
| 8990 | MI.eraseFromParent(); |
| 8991 | return Legalized; |
| 8992 | } |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 8993 | if (Mask == fcAllFlags) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 8994 | MIRBuilder.buildConstant(DstReg, 1); |
| 8995 | MI.eraseFromParent(); |
| 8996 | return Legalized; |
| 8997 | } |
| 8998 | |
Matt Arsenault | 61820f8 | 2023-02-02 10:28:05 -0400 | [diff] [blame] | 8999 | // TODO: Try inverting the test with getInvertedFPClassTest like the DAG |
| 9000 | // version |
| 9001 | |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9002 | unsigned BitSize = SrcTy.getScalarSizeInBits(); |
| 9003 | const fltSemantics &Semantics = getFltSemanticForLLT(SrcTy.getScalarType()); |
| 9004 | |
| 9005 | LLT IntTy = LLT::scalar(BitSize); |
| 9006 | if (SrcTy.isVector()) |
| 9007 | IntTy = LLT::vector(SrcTy.getElementCount(), IntTy); |
| 9008 | auto AsInt = MIRBuilder.buildCopy(IntTy, SrcReg); |
| 9009 | |
| 9010 | // Various masks. |
| 9011 | APInt SignBit = APInt::getSignMask(BitSize); |
| 9012 | APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign. |
| 9013 | APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit. |
| 9014 | APInt ExpMask = Inf; |
| 9015 | APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf; |
| 9016 | APInt QNaNBitMask = |
| 9017 | APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1); |
Kazu Hirata | b7ffd96 | 2023-02-19 22:54:23 -0800 | [diff] [blame] | 9018 | APInt InvertionMask = APInt::getAllOnes(DstTy.getScalarSizeInBits()); |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9019 | |
| 9020 | auto SignBitC = MIRBuilder.buildConstant(IntTy, SignBit); |
| 9021 | auto ValueMaskC = MIRBuilder.buildConstant(IntTy, ValueMask); |
| 9022 | auto InfC = MIRBuilder.buildConstant(IntTy, Inf); |
| 9023 | auto ExpMaskC = MIRBuilder.buildConstant(IntTy, ExpMask); |
| 9024 | auto ZeroC = MIRBuilder.buildConstant(IntTy, 0); |
| 9025 | |
| 9026 | auto Abs = MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC); |
| 9027 | auto Sign = |
| 9028 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs); |
| 9029 | |
| 9030 | auto Res = MIRBuilder.buildConstant(DstTy, 0); |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 9031 | // Clang doesn't support capture of structured bindings: |
| 9032 | LLT DstTyCopy = DstTy; |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9033 | const auto appendToRes = [&](MachineInstrBuilder ToAppend) { |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 9034 | Res = MIRBuilder.buildOr(DstTyCopy, Res, ToAppend); |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9035 | }; |
| 9036 | |
| 9037 | // Tests that involve more than one class should be processed first. |
| 9038 | if ((Mask & fcFinite) == fcFinite) { |
| 9039 | // finite(V) ==> abs(V) u< exp_mask |
| 9040 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs, |
| 9041 | ExpMaskC)); |
| 9042 | Mask &= ~fcFinite; |
| 9043 | } else if ((Mask & fcFinite) == fcPosFinite) { |
| 9044 | // finite(V) && V > 0 ==> V u< exp_mask |
| 9045 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, AsInt, |
| 9046 | ExpMaskC)); |
| 9047 | Mask &= ~fcPosFinite; |
| 9048 | } else if ((Mask & fcFinite) == fcNegFinite) { |
| 9049 | // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1 |
| 9050 | auto Cmp = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs, |
| 9051 | ExpMaskC); |
| 9052 | auto And = MIRBuilder.buildAnd(DstTy, Cmp, Sign); |
| 9053 | appendToRes(And); |
| 9054 | Mask &= ~fcNegFinite; |
| 9055 | } |
| 9056 | |
Matt Arsenault | 61820f8 | 2023-02-02 10:28:05 -0400 | [diff] [blame] | 9057 | if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) { |
| 9058 | // fcZero | fcSubnormal => test all exponent bits are 0 |
| 9059 | // TODO: Handle sign bit specific cases |
| 9060 | // TODO: Handle inverted case |
| 9061 | if (PartialCheck == (fcZero | fcSubnormal)) { |
| 9062 | auto ExpBits = MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC); |
| 9063 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, |
| 9064 | ExpBits, ZeroC)); |
| 9065 | Mask &= ~PartialCheck; |
| 9066 | } |
| 9067 | } |
| 9068 | |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9069 | // Check for individual classes. |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 9070 | if (FPClassTest PartialCheck = Mask & fcZero) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9071 | if (PartialCheck == fcPosZero) |
| 9072 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, |
| 9073 | AsInt, ZeroC)); |
| 9074 | else if (PartialCheck == fcZero) |
| 9075 | appendToRes( |
| 9076 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, ZeroC)); |
| 9077 | else // fcNegZero |
| 9078 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, |
| 9079 | AsInt, SignBitC)); |
| 9080 | } |
| 9081 | |
Matt Arsenault | 9356ec1 | 2023-02-02 10:14:36 -0400 | [diff] [blame] | 9082 | if (FPClassTest PartialCheck = Mask & fcSubnormal) { |
| 9083 | // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set) |
| 9084 | // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set) |
| 9085 | auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs; |
| 9086 | auto OneC = MIRBuilder.buildConstant(IntTy, 1); |
| 9087 | auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC); |
| 9088 | auto SubnormalRes = |
| 9089 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne, |
| 9090 | MIRBuilder.buildConstant(IntTy, AllOneMantissa)); |
| 9091 | if (PartialCheck == fcNegSubnormal) |
| 9092 | SubnormalRes = MIRBuilder.buildAnd(DstTy, SubnormalRes, Sign); |
| 9093 | appendToRes(SubnormalRes); |
| 9094 | } |
| 9095 | |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 9096 | if (FPClassTest PartialCheck = Mask & fcInf) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9097 | if (PartialCheck == fcPosInf) |
| 9098 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, |
| 9099 | AsInt, InfC)); |
| 9100 | else if (PartialCheck == fcInf) |
| 9101 | appendToRes( |
| 9102 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC)); |
| 9103 | else { // fcNegInf |
| 9104 | APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt(); |
| 9105 | auto NegInfC = MIRBuilder.buildConstant(IntTy, NegInf); |
| 9106 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, |
| 9107 | AsInt, NegInfC)); |
| 9108 | } |
| 9109 | } |
| 9110 | |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 9111 | if (FPClassTest PartialCheck = Mask & fcNan) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9112 | auto InfWithQnanBitC = MIRBuilder.buildConstant(IntTy, Inf | QNaNBitMask); |
| 9113 | if (PartialCheck == fcNan) { |
| 9114 | // isnan(V) ==> abs(V) u> int(inf) |
| 9115 | appendToRes( |
| 9116 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC)); |
| 9117 | } else if (PartialCheck == fcQNan) { |
| 9118 | // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit) |
| 9119 | appendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, Abs, |
| 9120 | InfWithQnanBitC)); |
| 9121 | } else { // fcSNan |
| 9122 | // issignaling(V) ==> abs(V) u> unsigned(Inf) && |
| 9123 | // abs(V) u< (unsigned(Inf) | quiet_bit) |
| 9124 | auto IsNan = |
| 9125 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC); |
| 9126 | auto IsNotQnan = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, |
| 9127 | Abs, InfWithQnanBitC); |
| 9128 | appendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan)); |
| 9129 | } |
| 9130 | } |
| 9131 | |
Matt Arsenault | 61f2f2c | 2023-03-17 09:21:57 -0400 | [diff] [blame] | 9132 | if (FPClassTest PartialCheck = Mask & fcNormal) { |
Janek van Oirschot | 587747d | 2022-12-06 20:36:07 +0000 | [diff] [blame] | 9133 | // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u< |
| 9134 | // (max_exp-1)) |
| 9135 | APInt ExpLSB = ExpMask & ~(ExpMask.shl(1)); |
| 9136 | auto ExpMinusOne = MIRBuilder.buildSub( |
| 9137 | IntTy, Abs, MIRBuilder.buildConstant(IntTy, ExpLSB)); |
| 9138 | APInt MaxExpMinusOne = ExpMask - ExpLSB; |
| 9139 | auto NormalRes = |
| 9140 | MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne, |
| 9141 | MIRBuilder.buildConstant(IntTy, MaxExpMinusOne)); |
| 9142 | if (PartialCheck == fcNegNormal) |
| 9143 | NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign); |
| 9144 | else if (PartialCheck == fcPosNormal) { |
| 9145 | auto PosSign = MIRBuilder.buildXor( |
| 9146 | DstTy, Sign, MIRBuilder.buildConstant(DstTy, InvertionMask)); |
| 9147 | NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign); |
| 9148 | } |
| 9149 | appendToRes(NormalRes); |
| 9150 | } |
| 9151 | |
| 9152 | MIRBuilder.buildCopy(DstReg, Res); |
| 9153 | MI.eraseFromParent(); |
| 9154 | return Legalized; |
| 9155 | } |
| 9156 | |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 9157 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) { |
Kai Nacke | f2d0bba | 2024-01-26 09:11:29 -0500 | [diff] [blame] | 9158 | // Implement G_SELECT in terms of XOR, AND, OR. |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 9159 | auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] = |
| 9160 | MI.getFirst4RegLLTs(); |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 9161 | |
Jay Foad | d57515bd | 2024-02-13 08:21:35 +0000 | [diff] [blame] | 9162 | bool IsEltPtr = DstTy.isPointerOrPointerVector(); |
Amara Emerson | f24f469 | 2022-09-11 16:28:44 +0100 | [diff] [blame] | 9163 | if (IsEltPtr) { |
| 9164 | LLT ScalarPtrTy = LLT::scalar(DstTy.getScalarSizeInBits()); |
| 9165 | LLT NewTy = DstTy.changeElementType(ScalarPtrTy); |
| 9166 | Op1Reg = MIRBuilder.buildPtrToInt(NewTy, Op1Reg).getReg(0); |
| 9167 | Op2Reg = MIRBuilder.buildPtrToInt(NewTy, Op2Reg).getReg(0); |
| 9168 | DstTy = NewTy; |
| 9169 | } |
| 9170 | |
Amara Emerson | 87ff156 | 2020-11-17 12:09:31 -0800 | [diff] [blame] | 9171 | if (MaskTy.isScalar()) { |
Kai Nacke | f2d0bba | 2024-01-26 09:11:29 -0500 | [diff] [blame] | 9172 | // Turn the scalar condition into a vector condition mask if needed. |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9173 | |
Amara Emerson | 87ff156 | 2020-11-17 12:09:31 -0800 | [diff] [blame] | 9174 | Register MaskElt = MaskReg; |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9175 | |
| 9176 | // The condition was potentially zero extended before, but we want a sign |
| 9177 | // extended boolean. |
Amara Emerson | 78833a4 | 2022-09-20 00:21:55 +0100 | [diff] [blame] | 9178 | if (MaskTy != LLT::scalar(1)) |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9179 | MaskElt = MIRBuilder.buildSExtInReg(MaskTy, MaskElt, 1).getReg(0); |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9180 | |
| 9181 | // Continue the sign extension (or truncate) to match the data type. |
Kai Nacke | f2d0bba | 2024-01-26 09:11:29 -0500 | [diff] [blame] | 9182 | MaskElt = |
| 9183 | MIRBuilder.buildSExtOrTrunc(DstTy.getScalarType(), MaskElt).getReg(0); |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9184 | |
Kai Nacke | f2d0bba | 2024-01-26 09:11:29 -0500 | [diff] [blame] | 9185 | if (DstTy.isVector()) { |
| 9186 | // Generate a vector splat idiom. |
| 9187 | auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt); |
| 9188 | MaskReg = ShufSplat.getReg(0); |
| 9189 | } else { |
| 9190 | MaskReg = MaskElt; |
| 9191 | } |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9192 | MaskTy = DstTy; |
Kai Nacke | f2d0bba | 2024-01-26 09:11:29 -0500 | [diff] [blame] | 9193 | } else if (!DstTy.isVector()) { |
| 9194 | // Cannot handle the case that mask is a vector and dst is a scalar. |
| 9195 | return UnableToLegalize; |
Amara Emerson | 87ff156 | 2020-11-17 12:09:31 -0800 | [diff] [blame] | 9196 | } |
| 9197 | |
Matt Arsenault | 3f2cc7c | 2022-04-11 21:11:26 -0400 | [diff] [blame] | 9198 | if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) { |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 9199 | return UnableToLegalize; |
Amara Emerson | 87ff156 | 2020-11-17 12:09:31 -0800 | [diff] [blame] | 9200 | } |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 9201 | |
| 9202 | auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg); |
| 9203 | auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg); |
| 9204 | auto NewOp2 = MIRBuilder.buildAnd(MaskTy, Op2Reg, NotMask); |
Amara Emerson | f24f469 | 2022-09-11 16:28:44 +0100 | [diff] [blame] | 9205 | if (IsEltPtr) { |
| 9206 | auto Or = MIRBuilder.buildOr(DstTy, NewOp1, NewOp2); |
| 9207 | MIRBuilder.buildIntToPtr(DstReg, Or); |
| 9208 | } else { |
| 9209 | MIRBuilder.buildOr(DstReg, NewOp1, NewOp2); |
| 9210 | } |
Amara Emerson | 0823219 | 2020-09-26 10:02:39 -0700 | [diff] [blame] | 9211 | MI.eraseFromParent(); |
| 9212 | return Legalized; |
Kazu Hirata | e3d3dbd33 | 2021-01-10 09:24:56 -0800 | [diff] [blame] | 9213 | } |
Christudasan Devadasan | 4c6ab48 | 2021-03-10 18:03:10 +0530 | [diff] [blame] | 9214 | |
| 9215 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) { |
| 9216 | // Split DIVREM into individual instructions. |
| 9217 | unsigned Opcode = MI.getOpcode(); |
| 9218 | |
| 9219 | MIRBuilder.buildInstr( |
| 9220 | Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV |
| 9221 | : TargetOpcode::G_UDIV, |
| 9222 | {MI.getOperand(0).getReg()}, {MI.getOperand(2), MI.getOperand(3)}); |
| 9223 | MIRBuilder.buildInstr( |
| 9224 | Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM |
| 9225 | : TargetOpcode::G_UREM, |
| 9226 | {MI.getOperand(1).getReg()}, {MI.getOperand(2), MI.getOperand(3)}); |
| 9227 | MI.eraseFromParent(); |
| 9228 | return Legalized; |
| 9229 | } |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 9230 | |
| 9231 | LegalizerHelper::LegalizeResult |
| 9232 | LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) { |
| 9233 | // Expand %res = G_ABS %a into: |
| 9234 | // %v1 = G_ASHR %a, scalar_size-1 |
| 9235 | // %v2 = G_ADD %a, %v1 |
| 9236 | // %res = G_XOR %v2, %v1 |
| 9237 | LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); |
| 9238 | Register OpReg = MI.getOperand(1).getReg(); |
| 9239 | auto ShiftAmt = |
| 9240 | MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1); |
| 9241 | auto Shift = MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt); |
| 9242 | auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift); |
| 9243 | MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift); |
| 9244 | MI.eraseFromParent(); |
| 9245 | return Legalized; |
| 9246 | } |
| 9247 | |
| 9248 | LegalizerHelper::LegalizeResult |
| 9249 | LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) { |
| 9250 | // Expand %res = G_ABS %a into: |
| 9251 | // %v1 = G_CONSTANT 0 |
| 9252 | // %v2 = G_SUB %v1, %a |
| 9253 | // %res = G_SMAX %a, %v2 |
| 9254 | Register SrcReg = MI.getOperand(1).getReg(); |
| 9255 | LLT Ty = MRI.getType(SrcReg); |
Madhur Amilkanthwar | 7bb87d5 | 2024-03-21 09:54:03 +0530 | [diff] [blame] | 9256 | auto Zero = MIRBuilder.buildConstant(Ty, 0); |
| 9257 | auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg); |
| 9258 | MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); |
| 9259 | MI.eraseFromParent(); |
| 9260 | return Legalized; |
| 9261 | } |
| 9262 | |
| 9263 | LegalizerHelper::LegalizeResult |
| 9264 | LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) { |
| 9265 | Register SrcReg = MI.getOperand(1).getReg(); |
| 9266 | Register DestReg = MI.getOperand(0).getReg(); |
| 9267 | LLT Ty = MRI.getType(SrcReg), IType = LLT::scalar(1); |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 9268 | auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0); |
| 9269 | auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0); |
Madhur Amilkanthwar | 7bb87d5 | 2024-03-21 09:54:03 +0530 | [diff] [blame] | 9270 | auto ICmp = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, IType, SrcReg, Zero); |
| 9271 | MIRBuilder.buildSelect(DestReg, ICmp, SrcReg, Sub); |
Mirko Brkusanin | 35ef4c9 | 2021-06-03 18:09:45 +0200 | [diff] [blame] | 9272 | MI.eraseFromParent(); |
| 9273 | return Legalized; |
| 9274 | } |
Jessica Paquette | 791006f | 2021-08-17 10:39:18 -0700 | [diff] [blame] | 9275 | |
Him188 | 0748f42 | 2024-09-03 12:47:26 +0100 | [diff] [blame] | 9276 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) { |
| 9277 | Register SrcReg = MI.getOperand(1).getReg(); |
| 9278 | Register DstReg = MI.getOperand(0).getReg(); |
| 9279 | |
| 9280 | LLT Ty = MRI.getType(DstReg); |
| 9281 | |
| 9282 | // Reset sign bit |
| 9283 | MIRBuilder.buildAnd( |
| 9284 | DstReg, SrcReg, |
| 9285 | MIRBuilder.buildConstant( |
| 9286 | Ty, APInt::getSignedMaxValue(Ty.getScalarSizeInBits()))); |
| 9287 | |
| 9288 | MI.eraseFromParent(); |
| 9289 | return Legalized; |
| 9290 | } |
| 9291 | |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 9292 | LegalizerHelper::LegalizeResult |
| 9293 | LegalizerHelper::lowerVectorReduction(MachineInstr &MI) { |
| 9294 | Register SrcReg = MI.getOperand(1).getReg(); |
| 9295 | LLT SrcTy = MRI.getType(SrcReg); |
| 9296 | LLT DstTy = MRI.getType(SrcReg); |
| 9297 | |
| 9298 | // The source could be a scalar if the IR type was <1 x sN>. |
| 9299 | if (SrcTy.isScalar()) { |
| 9300 | if (DstTy.getSizeInBits() > SrcTy.getSizeInBits()) |
| 9301 | return UnableToLegalize; // FIXME: handle extension. |
| 9302 | // This can be just a plain copy. |
| 9303 | Observer.changingInstr(MI); |
| 9304 | MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::COPY)); |
| 9305 | Observer.changedInstr(MI); |
| 9306 | return Legalized; |
| 9307 | } |
David Green | 2802739 | 2023-06-11 10:25:24 +0100 | [diff] [blame] | 9308 | return UnableToLegalize; |
Amara Emerson | 95ac3d1 | 2021-08-18 00:19:58 -0700 | [diff] [blame] | 9309 | } |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9310 | |
Michael Maitland | 6f9cb9a7 | 2023-12-08 13:24:27 -0500 | [diff] [blame] | 9311 | LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) { |
| 9312 | MachineFunction &MF = *MI.getMF(); |
| 9313 | const DataLayout &DL = MIRBuilder.getDataLayout(); |
| 9314 | LLVMContext &Ctx = MF.getFunction().getContext(); |
| 9315 | Register ListPtr = MI.getOperand(1).getReg(); |
| 9316 | LLT PtrTy = MRI.getType(ListPtr); |
| 9317 | |
| 9318 | // LstPtr is a pointer to the head of the list. Get the address |
| 9319 | // of the head of the list. |
| 9320 | Align PtrAlignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); |
| 9321 | MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand( |
| 9322 | MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, PtrAlignment); |
| 9323 | auto VAList = MIRBuilder.buildLoad(PtrTy, ListPtr, *PtrLoadMMO).getReg(0); |
| 9324 | |
| 9325 | const Align A(MI.getOperand(2).getImm()); |
| 9326 | LLT PtrTyAsScalarTy = LLT::scalar(PtrTy.getSizeInBits()); |
| 9327 | if (A > TLI.getMinStackArgumentAlignment()) { |
| 9328 | Register AlignAmt = |
| 9329 | MIRBuilder.buildConstant(PtrTyAsScalarTy, A.value() - 1).getReg(0); |
| 9330 | auto AddDst = MIRBuilder.buildPtrAdd(PtrTy, VAList, AlignAmt); |
| 9331 | auto AndDst = MIRBuilder.buildMaskLowPtrBits(PtrTy, AddDst, Log2(A)); |
| 9332 | VAList = AndDst.getReg(0); |
| 9333 | } |
| 9334 | |
| 9335 | // Increment the pointer, VAList, to the next vaarg |
| 9336 | // The list should be bumped by the size of element in the current head of |
| 9337 | // list. |
| 9338 | Register Dst = MI.getOperand(0).getReg(); |
| 9339 | LLT LLTTy = MRI.getType(Dst); |
| 9340 | Type *Ty = getTypeForLLT(LLTTy, Ctx); |
| 9341 | auto IncAmt = |
| 9342 | MIRBuilder.buildConstant(PtrTyAsScalarTy, DL.getTypeAllocSize(Ty)); |
| 9343 | auto Succ = MIRBuilder.buildPtrAdd(PtrTy, VAList, IncAmt); |
| 9344 | |
| 9345 | // Store the increment VAList to the legalized pointer |
| 9346 | MachineMemOperand *StoreMMO = MF.getMachineMemOperand( |
| 9347 | MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, PtrAlignment); |
| 9348 | MIRBuilder.buildStore(Succ, ListPtr, *StoreMMO); |
| 9349 | // Load the actual argument out of the pointer VAList |
| 9350 | Align EltAlignment = DL.getABITypeAlign(Ty); |
| 9351 | MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand( |
| 9352 | MachinePointerInfo(), MachineMemOperand::MOLoad, LLTTy, EltAlignment); |
| 9353 | MIRBuilder.buildLoad(Dst, VAList, *EltLoadMMO); |
| 9354 | |
| 9355 | MI.eraseFromParent(); |
| 9356 | return Legalized; |
| 9357 | } |
| 9358 | |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9359 | static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { |
| 9360 | // On Darwin, -Os means optimize for size without hurting performance, so |
| 9361 | // only really optimize for size when -Oz (MinSize) is used. |
| 9362 | if (MF.getTarget().getTargetTriple().isOSDarwin()) |
| 9363 | return MF.getFunction().hasMinSize(); |
| 9364 | return MF.getFunction().hasOptSize(); |
| 9365 | } |
| 9366 | |
| 9367 | // Returns a list of types to use for memory op lowering in MemOps. A partial |
| 9368 | // port of findOptimalMemOpLowering in TargetLowering. |
| 9369 | static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps, |
| 9370 | unsigned Limit, const MemOp &Op, |
| 9371 | unsigned DstAS, unsigned SrcAS, |
| 9372 | const AttributeList &FuncAttributes, |
| 9373 | const TargetLowering &TLI) { |
| 9374 | if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign()) |
| 9375 | return false; |
| 9376 | |
| 9377 | LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes); |
| 9378 | |
| 9379 | if (Ty == LLT()) { |
| 9380 | // Use the largest scalar type whose alignment constraints are satisfied. |
| 9381 | // We only need to check DstAlign here as SrcAlign is always greater or |
| 9382 | // equal to DstAlign (or zero). |
| 9383 | Ty = LLT::scalar(64); |
| 9384 | if (Op.isFixedDstAlign()) |
| 9385 | while (Op.getDstAlign() < Ty.getSizeInBytes() && |
| 9386 | !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign())) |
| 9387 | Ty = LLT::scalar(Ty.getSizeInBytes()); |
| 9388 | assert(Ty.getSizeInBits() > 0 && "Could not find valid type"); |
| 9389 | // FIXME: check for the largest legal type we can load/store to. |
| 9390 | } |
| 9391 | |
| 9392 | unsigned NumMemOps = 0; |
| 9393 | uint64_t Size = Op.size(); |
| 9394 | while (Size) { |
| 9395 | unsigned TySize = Ty.getSizeInBytes(); |
| 9396 | while (TySize > Size) { |
| 9397 | // For now, only use non-vector load / store's for the left-over pieces. |
| 9398 | LLT NewTy = Ty; |
| 9399 | // FIXME: check for mem op safety and legality of the types. Not all of |
| 9400 | // SDAGisms map cleanly to GISel concepts. |
| 9401 | if (NewTy.isVector()) |
| 9402 | NewTy = NewTy.getSizeInBits() > 64 ? LLT::scalar(64) : LLT::scalar(32); |
Kazu Hirata | f20b507 | 2023-01-28 09:06:31 -0800 | [diff] [blame] | 9403 | NewTy = LLT::scalar(llvm::bit_floor(NewTy.getSizeInBits() - 1)); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9404 | unsigned NewTySize = NewTy.getSizeInBytes(); |
| 9405 | assert(NewTySize > 0 && "Could not find appropriate type"); |
| 9406 | |
| 9407 | // If the new LLT cannot cover all of the remaining bits, then consider |
| 9408 | // issuing a (or a pair of) unaligned and overlapping load / store. |
Stanislav Mekhanoshin | bcaf31e | 2022-04-21 16:23:11 -0700 | [diff] [blame] | 9409 | unsigned Fast; |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9410 | // Need to get a VT equivalent for allowMisalignedMemoryAccesses(). |
| 9411 | MVT VT = getMVTForLLT(Ty); |
| 9412 | if (NumMemOps && Op.allowOverlap() && NewTySize < Size && |
| 9413 | TLI.allowsMisalignedMemoryAccesses( |
| 9414 | VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), |
| 9415 | MachineMemOperand::MONone, &Fast) && |
| 9416 | Fast) |
| 9417 | TySize = Size; |
| 9418 | else { |
| 9419 | Ty = NewTy; |
| 9420 | TySize = NewTySize; |
| 9421 | } |
| 9422 | } |
| 9423 | |
| 9424 | if (++NumMemOps > Limit) |
| 9425 | return false; |
| 9426 | |
| 9427 | MemOps.push_back(Ty); |
| 9428 | Size -= TySize; |
| 9429 | } |
| 9430 | |
| 9431 | return true; |
| 9432 | } |
| 9433 | |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9434 | // Get a vectorized representation of the memset value operand, GISel edition. |
| 9435 | static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { |
| 9436 | MachineRegisterInfo &MRI = *MIB.getMRI(); |
| 9437 | unsigned NumBits = Ty.getScalarSizeInBits(); |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 9438 | auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9439 | if (!Ty.isVector() && ValVRegAndVal) { |
Jay Foad | 6bec3e9 | 2021-10-06 10:54:07 +0100 | [diff] [blame] | 9440 | APInt Scalar = ValVRegAndVal->Value.trunc(8); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9441 | APInt SplatVal = APInt::getSplat(NumBits, Scalar); |
| 9442 | return MIB.buildConstant(Ty, SplatVal).getReg(0); |
| 9443 | } |
| 9444 | |
| 9445 | // Extend the byte value to the larger type, and then multiply by a magic |
| 9446 | // value 0x010101... in order to replicate it across every byte. |
| 9447 | // Unless it's zero, in which case just emit a larger G_CONSTANT 0. |
| 9448 | if (ValVRegAndVal && ValVRegAndVal->Value == 0) { |
| 9449 | return MIB.buildConstant(Ty, 0).getReg(0); |
| 9450 | } |
| 9451 | |
| 9452 | LLT ExtType = Ty.getScalarType(); |
| 9453 | auto ZExt = MIB.buildZExtOrTrunc(ExtType, Val); |
| 9454 | if (NumBits > 8) { |
| 9455 | APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); |
| 9456 | auto MagicMI = MIB.buildConstant(ExtType, Magic); |
| 9457 | Val = MIB.buildMul(ExtType, ZExt, MagicMI).getReg(0); |
| 9458 | } |
| 9459 | |
| 9460 | // For vector types create a G_BUILD_VECTOR. |
| 9461 | if (Ty.isVector()) |
Michael Maitland | 96049fc | 2024-03-07 09:50:29 -0500 | [diff] [blame] | 9462 | Val = MIB.buildSplatBuildVector(Ty, Val).getReg(0); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9463 | |
| 9464 | return Val; |
| 9465 | } |
| 9466 | |
| 9467 | LegalizerHelper::LegalizeResult |
| 9468 | LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val, |
| 9469 | uint64_t KnownLen, Align Alignment, |
| 9470 | bool IsVolatile) { |
| 9471 | auto &MF = *MI.getParent()->getParent(); |
| 9472 | const auto &TLI = *MF.getSubtarget().getTargetLowering(); |
| 9473 | auto &DL = MF.getDataLayout(); |
| 9474 | LLVMContext &C = MF.getFunction().getContext(); |
| 9475 | |
| 9476 | assert(KnownLen != 0 && "Have a zero length memset length!"); |
| 9477 | |
| 9478 | bool DstAlignCanChange = false; |
| 9479 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 9480 | bool OptSize = shouldLowerMemFuncForSize(MF); |
| 9481 | |
| 9482 | MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); |
| 9483 | if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) |
| 9484 | DstAlignCanChange = true; |
| 9485 | |
| 9486 | unsigned Limit = TLI.getMaxStoresPerMemset(OptSize); |
| 9487 | std::vector<LLT> MemOps; |
| 9488 | |
| 9489 | const auto &DstMMO = **MI.memoperands_begin(); |
| 9490 | MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); |
| 9491 | |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 9492 | auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9493 | bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0; |
| 9494 | |
| 9495 | if (!findGISelOptimalMemOpLowering(MemOps, Limit, |
| 9496 | MemOp::Set(KnownLen, DstAlignCanChange, |
| 9497 | Alignment, |
| 9498 | /*IsZeroMemset=*/IsZeroVal, |
| 9499 | /*IsVolatile=*/IsVolatile), |
| 9500 | DstPtrInfo.getAddrSpace(), ~0u, |
| 9501 | MF.getFunction().getAttributes(), TLI)) |
| 9502 | return UnableToLegalize; |
| 9503 | |
| 9504 | if (DstAlignCanChange) { |
| 9505 | // Get an estimate of the type from the LLT. |
| 9506 | Type *IRTy = getTypeForLLT(MemOps[0], C); |
| 9507 | Align NewAlign = DL.getABITypeAlign(IRTy); |
| 9508 | if (NewAlign > Alignment) { |
| 9509 | Alignment = NewAlign; |
| 9510 | unsigned FI = FIDef->getOperand(1).getIndex(); |
| 9511 | // Give the stack frame object a larger alignment if needed. |
| 9512 | if (MFI.getObjectAlign(FI) < Alignment) |
| 9513 | MFI.setObjectAlignment(FI, Alignment); |
| 9514 | } |
| 9515 | } |
| 9516 | |
| 9517 | MachineIRBuilder MIB(MI); |
| 9518 | // Find the largest store and generate the bit pattern for it. |
| 9519 | LLT LargestTy = MemOps[0]; |
| 9520 | for (unsigned i = 1; i < MemOps.size(); i++) |
| 9521 | if (MemOps[i].getSizeInBits() > LargestTy.getSizeInBits()) |
| 9522 | LargestTy = MemOps[i]; |
| 9523 | |
| 9524 | // The memset stored value is always defined as an s8, so in order to make it |
| 9525 | // work with larger store types we need to repeat the bit pattern across the |
| 9526 | // wider type. |
| 9527 | Register MemSetValue = getMemsetValue(Val, LargestTy, MIB); |
| 9528 | |
| 9529 | if (!MemSetValue) |
| 9530 | return UnableToLegalize; |
| 9531 | |
| 9532 | // Generate the stores. For each store type in the list, we generate the |
| 9533 | // matching store of that type to the destination address. |
| 9534 | LLT PtrTy = MRI.getType(Dst); |
| 9535 | unsigned DstOff = 0; |
| 9536 | unsigned Size = KnownLen; |
| 9537 | for (unsigned I = 0; I < MemOps.size(); I++) { |
| 9538 | LLT Ty = MemOps[I]; |
| 9539 | unsigned TySize = Ty.getSizeInBytes(); |
| 9540 | if (TySize > Size) { |
| 9541 | // Issuing an unaligned load / store pair that overlaps with the previous |
| 9542 | // pair. Adjust the offset accordingly. |
| 9543 | assert(I == MemOps.size() - 1 && I != 0); |
| 9544 | DstOff -= TySize - Size; |
| 9545 | } |
| 9546 | |
| 9547 | // If this store is smaller than the largest store see whether we can get |
| 9548 | // the smaller value for free with a truncate. |
| 9549 | Register Value = MemSetValue; |
| 9550 | if (Ty.getSizeInBits() < LargestTy.getSizeInBits()) { |
| 9551 | MVT VT = getMVTForLLT(Ty); |
| 9552 | MVT LargestVT = getMVTForLLT(LargestTy); |
| 9553 | if (!LargestTy.isVector() && !Ty.isVector() && |
| 9554 | TLI.isTruncateFree(LargestVT, VT)) |
| 9555 | Value = MIB.buildTrunc(Ty, MemSetValue).getReg(0); |
| 9556 | else |
| 9557 | Value = getMemsetValue(Val, Ty, MIB); |
| 9558 | if (!Value) |
| 9559 | return UnableToLegalize; |
| 9560 | } |
| 9561 | |
| 9562 | auto *StoreMMO = MF.getMachineMemOperand(&DstMMO, DstOff, Ty); |
| 9563 | |
| 9564 | Register Ptr = Dst; |
| 9565 | if (DstOff != 0) { |
| 9566 | auto Offset = |
| 9567 | MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), DstOff); |
| 9568 | Ptr = MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0); |
| 9569 | } |
| 9570 | |
| 9571 | MIB.buildStore(Value, Ptr, *StoreMMO); |
| 9572 | DstOff += Ty.getSizeInBytes(); |
| 9573 | Size -= TySize; |
| 9574 | } |
| 9575 | |
| 9576 | MI.eraseFromParent(); |
| 9577 | return Legalized; |
| 9578 | } |
| 9579 | |
| 9580 | LegalizerHelper::LegalizeResult |
| 9581 | LegalizerHelper::lowerMemcpyInline(MachineInstr &MI) { |
| 9582 | assert(MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE); |
| 9583 | |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 9584 | auto [Dst, Src, Len] = MI.getFirst3Regs(); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9585 | |
| 9586 | const auto *MMOIt = MI.memoperands_begin(); |
| 9587 | const MachineMemOperand *MemOp = *MMOIt; |
| 9588 | bool IsVolatile = MemOp->isVolatile(); |
| 9589 | |
| 9590 | // See if this is a constant length copy |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 9591 | auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9592 | // FIXME: support dynamically sized G_MEMCPY_INLINE |
Kazu Hirata | 5413bf1 | 2022-06-20 11:33:56 -0700 | [diff] [blame] | 9593 | assert(LenVRegAndVal && |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9594 | "inline memcpy with dynamic size is not yet supported"); |
| 9595 | uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue(); |
| 9596 | if (KnownLen == 0) { |
| 9597 | MI.eraseFromParent(); |
| 9598 | return Legalized; |
| 9599 | } |
| 9600 | |
| 9601 | const auto &DstMMO = **MI.memoperands_begin(); |
| 9602 | const auto &SrcMMO = **std::next(MI.memoperands_begin()); |
| 9603 | Align DstAlign = DstMMO.getBaseAlign(); |
| 9604 | Align SrcAlign = SrcMMO.getBaseAlign(); |
| 9605 | |
| 9606 | return lowerMemcpyInline(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, |
| 9607 | IsVolatile); |
| 9608 | } |
| 9609 | |
| 9610 | LegalizerHelper::LegalizeResult |
| 9611 | LegalizerHelper::lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src, |
| 9612 | uint64_t KnownLen, Align DstAlign, |
| 9613 | Align SrcAlign, bool IsVolatile) { |
| 9614 | assert(MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE); |
| 9615 | return lowerMemcpy(MI, Dst, Src, KnownLen, |
| 9616 | std::numeric_limits<uint64_t>::max(), DstAlign, SrcAlign, |
| 9617 | IsVolatile); |
| 9618 | } |
| 9619 | |
| 9620 | LegalizerHelper::LegalizeResult |
| 9621 | LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, |
| 9622 | uint64_t KnownLen, uint64_t Limit, Align DstAlign, |
| 9623 | Align SrcAlign, bool IsVolatile) { |
| 9624 | auto &MF = *MI.getParent()->getParent(); |
| 9625 | const auto &TLI = *MF.getSubtarget().getTargetLowering(); |
| 9626 | auto &DL = MF.getDataLayout(); |
| 9627 | LLVMContext &C = MF.getFunction().getContext(); |
| 9628 | |
| 9629 | assert(KnownLen != 0 && "Have a zero length memcpy length!"); |
| 9630 | |
| 9631 | bool DstAlignCanChange = false; |
| 9632 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
Guillaume Chatelet | 3c126d5 | 2022-06-22 15:02:48 +0000 | [diff] [blame] | 9633 | Align Alignment = std::min(DstAlign, SrcAlign); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9634 | |
| 9635 | MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); |
| 9636 | if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) |
| 9637 | DstAlignCanChange = true; |
| 9638 | |
| 9639 | // FIXME: infer better src pointer alignment like SelectionDAG does here. |
| 9640 | // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining |
| 9641 | // if the memcpy is in a tail call position. |
| 9642 | |
| 9643 | std::vector<LLT> MemOps; |
| 9644 | |
| 9645 | const auto &DstMMO = **MI.memoperands_begin(); |
| 9646 | const auto &SrcMMO = **std::next(MI.memoperands_begin()); |
| 9647 | MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); |
| 9648 | MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); |
| 9649 | |
| 9650 | if (!findGISelOptimalMemOpLowering( |
| 9651 | MemOps, Limit, |
| 9652 | MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign, |
| 9653 | IsVolatile), |
| 9654 | DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), |
| 9655 | MF.getFunction().getAttributes(), TLI)) |
| 9656 | return UnableToLegalize; |
| 9657 | |
| 9658 | if (DstAlignCanChange) { |
| 9659 | // Get an estimate of the type from the LLT. |
| 9660 | Type *IRTy = getTypeForLLT(MemOps[0], C); |
| 9661 | Align NewAlign = DL.getABITypeAlign(IRTy); |
| 9662 | |
| 9663 | // Don't promote to an alignment that would require dynamic stack |
| 9664 | // realignment. |
| 9665 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
| 9666 | if (!TRI->hasStackRealignment(MF)) |
Sergei Barannikov | 4d7a0ab | 2024-08-27 22:59:33 +0300 | [diff] [blame] | 9667 | if (MaybeAlign StackAlign = DL.getStackAlignment()) |
| 9668 | NewAlign = std::min(NewAlign, *StackAlign); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9669 | |
| 9670 | if (NewAlign > Alignment) { |
| 9671 | Alignment = NewAlign; |
| 9672 | unsigned FI = FIDef->getOperand(1).getIndex(); |
| 9673 | // Give the stack frame object a larger alignment if needed. |
| 9674 | if (MFI.getObjectAlign(FI) < Alignment) |
| 9675 | MFI.setObjectAlignment(FI, Alignment); |
| 9676 | } |
| 9677 | } |
| 9678 | |
| 9679 | LLVM_DEBUG(dbgs() << "Inlining memcpy: " << MI << " into loads & stores\n"); |
| 9680 | |
| 9681 | MachineIRBuilder MIB(MI); |
| 9682 | // Now we need to emit a pair of load and stores for each of the types we've |
| 9683 | // collected. I.e. for each type, generate a load from the source pointer of |
| 9684 | // that type width, and then generate a corresponding store to the dest buffer |
| 9685 | // of that value loaded. This can result in a sequence of loads and stores |
| 9686 | // mixed types, depending on what the target specifies as good types to use. |
| 9687 | unsigned CurrOffset = 0; |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9688 | unsigned Size = KnownLen; |
| 9689 | for (auto CopyTy : MemOps) { |
| 9690 | // Issuing an unaligned load / store pair that overlaps with the previous |
| 9691 | // pair. Adjust the offset accordingly. |
| 9692 | if (CopyTy.getSizeInBytes() > Size) |
| 9693 | CurrOffset -= CopyTy.getSizeInBytes() - Size; |
| 9694 | |
| 9695 | // Construct MMOs for the accesses. |
| 9696 | auto *LoadMMO = |
| 9697 | MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes()); |
| 9698 | auto *StoreMMO = |
| 9699 | MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.getSizeInBytes()); |
| 9700 | |
| 9701 | // Create the load. |
| 9702 | Register LoadPtr = Src; |
| 9703 | Register Offset; |
| 9704 | if (CurrOffset != 0) { |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9705 | LLT SrcTy = MRI.getType(Src); |
| 9706 | Offset = MIB.buildConstant(LLT::scalar(SrcTy.getSizeInBits()), CurrOffset) |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9707 | .getReg(0); |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9708 | LoadPtr = MIB.buildPtrAdd(SrcTy, Src, Offset).getReg(0); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9709 | } |
| 9710 | auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO); |
| 9711 | |
| 9712 | // Create the store. |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9713 | Register StorePtr = Dst; |
| 9714 | if (CurrOffset != 0) { |
| 9715 | LLT DstTy = MRI.getType(Dst); |
| 9716 | StorePtr = MIB.buildPtrAdd(DstTy, Dst, Offset).getReg(0); |
| 9717 | } |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9718 | MIB.buildStore(LdVal, StorePtr, *StoreMMO); |
| 9719 | CurrOffset += CopyTy.getSizeInBytes(); |
| 9720 | Size -= CopyTy.getSizeInBytes(); |
| 9721 | } |
| 9722 | |
| 9723 | MI.eraseFromParent(); |
| 9724 | return Legalized; |
| 9725 | } |
| 9726 | |
| 9727 | LegalizerHelper::LegalizeResult |
| 9728 | LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, |
| 9729 | uint64_t KnownLen, Align DstAlign, Align SrcAlign, |
| 9730 | bool IsVolatile) { |
| 9731 | auto &MF = *MI.getParent()->getParent(); |
| 9732 | const auto &TLI = *MF.getSubtarget().getTargetLowering(); |
| 9733 | auto &DL = MF.getDataLayout(); |
| 9734 | LLVMContext &C = MF.getFunction().getContext(); |
| 9735 | |
| 9736 | assert(KnownLen != 0 && "Have a zero length memmove length!"); |
| 9737 | |
| 9738 | bool DstAlignCanChange = false; |
| 9739 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
| 9740 | bool OptSize = shouldLowerMemFuncForSize(MF); |
Guillaume Chatelet | 3c126d5 | 2022-06-22 15:02:48 +0000 | [diff] [blame] | 9741 | Align Alignment = std::min(DstAlign, SrcAlign); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9742 | |
| 9743 | MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); |
| 9744 | if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) |
| 9745 | DstAlignCanChange = true; |
| 9746 | |
| 9747 | unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize); |
| 9748 | std::vector<LLT> MemOps; |
| 9749 | |
| 9750 | const auto &DstMMO = **MI.memoperands_begin(); |
| 9751 | const auto &SrcMMO = **std::next(MI.memoperands_begin()); |
| 9752 | MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); |
| 9753 | MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); |
| 9754 | |
| 9755 | // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently due |
| 9756 | // to a bug in it's findOptimalMemOpLowering implementation. For now do the |
| 9757 | // same thing here. |
| 9758 | if (!findGISelOptimalMemOpLowering( |
| 9759 | MemOps, Limit, |
| 9760 | MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign, |
| 9761 | /*IsVolatile*/ true), |
| 9762 | DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), |
| 9763 | MF.getFunction().getAttributes(), TLI)) |
| 9764 | return UnableToLegalize; |
| 9765 | |
| 9766 | if (DstAlignCanChange) { |
| 9767 | // Get an estimate of the type from the LLT. |
| 9768 | Type *IRTy = getTypeForLLT(MemOps[0], C); |
| 9769 | Align NewAlign = DL.getABITypeAlign(IRTy); |
| 9770 | |
| 9771 | // Don't promote to an alignment that would require dynamic stack |
| 9772 | // realignment. |
| 9773 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
| 9774 | if (!TRI->hasStackRealignment(MF)) |
Sergei Barannikov | 4d7a0ab | 2024-08-27 22:59:33 +0300 | [diff] [blame] | 9775 | if (MaybeAlign StackAlign = DL.getStackAlignment()) |
| 9776 | NewAlign = std::min(NewAlign, *StackAlign); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9777 | |
| 9778 | if (NewAlign > Alignment) { |
| 9779 | Alignment = NewAlign; |
| 9780 | unsigned FI = FIDef->getOperand(1).getIndex(); |
| 9781 | // Give the stack frame object a larger alignment if needed. |
| 9782 | if (MFI.getObjectAlign(FI) < Alignment) |
| 9783 | MFI.setObjectAlignment(FI, Alignment); |
| 9784 | } |
| 9785 | } |
| 9786 | |
| 9787 | LLVM_DEBUG(dbgs() << "Inlining memmove: " << MI << " into loads & stores\n"); |
| 9788 | |
| 9789 | MachineIRBuilder MIB(MI); |
| 9790 | // Memmove requires that we perform the loads first before issuing the stores. |
| 9791 | // Apart from that, this loop is pretty much doing the same thing as the |
| 9792 | // memcpy codegen function. |
| 9793 | unsigned CurrOffset = 0; |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9794 | SmallVector<Register, 16> LoadVals; |
| 9795 | for (auto CopyTy : MemOps) { |
| 9796 | // Construct MMO for the load. |
| 9797 | auto *LoadMMO = |
| 9798 | MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes()); |
| 9799 | |
| 9800 | // Create the load. |
| 9801 | Register LoadPtr = Src; |
| 9802 | if (CurrOffset != 0) { |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9803 | LLT SrcTy = MRI.getType(Src); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9804 | auto Offset = |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9805 | MIB.buildConstant(LLT::scalar(SrcTy.getSizeInBits()), CurrOffset); |
| 9806 | LoadPtr = MIB.buildPtrAdd(SrcTy, Src, Offset).getReg(0); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9807 | } |
| 9808 | LoadVals.push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0)); |
| 9809 | CurrOffset += CopyTy.getSizeInBytes(); |
| 9810 | } |
| 9811 | |
| 9812 | CurrOffset = 0; |
| 9813 | for (unsigned I = 0; I < MemOps.size(); ++I) { |
| 9814 | LLT CopyTy = MemOps[I]; |
| 9815 | // Now store the values loaded. |
| 9816 | auto *StoreMMO = |
| 9817 | MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.getSizeInBytes()); |
| 9818 | |
| 9819 | Register StorePtr = Dst; |
| 9820 | if (CurrOffset != 0) { |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9821 | LLT DstTy = MRI.getType(Dst); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9822 | auto Offset = |
Jameson Nash | 0332d10 | 2021-10-21 11:58:02 -0400 | [diff] [blame] | 9823 | MIB.buildConstant(LLT::scalar(DstTy.getSizeInBits()), CurrOffset); |
| 9824 | StorePtr = MIB.buildPtrAdd(DstTy, Dst, Offset).getReg(0); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9825 | } |
| 9826 | MIB.buildStore(LoadVals[I], StorePtr, *StoreMMO); |
| 9827 | CurrOffset += CopyTy.getSizeInBytes(); |
| 9828 | } |
| 9829 | MI.eraseFromParent(); |
| 9830 | return Legalized; |
| 9831 | } |
| 9832 | |
| 9833 | LegalizerHelper::LegalizeResult |
| 9834 | LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { |
| 9835 | const unsigned Opc = MI.getOpcode(); |
| 9836 | // This combine is fairly complex so it's not written with a separate |
| 9837 | // matcher function. |
| 9838 | assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMMOVE || |
| 9839 | Opc == TargetOpcode::G_MEMSET) && |
| 9840 | "Expected memcpy like instruction"); |
| 9841 | |
| 9842 | auto MMOIt = MI.memoperands_begin(); |
| 9843 | const MachineMemOperand *MemOp = *MMOIt; |
| 9844 | |
| 9845 | Align DstAlign = MemOp->getBaseAlign(); |
| 9846 | Align SrcAlign; |
Amara Emerson | 719024a | 2023-02-23 16:35:39 -0800 | [diff] [blame] | 9847 | auto [Dst, Src, Len] = MI.getFirst3Regs(); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9848 | |
| 9849 | if (Opc != TargetOpcode::G_MEMSET) { |
| 9850 | assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI"); |
| 9851 | MemOp = *(++MMOIt); |
| 9852 | SrcAlign = MemOp->getBaseAlign(); |
| 9853 | } |
| 9854 | |
| 9855 | // See if this is a constant length copy |
Petar Avramovic | d477a7c | 2021-09-17 11:21:55 +0200 | [diff] [blame] | 9856 | auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI); |
Mirko Brkusanin | 36527cb | 2021-09-07 11:30:11 +0200 | [diff] [blame] | 9857 | if (!LenVRegAndVal) |
| 9858 | return UnableToLegalize; |
| 9859 | uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue(); |
| 9860 | |
| 9861 | if (KnownLen == 0) { |
| 9862 | MI.eraseFromParent(); |
| 9863 | return Legalized; |
| 9864 | } |
| 9865 | |
| 9866 | bool IsVolatile = MemOp->isVolatile(); |
| 9867 | if (Opc == TargetOpcode::G_MEMCPY_INLINE) |
| 9868 | return lowerMemcpyInline(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, |
| 9869 | IsVolatile); |
| 9870 | |
| 9871 | // Don't try to optimize volatile. |
| 9872 | if (IsVolatile) |
| 9873 | return UnableToLegalize; |
| 9874 | |
| 9875 | if (MaxLen && KnownLen > MaxLen) |
| 9876 | return UnableToLegalize; |
| 9877 | |
| 9878 | if (Opc == TargetOpcode::G_MEMCPY) { |
| 9879 | auto &MF = *MI.getParent()->getParent(); |
| 9880 | const auto &TLI = *MF.getSubtarget().getTargetLowering(); |
| 9881 | bool OptSize = shouldLowerMemFuncForSize(MF); |
| 9882 | uint64_t Limit = TLI.getMaxStoresPerMemcpy(OptSize); |
| 9883 | return lowerMemcpy(MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign, |
| 9884 | IsVolatile); |
| 9885 | } |
| 9886 | if (Opc == TargetOpcode::G_MEMMOVE) |
| 9887 | return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile); |
| 9888 | if (Opc == TargetOpcode::G_MEMSET) |
| 9889 | return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile); |
| 9890 | return UnableToLegalize; |
| 9891 | } |