blob: 79297d35b0c6364abc26989e8f66c2f859d48635 [file] [log] [blame]
Tim Northover69fa84a2016-10-14 22:18:18 +00001//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
Tim Northover33b07d62016-07-22 20:03:43 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tim Northover33b07d62016-07-22 20:03:43 +00006//
7//===----------------------------------------------------------------------===//
8//
Tim Northover69fa84a2016-10-14 22:18:18 +00009/// \file This file implements the LegalizerHelper class to legalize
Tim Northover33b07d62016-07-22 20:03:43 +000010/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
Tim Northover69fa84a2016-10-14 22:18:18 +000015#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
Tim Northoveredb3c8c2016-08-29 19:07:16 +000016#include "llvm/CodeGen/GlobalISel/CallLowering.h"
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000017#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
Tim Northover69fa84a2016-10-14 22:18:18 +000018#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000019#include "llvm/CodeGen/MachineRegisterInfo.h"
Amara Emersone20b91c2019-08-27 19:54:27 +000020#include "llvm/CodeGen/TargetFrameLowering.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000021#include "llvm/CodeGen/TargetInstrInfo.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000022#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetSubtargetInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000024#include "llvm/Support/Debug.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000025#include "llvm/Support/MathExtras.h"
Tim Northover33b07d62016-07-22 20:03:43 +000026#include "llvm/Support/raw_ostream.h"
Tim Northover33b07d62016-07-22 20:03:43 +000027
Daniel Sanders5377fb32017-04-20 15:46:12 +000028#define DEBUG_TYPE "legalizer"
Tim Northover33b07d62016-07-22 20:03:43 +000029
30using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000031using namespace LegalizeActions;
Tim Northover33b07d62016-07-22 20:03:43 +000032
Matt Arsenaultc83b8232019-02-07 17:38:00 +000033/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
34///
35/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
36/// with any leftover piece as type \p LeftoverTy
37///
Matt Arsenaultd3093c22019-02-28 00:16:32 +000038/// Returns -1 in the first element of the pair if the breakdown is not
39/// satisfiable.
40static std::pair<int, int>
41getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
Matt Arsenaultc83b8232019-02-07 17:38:00 +000042 assert(!LeftoverTy.isValid() && "this is an out argument");
43
44 unsigned Size = OrigTy.getSizeInBits();
45 unsigned NarrowSize = NarrowTy.getSizeInBits();
46 unsigned NumParts = Size / NarrowSize;
47 unsigned LeftoverSize = Size - NumParts * NarrowSize;
48 assert(Size > NarrowSize);
49
50 if (LeftoverSize == 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000051 return {NumParts, 0};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000052
53 if (NarrowTy.isVector()) {
54 unsigned EltSize = OrigTy.getScalarSizeInBits();
55 if (LeftoverSize % EltSize != 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000056 return {-1, -1};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000057 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
58 } else {
59 LeftoverTy = LLT::scalar(LeftoverSize);
60 }
61
Matt Arsenaultd3093c22019-02-28 00:16:32 +000062 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
63 return std::make_pair(NumParts, NumLeftover);
Matt Arsenaultc83b8232019-02-07 17:38:00 +000064}
65
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000066LegalizerHelper::LegalizerHelper(MachineFunction &MF,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +000067 GISelChangeObserver &Observer,
68 MachineIRBuilder &Builder)
69 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
70 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
Tim Northover33b07d62016-07-22 20:03:43 +000071 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000072 MIRBuilder.setChangeObserver(Observer);
Tim Northover33b07d62016-07-22 20:03:43 +000073}
74
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000075LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +000076 GISelChangeObserver &Observer,
77 MachineIRBuilder &B)
78 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000079 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000080 MIRBuilder.setChangeObserver(Observer);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000081}
Tim Northover69fa84a2016-10-14 22:18:18 +000082LegalizerHelper::LegalizeResult
Volkan Keles685fbda2017-03-10 18:34:57 +000083LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +000084 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
Daniel Sanders5377fb32017-04-20 15:46:12 +000085
Aditya Nandakumar1023a2e2019-07-01 17:53:50 +000086 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
87 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
88 return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized
89 : UnableToLegalize;
Daniel Sanders262ed0e2018-01-24 17:17:46 +000090 auto Step = LI.getAction(MI, MRI);
91 switch (Step.Action) {
Daniel Sanders9ade5592018-01-29 17:37:29 +000092 case Legal:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000093 LLVM_DEBUG(dbgs() << ".. Already legal\n");
Tim Northover33b07d62016-07-22 20:03:43 +000094 return AlreadyLegal;
Daniel Sanders9ade5592018-01-29 17:37:29 +000095 case Libcall:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000096 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
Tim Northoveredb3c8c2016-08-29 19:07:16 +000097 return libcall(MI);
Daniel Sanders9ade5592018-01-29 17:37:29 +000098 case NarrowScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000099 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000100 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000101 case WidenScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000102 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000103 return widenScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000104 case Lower:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000105 LLVM_DEBUG(dbgs() << ".. Lower\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000106 return lower(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000107 case FewerElements:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000108 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000109 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
Matt Arsenault18ec3822019-02-11 22:00:39 +0000110 case MoreElements:
111 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
112 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000113 case Custom:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000114 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000115 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
116 : UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +0000117 default:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000118 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
Tim Northover33b07d62016-07-22 20:03:43 +0000119 return UnableToLegalize;
120 }
121}
122
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000123void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
124 SmallVectorImpl<Register> &VRegs) {
Tim Northoverbf017292017-03-03 22:46:09 +0000125 for (int i = 0; i < NumParts; ++i)
Tim Northover0f140c72016-09-09 11:46:34 +0000126 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
Tim Northoverbf017292017-03-03 22:46:09 +0000127 MIRBuilder.buildUnmerge(VRegs, Reg);
Tim Northover33b07d62016-07-22 20:03:43 +0000128}
129
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000130bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000131 LLT MainTy, LLT &LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000132 SmallVectorImpl<Register> &VRegs,
133 SmallVectorImpl<Register> &LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000134 assert(!LeftoverTy.isValid() && "this is an out argument");
135
136 unsigned RegSize = RegTy.getSizeInBits();
137 unsigned MainSize = MainTy.getSizeInBits();
138 unsigned NumParts = RegSize / MainSize;
139 unsigned LeftoverSize = RegSize - NumParts * MainSize;
140
141 // Use an unmerge when possible.
142 if (LeftoverSize == 0) {
143 for (unsigned I = 0; I < NumParts; ++I)
144 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
145 MIRBuilder.buildUnmerge(VRegs, Reg);
146 return true;
147 }
148
149 if (MainTy.isVector()) {
150 unsigned EltSize = MainTy.getScalarSizeInBits();
151 if (LeftoverSize % EltSize != 0)
152 return false;
153 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
154 } else {
155 LeftoverTy = LLT::scalar(LeftoverSize);
156 }
157
158 // For irregular sizes, extract the individual parts.
159 for (unsigned I = 0; I != NumParts; ++I) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000160 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000161 VRegs.push_back(NewReg);
162 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
163 }
164
165 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
166 Offset += LeftoverSize) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000167 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000168 LeftoverRegs.push_back(NewReg);
169 MIRBuilder.buildExtract(NewReg, Reg, Offset);
170 }
171
172 return true;
173}
174
Matt Arsenault28215ca2019-08-13 16:26:28 +0000175static LLT getGCDType(LLT OrigTy, LLT TargetTy) {
176 if (OrigTy.isVector() && TargetTy.isVector()) {
177 assert(OrigTy.getElementType() == TargetTy.getElementType());
178 int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
179 TargetTy.getNumElements());
180 return LLT::scalarOrVector(GCD, OrigTy.getElementType());
181 }
182
183 if (OrigTy.isVector() && !TargetTy.isVector()) {
184 assert(OrigTy.getElementType() == TargetTy);
185 return TargetTy;
186 }
187
188 assert(!OrigTy.isVector() && !TargetTy.isVector());
189
190 int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(),
191 TargetTy.getSizeInBits());
192 return LLT::scalar(GCD);
193}
194
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000195void LegalizerHelper::insertParts(Register DstReg,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000196 LLT ResultTy, LLT PartTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000197 ArrayRef<Register> PartRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000198 LLT LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000199 ArrayRef<Register> LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000200 if (!LeftoverTy.isValid()) {
201 assert(LeftoverRegs.empty());
202
Matt Arsenault81511e52019-02-05 00:13:44 +0000203 if (!ResultTy.isVector()) {
204 MIRBuilder.buildMerge(DstReg, PartRegs);
205 return;
206 }
207
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000208 if (PartTy.isVector())
209 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
210 else
211 MIRBuilder.buildBuildVector(DstReg, PartRegs);
212 return;
213 }
214
215 unsigned PartSize = PartTy.getSizeInBits();
216 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
217
Matt Arsenault3018d182019-06-28 01:47:44 +0000218 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000219 MIRBuilder.buildUndef(CurResultReg);
220
221 unsigned Offset = 0;
Matt Arsenault3018d182019-06-28 01:47:44 +0000222 for (Register PartReg : PartRegs) {
223 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000224 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
225 CurResultReg = NewResultReg;
226 Offset += PartSize;
227 }
228
229 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
230 // Use the original output register for the final insert to avoid a copy.
Matt Arsenault3018d182019-06-28 01:47:44 +0000231 Register NewResultReg = (I + 1 == E) ?
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000232 DstReg : MRI.createGenericVirtualRegister(ResultTy);
233
234 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
235 CurResultReg = NewResultReg;
236 Offset += LeftoverPartSize;
237 }
238}
239
Tim Northovere0418412017-02-08 23:23:39 +0000240static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
241 switch (Opcode) {
Diana Picuse97822e2017-04-24 07:22:31 +0000242 case TargetOpcode::G_SDIV:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000243 assert((Size == 32 || Size == 64) && "Unsupported size");
244 return Size == 64 ? RTLIB::SDIV_I64 : RTLIB::SDIV_I32;
Diana Picuse97822e2017-04-24 07:22:31 +0000245 case TargetOpcode::G_UDIV:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000246 assert((Size == 32 || Size == 64) && "Unsupported size");
247 return Size == 64 ? RTLIB::UDIV_I64 : RTLIB::UDIV_I32;
Diana Picus02e11012017-06-15 10:53:31 +0000248 case TargetOpcode::G_SREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000249 assert((Size == 32 || Size == 64) && "Unsupported size");
250 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
Diana Picus02e11012017-06-15 10:53:31 +0000251 case TargetOpcode::G_UREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000252 assert((Size == 32 || Size == 64) && "Unsupported size");
253 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
Diana Picus0528e2c2018-11-26 11:07:02 +0000254 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
255 assert(Size == 32 && "Unsupported size");
256 return RTLIB::CTLZ_I32;
Diana Picus1314a282017-04-11 10:52:34 +0000257 case TargetOpcode::G_FADD:
258 assert((Size == 32 || Size == 64) && "Unsupported size");
259 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
Javed Absar5cde1cc2017-10-30 13:51:56 +0000260 case TargetOpcode::G_FSUB:
261 assert((Size == 32 || Size == 64) && "Unsupported size");
262 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
Diana Picus9faa09b2017-11-23 12:44:20 +0000263 case TargetOpcode::G_FMUL:
264 assert((Size == 32 || Size == 64) && "Unsupported size");
265 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
Diana Picusc01f7f12017-11-23 13:26:07 +0000266 case TargetOpcode::G_FDIV:
267 assert((Size == 32 || Size == 64) && "Unsupported size");
268 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
Jessica Paquette84bedac2019-01-30 23:46:15 +0000269 case TargetOpcode::G_FEXP:
270 assert((Size == 32 || Size == 64) && "Unsupported size");
271 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
Jessica Paquettee7941212019-04-03 16:58:32 +0000272 case TargetOpcode::G_FEXP2:
273 assert((Size == 32 || Size == 64) && "Unsupported size");
274 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000275 case TargetOpcode::G_FREM:
276 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
277 case TargetOpcode::G_FPOW:
278 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
Diana Picuse74243d2018-01-12 11:30:45 +0000279 case TargetOpcode::G_FMA:
280 assert((Size == 32 || Size == 64) && "Unsupported size");
281 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
Jessica Paquette7db82d72019-01-28 18:34:18 +0000282 case TargetOpcode::G_FSIN:
283 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
284 return Size == 128 ? RTLIB::SIN_F128
285 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
286 case TargetOpcode::G_FCOS:
287 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
288 return Size == 128 ? RTLIB::COS_F128
289 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
Jessica Paquettec49428a2019-01-28 19:53:14 +0000290 case TargetOpcode::G_FLOG10:
291 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
292 return Size == 128 ? RTLIB::LOG10_F128
293 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000294 case TargetOpcode::G_FLOG:
295 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
296 return Size == 128 ? RTLIB::LOG_F128
297 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
Jessica Paquette0154bd12019-01-30 21:16:04 +0000298 case TargetOpcode::G_FLOG2:
299 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
300 return Size == 128 ? RTLIB::LOG2_F128
301 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000302 case TargetOpcode::G_FCEIL:
303 assert((Size == 32 || Size == 64) && "Unsupported size");
304 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
305 case TargetOpcode::G_FFLOOR:
306 assert((Size == 32 || Size == 64) && "Unsupported size");
307 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000308 }
309 llvm_unreachable("Unknown libcall function");
310}
311
Diana Picusfc1675e2017-07-05 12:57:24 +0000312LegalizerHelper::LegalizeResult
313llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
314 const CallLowering::ArgInfo &Result,
315 ArrayRef<CallLowering::ArgInfo> Args) {
Diana Picuse97822e2017-04-24 07:22:31 +0000316 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
317 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
Diana Picuse97822e2017-04-24 07:22:31 +0000318 const char *Name = TLI.getLibcallName(Libcall);
Diana Picusd0104ea2017-07-06 09:09:33 +0000319
Diana Picuse97822e2017-04-24 07:22:31 +0000320 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
Tim Northovere1a5f662019-08-09 08:26:38 +0000321
322 CallLowering::CallLoweringInfo Info;
323 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
324 Info.Callee = MachineOperand::CreateES(Name);
325 Info.OrigRet = Result;
326 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
327 if (!CLI.lowerCall(MIRBuilder, Info))
Diana Picus02e11012017-06-15 10:53:31 +0000328 return LegalizerHelper::UnableToLegalize;
Diana Picusd0104ea2017-07-06 09:09:33 +0000329
Diana Picuse97822e2017-04-24 07:22:31 +0000330 return LegalizerHelper::Legalized;
331}
332
Diana Picus65ed3642018-01-17 13:34:10 +0000333// Useful for libcalls where all operands have the same type.
Diana Picus02e11012017-06-15 10:53:31 +0000334static LegalizerHelper::LegalizeResult
335simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
336 Type *OpType) {
337 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
Diana Picuse74243d2018-01-12 11:30:45 +0000338
339 SmallVector<CallLowering::ArgInfo, 3> Args;
340 for (unsigned i = 1; i < MI.getNumOperands(); i++)
341 Args.push_back({MI.getOperand(i).getReg(), OpType});
Diana Picusfc1675e2017-07-05 12:57:24 +0000342 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
Diana Picuse74243d2018-01-12 11:30:45 +0000343 Args);
Diana Picus02e11012017-06-15 10:53:31 +0000344}
345
Amara Emersoncf12c782019-07-19 00:24:45 +0000346LegalizerHelper::LegalizeResult
347llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
348 MachineInstr &MI) {
349 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
350 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
351
352 SmallVector<CallLowering::ArgInfo, 3> Args;
353 for (unsigned i = 1; i < MI.getNumOperands(); i++) {
354 Register Reg = MI.getOperand(i).getReg();
355
356 // Need derive an IR type for call lowering.
357 LLT OpLLT = MRI.getType(Reg);
358 Type *OpTy = nullptr;
359 if (OpLLT.isPointer())
360 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
361 else
362 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
363 Args.push_back({Reg, OpTy});
364 }
365
366 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
367 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
368 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID();
369 RTLIB::Libcall RTLibcall;
370 switch (ID) {
371 case Intrinsic::memcpy:
372 RTLibcall = RTLIB::MEMCPY;
373 break;
374 case Intrinsic::memset:
375 RTLibcall = RTLIB::MEMSET;
376 break;
377 case Intrinsic::memmove:
378 RTLibcall = RTLIB::MEMMOVE;
379 break;
380 default:
381 return LegalizerHelper::UnableToLegalize;
382 }
383 const char *Name = TLI.getLibcallName(RTLibcall);
384
385 MIRBuilder.setInstr(MI);
386 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
Tim Northovere1a5f662019-08-09 08:26:38 +0000387
388 CallLowering::CallLoweringInfo Info;
389 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
390 Info.Callee = MachineOperand::CreateES(Name);
391 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
392 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
393 if (!CLI.lowerCall(MIRBuilder, Info))
Amara Emersoncf12c782019-07-19 00:24:45 +0000394 return LegalizerHelper::UnableToLegalize;
395
396 return LegalizerHelper::Legalized;
397}
398
Diana Picus65ed3642018-01-17 13:34:10 +0000399static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
400 Type *FromType) {
401 auto ToMVT = MVT::getVT(ToType);
402 auto FromMVT = MVT::getVT(FromType);
403
404 switch (Opcode) {
405 case TargetOpcode::G_FPEXT:
406 return RTLIB::getFPEXT(FromMVT, ToMVT);
407 case TargetOpcode::G_FPTRUNC:
408 return RTLIB::getFPROUND(FromMVT, ToMVT);
Diana Picus4ed0ee72018-01-30 07:54:52 +0000409 case TargetOpcode::G_FPTOSI:
410 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
411 case TargetOpcode::G_FPTOUI:
412 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
Diana Picus517531e2018-01-30 09:15:17 +0000413 case TargetOpcode::G_SITOFP:
414 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
415 case TargetOpcode::G_UITOFP:
416 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
Diana Picus65ed3642018-01-17 13:34:10 +0000417 }
418 llvm_unreachable("Unsupported libcall function");
419}
420
421static LegalizerHelper::LegalizeResult
422conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
423 Type *FromType) {
424 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
425 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
426 {{MI.getOperand(1).getReg(), FromType}});
427}
428
Tim Northover69fa84a2016-10-14 22:18:18 +0000429LegalizerHelper::LegalizeResult
430LegalizerHelper::libcall(MachineInstr &MI) {
Diana Picus02e11012017-06-15 10:53:31 +0000431 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
432 unsigned Size = LLTy.getSizeInBits();
Matthias Braunf1caa282017-12-15 22:22:58 +0000433 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000434
Diana Picusfc1675e2017-07-05 12:57:24 +0000435 MIRBuilder.setInstr(MI);
436
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000437 switch (MI.getOpcode()) {
438 default:
439 return UnableToLegalize;
Diana Picuse97822e2017-04-24 07:22:31 +0000440 case TargetOpcode::G_SDIV:
Diana Picus02e11012017-06-15 10:53:31 +0000441 case TargetOpcode::G_UDIV:
442 case TargetOpcode::G_SREM:
Diana Picus0528e2c2018-11-26 11:07:02 +0000443 case TargetOpcode::G_UREM:
444 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000445 Type *HLTy = IntegerType::get(Ctx, Size);
Diana Picusfc1675e2017-07-05 12:57:24 +0000446 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
447 if (Status != Legalized)
448 return Status;
449 break;
Diana Picuse97822e2017-04-24 07:22:31 +0000450 }
Diana Picus1314a282017-04-11 10:52:34 +0000451 case TargetOpcode::G_FADD:
Javed Absar5cde1cc2017-10-30 13:51:56 +0000452 case TargetOpcode::G_FSUB:
Diana Picus9faa09b2017-11-23 12:44:20 +0000453 case TargetOpcode::G_FMUL:
Diana Picusc01f7f12017-11-23 13:26:07 +0000454 case TargetOpcode::G_FDIV:
Diana Picuse74243d2018-01-12 11:30:45 +0000455 case TargetOpcode::G_FMA:
Tim Northovere0418412017-02-08 23:23:39 +0000456 case TargetOpcode::G_FPOW:
Jessica Paquette7db82d72019-01-28 18:34:18 +0000457 case TargetOpcode::G_FREM:
458 case TargetOpcode::G_FCOS:
Jessica Paquettec49428a2019-01-28 19:53:14 +0000459 case TargetOpcode::G_FSIN:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000460 case TargetOpcode::G_FLOG10:
Jessica Paquette0154bd12019-01-30 21:16:04 +0000461 case TargetOpcode::G_FLOG:
Jessica Paquette84bedac2019-01-30 23:46:15 +0000462 case TargetOpcode::G_FLOG2:
Jessica Paquettee7941212019-04-03 16:58:32 +0000463 case TargetOpcode::G_FEXP:
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000464 case TargetOpcode::G_FEXP2:
465 case TargetOpcode::G_FCEIL:
466 case TargetOpcode::G_FFLOOR: {
Jessica Paquette7db82d72019-01-28 18:34:18 +0000467 if (Size > 64) {
468 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
469 return UnableToLegalize;
470 }
Diana Picus02e11012017-06-15 10:53:31 +0000471 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
Diana Picusfc1675e2017-07-05 12:57:24 +0000472 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
473 if (Status != Legalized)
474 return Status;
475 break;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000476 }
Diana Picus65ed3642018-01-17 13:34:10 +0000477 case TargetOpcode::G_FPEXT: {
478 // FIXME: Support other floating point types (half, fp128 etc)
479 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
480 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
481 if (ToSize != 64 || FromSize != 32)
482 return UnableToLegalize;
483 LegalizeResult Status = conversionLibcall(
484 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
485 if (Status != Legalized)
486 return Status;
487 break;
488 }
489 case TargetOpcode::G_FPTRUNC: {
490 // FIXME: Support other floating point types (half, fp128 etc)
491 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
492 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
493 if (ToSize != 32 || FromSize != 64)
494 return UnableToLegalize;
495 LegalizeResult Status = conversionLibcall(
496 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
497 if (Status != Legalized)
498 return Status;
499 break;
500 }
Diana Picus4ed0ee72018-01-30 07:54:52 +0000501 case TargetOpcode::G_FPTOSI:
502 case TargetOpcode::G_FPTOUI: {
503 // FIXME: Support other types
504 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
505 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000506 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
Diana Picus4ed0ee72018-01-30 07:54:52 +0000507 return UnableToLegalize;
508 LegalizeResult Status = conversionLibcall(
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000509 MI, MIRBuilder,
510 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
Diana Picus4ed0ee72018-01-30 07:54:52 +0000511 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
512 if (Status != Legalized)
513 return Status;
514 break;
515 }
Diana Picus517531e2018-01-30 09:15:17 +0000516 case TargetOpcode::G_SITOFP:
517 case TargetOpcode::G_UITOFP: {
518 // FIXME: Support other types
519 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
520 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic153bd242019-06-20 09:05:02 +0000521 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
Diana Picus517531e2018-01-30 09:15:17 +0000522 return UnableToLegalize;
523 LegalizeResult Status = conversionLibcall(
524 MI, MIRBuilder,
525 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
Petar Avramovic153bd242019-06-20 09:05:02 +0000526 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
Diana Picus517531e2018-01-30 09:15:17 +0000527 if (Status != Legalized)
528 return Status;
529 break;
530 }
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000531 }
Diana Picusfc1675e2017-07-05 12:57:24 +0000532
533 MI.eraseFromParent();
534 return Legalized;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000535}
536
Tim Northover69fa84a2016-10-14 22:18:18 +0000537LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
538 unsigned TypeIdx,
539 LLT NarrowTy) {
Justin Bognerfde01042017-01-18 17:29:54 +0000540 MIRBuilder.setInstr(MI);
541
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000542 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
543 uint64_t NarrowSize = NarrowTy.getSizeInBits();
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000544
Tim Northover9656f142016-08-04 20:54:13 +0000545 switch (MI.getOpcode()) {
546 default:
547 return UnableToLegalize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000548 case TargetOpcode::G_IMPLICIT_DEF: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000549 // FIXME: add support for when SizeOp0 isn't an exact multiple of
550 // NarrowSize.
551 if (SizeOp0 % NarrowSize != 0)
552 return UnableToLegalize;
553 int NumParts = SizeOp0 / NarrowSize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000554
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000555 SmallVector<Register, 2> DstRegs;
Volkan Keles02bb1742018-02-14 19:58:36 +0000556 for (int i = 0; i < NumParts; ++i)
557 DstRegs.push_back(
558 MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
Amara Emerson5ec146042018-12-10 18:44:58 +0000559
Matt Arsenault3018d182019-06-28 01:47:44 +0000560 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000561 if(MRI.getType(DstReg).isVector())
562 MIRBuilder.buildBuildVector(DstReg, DstRegs);
563 else
564 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northoverff5e7e12017-06-30 20:27:36 +0000565 MI.eraseFromParent();
566 return Legalized;
567 }
Matt Arsenault71872722019-04-10 17:27:53 +0000568 case TargetOpcode::G_CONSTANT: {
569 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
570 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
571 unsigned TotalSize = Ty.getSizeInBits();
572 unsigned NarrowSize = NarrowTy.getSizeInBits();
573 int NumParts = TotalSize / NarrowSize;
574
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000575 SmallVector<Register, 4> PartRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000576 for (int I = 0; I != NumParts; ++I) {
577 unsigned Offset = I * NarrowSize;
578 auto K = MIRBuilder.buildConstant(NarrowTy,
579 Val.lshr(Offset).trunc(NarrowSize));
580 PartRegs.push_back(K.getReg(0));
581 }
582
583 LLT LeftoverTy;
584 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000585 SmallVector<Register, 1> LeftoverRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000586 if (LeftoverBits != 0) {
587 LeftoverTy = LLT::scalar(LeftoverBits);
588 auto K = MIRBuilder.buildConstant(
589 LeftoverTy,
590 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
591 LeftoverRegs.push_back(K.getReg(0));
592 }
593
594 insertParts(MI.getOperand(0).getReg(),
595 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
596
597 MI.eraseFromParent();
598 return Legalized;
599 }
Amara Emerson7bc4fad2019-07-26 23:46:38 +0000600 case TargetOpcode::G_SEXT: {
601 if (TypeIdx != 0)
602 return UnableToLegalize;
603
604 if (NarrowTy.getSizeInBits() != SizeOp0 / 2) {
605 LLVM_DEBUG(dbgs() << "Can't narrow sext to type " << NarrowTy << "\n");
606 return UnableToLegalize;
607 }
608
609 Register SrcReg = MI.getOperand(1).getReg();
610
611 // Shift the sign bit of the low register through the high register.
612 auto ShiftAmt =
613 MIRBuilder.buildConstant(LLT::scalar(64), NarrowTy.getSizeInBits() - 1);
614 auto Shift = MIRBuilder.buildAShr(NarrowTy, SrcReg, ShiftAmt);
615 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {SrcReg, Shift.getReg(0)});
616 MI.eraseFromParent();
617 return Legalized;
618 }
Amara Emerson56606a42019-08-21 00:12:37 +0000619 case TargetOpcode::G_ZEXT: {
620 if (TypeIdx != 0)
621 return UnableToLegalize;
622
623 if (SizeOp0 % NarrowTy.getSizeInBits() != 0)
624 return UnableToLegalize;
625
626 // Generate a merge where the bottom bits are taken from the source, and
627 // zero everything else.
628 Register ZeroReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
629 unsigned NumParts = SizeOp0 / NarrowTy.getSizeInBits();
630 SmallVector<Register, 4> Srcs = {MI.getOperand(1).getReg()};
631 for (unsigned Part = 1; Part < NumParts; ++Part)
632 Srcs.push_back(ZeroReg);
633 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), Srcs);
634 MI.eraseFromParent();
635 return Legalized;
636 }
Petar Avramovic5b4c5c22019-08-21 09:26:39 +0000637 case TargetOpcode::G_TRUNC: {
638 if (TypeIdx != 1)
639 return UnableToLegalize;
640
641 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
642 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
643 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
644 return UnableToLegalize;
645 }
646
647 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1).getReg());
648 MIRBuilder.buildCopy(MI.getOperand(0).getReg(), Unmerge.getReg(0));
649 MI.eraseFromParent();
650 return Legalized;
651 }
Amara Emerson7bc4fad2019-07-26 23:46:38 +0000652
Tim Northover9656f142016-08-04 20:54:13 +0000653 case TargetOpcode::G_ADD: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000654 // FIXME: add support for when SizeOp0 isn't an exact multiple of
655 // NarrowSize.
656 if (SizeOp0 % NarrowSize != 0)
657 return UnableToLegalize;
Tim Northover9656f142016-08-04 20:54:13 +0000658 // Expand in terms of carry-setting/consuming G_ADDE instructions.
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000659 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
Tim Northover9656f142016-08-04 20:54:13 +0000660
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000661 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Tim Northover9656f142016-08-04 20:54:13 +0000662 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
663 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
664
Matt Arsenaultfba82852019-08-22 17:29:17 +0000665 Register CarryIn;
Tim Northover9656f142016-08-04 20:54:13 +0000666 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000667 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
668 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Tim Northover9656f142016-08-04 20:54:13 +0000669
Matt Arsenaultfba82852019-08-22 17:29:17 +0000670 if (i == 0)
671 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
672 else {
673 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
674 Src2Regs[i], CarryIn);
675 }
Tim Northover9656f142016-08-04 20:54:13 +0000676
677 DstRegs.push_back(DstReg);
678 CarryIn = CarryOut;
679 }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000680 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000681 if(MRI.getType(DstReg).isVector())
682 MIRBuilder.buildBuildVector(DstReg, DstRegs);
683 else
684 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northover9656f142016-08-04 20:54:13 +0000685 MI.eraseFromParent();
686 return Legalized;
687 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000688 case TargetOpcode::G_SUB: {
689 // FIXME: add support for when SizeOp0 isn't an exact multiple of
690 // NarrowSize.
691 if (SizeOp0 % NarrowSize != 0)
692 return UnableToLegalize;
693
694 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
695
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000696 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000697 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
698 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
699
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000700 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
701 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000702 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
703 {Src1Regs[0], Src2Regs[0]});
704 DstRegs.push_back(DstReg);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000705 Register BorrowIn = BorrowOut;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000706 for (int i = 1; i < NumParts; ++i) {
707 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
708 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
709
710 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
711 {Src1Regs[i], Src2Regs[i], BorrowIn});
712
713 DstRegs.push_back(DstReg);
714 BorrowIn = BorrowOut;
715 }
716 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
717 MI.eraseFromParent();
718 return Legalized;
719 }
Matt Arsenault211e89d2019-01-27 00:52:51 +0000720 case TargetOpcode::G_MUL:
Petar Avramovic5229f472019-03-11 10:08:44 +0000721 case TargetOpcode::G_UMULH:
Petar Avramovic0b17e592019-03-11 10:00:17 +0000722 return narrowScalarMul(MI, NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +0000723 case TargetOpcode::G_EXTRACT:
724 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
725 case TargetOpcode::G_INSERT:
726 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000727 case TargetOpcode::G_LOAD: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000728 const auto &MMO = **MI.memoperands_begin();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000729 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000730 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000731 if (DstTy.isVector())
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000732 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000733
734 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000735 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000736 auto &MMO = **MI.memoperands_begin();
737 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
738 MIRBuilder.buildAnyExt(DstReg, TmpReg);
739 MI.eraseFromParent();
740 return Legalized;
741 }
742
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000743 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000744 }
Matt Arsenault6614f852019-01-22 19:02:10 +0000745 case TargetOpcode::G_ZEXTLOAD:
746 case TargetOpcode::G_SEXTLOAD: {
747 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000748 Register DstReg = MI.getOperand(0).getReg();
749 Register PtrReg = MI.getOperand(1).getReg();
Matt Arsenault6614f852019-01-22 19:02:10 +0000750
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000751 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault6614f852019-01-22 19:02:10 +0000752 auto &MMO = **MI.memoperands_begin();
Amara Emersond51adf02019-04-17 22:21:05 +0000753 if (MMO.getSizeInBits() == NarrowSize) {
Matt Arsenault6614f852019-01-22 19:02:10 +0000754 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
755 } else {
756 unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
757 : TargetOpcode::G_SEXTLOAD;
758 MIRBuilder.buildInstr(ExtLoad)
759 .addDef(TmpReg)
760 .addUse(PtrReg)
761 .addMemOperand(&MMO);
762 }
763
764 if (ZExt)
765 MIRBuilder.buildZExt(DstReg, TmpReg);
766 else
767 MIRBuilder.buildSExt(DstReg, TmpReg);
768
769 MI.eraseFromParent();
770 return Legalized;
771 }
Justin Bognerfde01042017-01-18 17:29:54 +0000772 case TargetOpcode::G_STORE: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000773 const auto &MMO = **MI.memoperands_begin();
Matt Arsenault18619af2019-01-29 18:13:02 +0000774
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000775 Register SrcReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000776 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000777 if (SrcTy.isVector())
778 return UnableToLegalize;
779
780 int NumParts = SizeOp0 / NarrowSize;
781 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
782 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
783 if (SrcTy.isVector() && LeftoverBits != 0)
784 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000785
786 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000787 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000788 auto &MMO = **MI.memoperands_begin();
789 MIRBuilder.buildTrunc(TmpReg, SrcReg);
790 MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
791 MI.eraseFromParent();
792 return Legalized;
793 }
794
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000795 return reduceLoadStoreWidth(MI, 0, NarrowTy);
Justin Bognerfde01042017-01-18 17:29:54 +0000796 }
Matt Arsenault81511e52019-02-05 00:13:44 +0000797 case TargetOpcode::G_SELECT:
798 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
Petar Avramovic150fd432018-12-18 11:36:14 +0000799 case TargetOpcode::G_AND:
800 case TargetOpcode::G_OR:
801 case TargetOpcode::G_XOR: {
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000802 // Legalize bitwise operation:
803 // A = BinOp<Ty> B, C
804 // into:
805 // B1, ..., BN = G_UNMERGE_VALUES B
806 // C1, ..., CN = G_UNMERGE_VALUES C
807 // A1 = BinOp<Ty/N> B1, C2
808 // ...
809 // AN = BinOp<Ty/N> BN, CN
810 // A = G_MERGE_VALUES A1, ..., AN
Matt Arsenault9e0eeba2019-04-10 17:07:56 +0000811 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000812 }
Matt Arsenault30989e42019-01-22 21:42:11 +0000813 case TargetOpcode::G_SHL:
814 case TargetOpcode::G_LSHR:
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000815 case TargetOpcode::G_ASHR:
816 return narrowScalarShift(MI, TypeIdx, NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000817 case TargetOpcode::G_CTLZ:
818 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
819 case TargetOpcode::G_CTTZ:
820 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
821 case TargetOpcode::G_CTPOP:
822 if (TypeIdx != 0)
823 return UnableToLegalize; // TODO
824
825 Observer.changingInstr(MI);
826 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
827 Observer.changedInstr(MI);
828 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000829 case TargetOpcode::G_INTTOPTR:
830 if (TypeIdx != 1)
831 return UnableToLegalize;
832
833 Observer.changingInstr(MI);
834 narrowScalarSrc(MI, NarrowTy, 1);
835 Observer.changedInstr(MI);
836 return Legalized;
837 case TargetOpcode::G_PTRTOINT:
838 if (TypeIdx != 0)
839 return UnableToLegalize;
840
841 Observer.changingInstr(MI);
842 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
843 Observer.changedInstr(MI);
844 return Legalized;
Petar Avramovicbe20e362019-07-09 14:36:17 +0000845 case TargetOpcode::G_PHI: {
846 unsigned NumParts = SizeOp0 / NarrowSize;
847 SmallVector<Register, 2> DstRegs;
848 SmallVector<SmallVector<Register, 2>, 2> SrcRegs;
849 DstRegs.resize(NumParts);
850 SrcRegs.resize(MI.getNumOperands() / 2);
851 Observer.changingInstr(MI);
852 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
853 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
854 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
855 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
856 SrcRegs[i / 2]);
857 }
858 MachineBasicBlock &MBB = *MI.getParent();
859 MIRBuilder.setInsertPt(MBB, MI);
860 for (unsigned i = 0; i < NumParts; ++i) {
861 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
862 MachineInstrBuilder MIB =
863 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
864 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
865 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
866 }
867 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
868 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
869 Observer.changedInstr(MI);
870 MI.eraseFromParent();
871 return Legalized;
872 }
Matt Arsenault434d6642019-07-15 19:37:34 +0000873 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
874 case TargetOpcode::G_INSERT_VECTOR_ELT: {
875 if (TypeIdx != 2)
876 return UnableToLegalize;
877
878 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
879 Observer.changingInstr(MI);
880 narrowScalarSrc(MI, NarrowTy, OpIdx);
881 Observer.changedInstr(MI);
882 return Legalized;
883 }
Petar Avramovic1e626352019-07-17 12:08:01 +0000884 case TargetOpcode::G_ICMP: {
885 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
886 if (NarrowSize * 2 != SrcSize)
887 return UnableToLegalize;
888
889 Observer.changingInstr(MI);
890 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
891 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
892 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2).getReg());
893
894 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
895 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
896 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3).getReg());
897
898 CmpInst::Predicate Pred =
899 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
Amara Emersona1997ce2019-07-24 20:46:42 +0000900 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
Petar Avramovic1e626352019-07-17 12:08:01 +0000901
902 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
903 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
904 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
905 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
906 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
907 MIRBuilder.buildICmp(Pred, MI.getOperand(0).getReg(), Or, Zero);
908 } else {
Amara Emersona1997ce2019-07-24 20:46:42 +0000909 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
Petar Avramovic1e626352019-07-17 12:08:01 +0000910 MachineInstrBuilder CmpHEQ =
Amara Emersona1997ce2019-07-24 20:46:42 +0000911 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
Petar Avramovic1e626352019-07-17 12:08:01 +0000912 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
Amara Emersona1997ce2019-07-24 20:46:42 +0000913 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
Petar Avramovic1e626352019-07-17 12:08:01 +0000914 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), CmpHEQ, CmpLU, CmpH);
915 }
916 Observer.changedInstr(MI);
917 MI.eraseFromParent();
918 return Legalized;
919 }
Daniel Sanderse9a57c22019-08-09 21:11:20 +0000920 case TargetOpcode::G_SEXT_INREG: {
921 if (TypeIdx != 0)
922 return UnableToLegalize;
923
924 if (!MI.getOperand(2).isImm())
925 return UnableToLegalize;
926 int64_t SizeInBits = MI.getOperand(2).getImm();
927
928 // So long as the new type has more bits than the bits we're extending we
929 // don't need to break it apart.
930 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
931 Observer.changingInstr(MI);
932 // We don't lose any non-extension bits by truncating the src and
933 // sign-extending the dst.
934 MachineOperand &MO1 = MI.getOperand(1);
935 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1.getReg());
936 MO1.setReg(TruncMIB->getOperand(0).getReg());
937
938 MachineOperand &MO2 = MI.getOperand(0);
939 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
940 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
941 MIRBuilder.buildInstr(TargetOpcode::G_SEXT, {MO2.getReg()}, {DstExt});
942 MO2.setReg(DstExt);
943 Observer.changedInstr(MI);
944 return Legalized;
945 }
946
947 // Break it apart. Components below the extension point are unmodified. The
948 // component containing the extension point becomes a narrower SEXT_INREG.
949 // Components above it are ashr'd from the component containing the
950 // extension point.
951 if (SizeOp0 % NarrowSize != 0)
952 return UnableToLegalize;
953 int NumParts = SizeOp0 / NarrowSize;
954
955 // List the registers where the destination will be scattered.
956 SmallVector<Register, 2> DstRegs;
957 // List the registers where the source will be split.
958 SmallVector<Register, 2> SrcRegs;
959
960 // Create all the temporary registers.
961 for (int i = 0; i < NumParts; ++i) {
962 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
963
964 SrcRegs.push_back(SrcReg);
965 }
966
967 // Explode the big arguments into smaller chunks.
968 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1).getReg());
969
970 Register AshrCstReg =
971 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
972 ->getOperand(0)
973 .getReg();
974 Register FullExtensionReg = 0;
975 Register PartialExtensionReg = 0;
976
977 // Do the operation on each small part.
978 for (int i = 0; i < NumParts; ++i) {
979 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
980 DstRegs.push_back(SrcRegs[i]);
981 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
982 assert(PartialExtensionReg &&
983 "Expected to visit partial extension before full");
984 if (FullExtensionReg) {
985 DstRegs.push_back(FullExtensionReg);
986 continue;
987 }
988 DstRegs.push_back(MIRBuilder
989 .buildInstr(TargetOpcode::G_ASHR, {NarrowTy},
990 {PartialExtensionReg, AshrCstReg})
991 ->getOperand(0)
992 .getReg());
993 FullExtensionReg = DstRegs.back();
994 } else {
995 DstRegs.push_back(
996 MIRBuilder
997 .buildInstr(
998 TargetOpcode::G_SEXT_INREG, {NarrowTy},
999 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1000 ->getOperand(0)
1001 .getReg());
1002 PartialExtensionReg = DstRegs.back();
1003 }
1004 }
1005
1006 // Gather the destination registers into the final destination.
1007 Register DstReg = MI.getOperand(0).getReg();
1008 MIRBuilder.buildMerge(DstReg, DstRegs);
1009 MI.eraseFromParent();
1010 return Legalized;
1011 }
Tim Northover9656f142016-08-04 20:54:13 +00001012 }
Tim Northover33b07d62016-07-22 20:03:43 +00001013}
1014
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001015void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1016 unsigned OpIdx, unsigned ExtOpcode) {
1017 MachineOperand &MO = MI.getOperand(OpIdx);
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001018 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001019 MO.setReg(ExtB->getOperand(0).getReg());
1020}
1021
Matt Arsenault30989e42019-01-22 21:42:11 +00001022void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1023 unsigned OpIdx) {
1024 MachineOperand &MO = MI.getOperand(OpIdx);
1025 auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
1026 {MO.getReg()});
1027 MO.setReg(ExtB->getOperand(0).getReg());
1028}
1029
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001030void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1031 unsigned OpIdx, unsigned TruncOpcode) {
1032 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001033 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001034 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001035 MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001036 MO.setReg(DstExt);
1037}
1038
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001039void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1040 unsigned OpIdx, unsigned ExtOpcode) {
1041 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001042 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001043 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1044 MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
1045 MO.setReg(DstTrunc);
1046}
1047
Matt Arsenault18ec3822019-02-11 22:00:39 +00001048void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1049 unsigned OpIdx) {
1050 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001051 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +00001052 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1053 MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
1054 MO.setReg(DstExt);
1055}
1056
Matt Arsenault26b7e852019-02-19 16:30:19 +00001057void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1058 unsigned OpIdx) {
1059 MachineOperand &MO = MI.getOperand(OpIdx);
1060
1061 LLT OldTy = MRI.getType(MO.getReg());
1062 unsigned OldElts = OldTy.getNumElements();
1063 unsigned NewElts = MoreTy.getNumElements();
1064
1065 unsigned NumParts = NewElts / OldElts;
1066
1067 // Use concat_vectors if the result is a multiple of the number of elements.
1068 if (NumParts * OldElts == NewElts) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001069 SmallVector<Register, 8> Parts;
Matt Arsenault26b7e852019-02-19 16:30:19 +00001070 Parts.push_back(MO.getReg());
1071
Matt Arsenault3018d182019-06-28 01:47:44 +00001072 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +00001073 for (unsigned I = 1; I != NumParts; ++I)
1074 Parts.push_back(ImpDef);
1075
1076 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1077 MO.setReg(Concat.getReg(0));
1078 return;
1079 }
1080
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001081 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1082 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +00001083 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1084 MO.setReg(MoreReg);
1085}
1086
Tim Northover69fa84a2016-10-14 22:18:18 +00001087LegalizerHelper::LegalizeResult
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001088LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1089 LLT WideTy) {
1090 if (TypeIdx != 1)
1091 return UnableToLegalize;
1092
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001093 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001094 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault43cbca52019-07-03 23:08:06 +00001095 if (DstTy.isVector())
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001096 return UnableToLegalize;
1097
Matt Arsenaultc9f14f22019-07-01 19:36:10 +00001098 Register Src1 = MI.getOperand(1).getReg();
1099 LLT SrcTy = MRI.getType(Src1);
Matt Arsenault0966dd02019-07-17 20:22:44 +00001100 const int DstSize = DstTy.getSizeInBits();
1101 const int SrcSize = SrcTy.getSizeInBits();
1102 const int WideSize = WideTy.getSizeInBits();
1103 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
Matt Arsenaultc9f14f22019-07-01 19:36:10 +00001104
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001105 unsigned NumOps = MI.getNumOperands();
1106 unsigned NumSrc = MI.getNumOperands() - 1;
1107 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1108
Matt Arsenault0966dd02019-07-17 20:22:44 +00001109 if (WideSize >= DstSize) {
1110 // Directly pack the bits in the target type.
1111 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001112
Matt Arsenault0966dd02019-07-17 20:22:44 +00001113 for (unsigned I = 2; I != NumOps; ++I) {
1114 const unsigned Offset = (I - 1) * PartSize;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001115
Matt Arsenault0966dd02019-07-17 20:22:44 +00001116 Register SrcReg = MI.getOperand(I).getReg();
1117 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
1118
1119 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1120
Matt Arsenault5faa5332019-08-01 18:13:16 +00001121 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
Matt Arsenault0966dd02019-07-17 20:22:44 +00001122 MRI.createGenericVirtualRegister(WideTy);
1123
1124 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1125 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1126 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1127 ResultReg = NextResult;
1128 }
1129
1130 if (WideSize > DstSize)
1131 MIRBuilder.buildTrunc(DstReg, ResultReg);
Matt Arsenault5faa5332019-08-01 18:13:16 +00001132 else if (DstTy.isPointer())
1133 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
Matt Arsenault0966dd02019-07-17 20:22:44 +00001134
1135 MI.eraseFromParent();
1136 return Legalized;
1137 }
1138
1139 // Unmerge the original values to the GCD type, and recombine to the next
1140 // multiple greater than the original type.
1141 //
1142 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1143 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1144 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1145 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1146 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1147 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1148 // %12:_(s12) = G_MERGE_VALUES %10, %11
1149 //
1150 // Padding with undef if necessary:
1151 //
1152 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1153 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1154 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1155 // %7:_(s2) = G_IMPLICIT_DEF
1156 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1157 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1158 // %10:_(s12) = G_MERGE_VALUES %8, %9
1159
1160 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1161 LLT GCDTy = LLT::scalar(GCD);
1162
1163 SmallVector<Register, 8> Parts;
1164 SmallVector<Register, 8> NewMergeRegs;
1165 SmallVector<Register, 8> Unmerges;
1166 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1167
1168 // Decompose the original operands if they don't evenly divide.
1169 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001170 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenault0966dd02019-07-17 20:22:44 +00001171 if (GCD == SrcSize) {
1172 Unmerges.push_back(SrcReg);
1173 } else {
1174 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1175 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1176 Unmerges.push_back(Unmerge.getReg(J));
1177 }
1178 }
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001179
Matt Arsenault0966dd02019-07-17 20:22:44 +00001180 // Pad with undef to the next size that is a multiple of the requested size.
1181 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1182 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1183 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1184 Unmerges.push_back(UndefReg);
1185 }
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001186
Matt Arsenault0966dd02019-07-17 20:22:44 +00001187 const int PartsPerGCD = WideSize / GCD;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001188
Matt Arsenault0966dd02019-07-17 20:22:44 +00001189 // Build merges of each piece.
1190 ArrayRef<Register> Slicer(Unmerges);
1191 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1192 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1193 NewMergeRegs.push_back(Merge.getReg(0));
1194 }
1195
1196 // A truncate may be necessary if the requested type doesn't evenly divide the
1197 // original result type.
1198 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1199 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1200 } else {
1201 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1202 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001203 }
1204
1205 MI.eraseFromParent();
1206 return Legalized;
1207}
1208
1209LegalizerHelper::LegalizeResult
1210LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1211 LLT WideTy) {
1212 if (TypeIdx != 0)
1213 return UnableToLegalize;
1214
1215 unsigned NumDst = MI.getNumOperands() - 1;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001216 Register SrcReg = MI.getOperand(NumDst).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001217 LLT SrcTy = MRI.getType(SrcReg);
1218 if (!SrcTy.isScalar())
1219 return UnableToLegalize;
1220
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001221 Register Dst0Reg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001222 LLT DstTy = MRI.getType(Dst0Reg);
1223 if (!DstTy.isScalar())
1224 return UnableToLegalize;
1225
1226 unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
1227 LLT NewSrcTy = LLT::scalar(NewSrcSize);
1228 unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
1229
1230 auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
1231
1232 for (unsigned I = 1; I != NumDst; ++I) {
1233 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
1234 auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
1235 WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
1236 }
1237
1238 Observer.changingInstr(MI);
1239
1240 MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
1241 for (unsigned I = 0; I != NumDst; ++I)
1242 widenScalarDst(MI, WideTy, I);
1243
1244 Observer.changedInstr(MI);
1245
1246 return Legalized;
1247}
1248
1249LegalizerHelper::LegalizeResult
Matt Arsenault1cf713662019-02-12 14:54:52 +00001250LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1251 LLT WideTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001252 Register DstReg = MI.getOperand(0).getReg();
1253 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00001254 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenaultfbe92a52019-02-18 22:39:27 +00001255
1256 LLT DstTy = MRI.getType(DstReg);
1257 unsigned Offset = MI.getOperand(2).getImm();
1258
1259 if (TypeIdx == 0) {
1260 if (SrcTy.isVector() || DstTy.isVector())
1261 return UnableToLegalize;
1262
1263 SrcOp Src(SrcReg);
1264 if (SrcTy.isPointer()) {
1265 // Extracts from pointers can be handled only if they are really just
1266 // simple integers.
1267 const DataLayout &DL = MIRBuilder.getDataLayout();
1268 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1269 return UnableToLegalize;
1270
1271 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1272 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1273 SrcTy = SrcAsIntTy;
1274 }
1275
1276 if (DstTy.isPointer())
1277 return UnableToLegalize;
1278
1279 if (Offset == 0) {
1280 // Avoid a shift in the degenerate case.
1281 MIRBuilder.buildTrunc(DstReg,
1282 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1283 MI.eraseFromParent();
1284 return Legalized;
1285 }
1286
1287 // Do a shift in the source type.
1288 LLT ShiftTy = SrcTy;
1289 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1290 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1291 ShiftTy = WideTy;
1292 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
1293 return UnableToLegalize;
1294
1295 auto LShr = MIRBuilder.buildLShr(
1296 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1297 MIRBuilder.buildTrunc(DstReg, LShr);
1298 MI.eraseFromParent();
1299 return Legalized;
1300 }
1301
Matt Arsenault8f624ab2019-04-22 15:10:42 +00001302 if (SrcTy.isScalar()) {
1303 Observer.changingInstr(MI);
1304 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1305 Observer.changedInstr(MI);
1306 return Legalized;
1307 }
1308
Matt Arsenault1cf713662019-02-12 14:54:52 +00001309 if (!SrcTy.isVector())
1310 return UnableToLegalize;
1311
Matt Arsenault1cf713662019-02-12 14:54:52 +00001312 if (DstTy != SrcTy.getElementType())
1313 return UnableToLegalize;
1314
Matt Arsenault1cf713662019-02-12 14:54:52 +00001315 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1316 return UnableToLegalize;
1317
1318 Observer.changingInstr(MI);
1319 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1320
1321 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1322 Offset);
1323 widenScalarDst(MI, WideTy.getScalarType(), 0);
1324 Observer.changedInstr(MI);
1325 return Legalized;
1326}
1327
1328LegalizerHelper::LegalizeResult
1329LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1330 LLT WideTy) {
1331 if (TypeIdx != 0)
1332 return UnableToLegalize;
1333 Observer.changingInstr(MI);
1334 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1335 widenScalarDst(MI, WideTy);
1336 Observer.changedInstr(MI);
1337 return Legalized;
1338}
1339
1340LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00001341LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Tim Northover3c73e362016-08-23 18:20:09 +00001342 MIRBuilder.setInstr(MI);
1343
Tim Northover32335812016-08-04 18:35:11 +00001344 switch (MI.getOpcode()) {
1345 default:
1346 return UnableToLegalize;
Matt Arsenault1cf713662019-02-12 14:54:52 +00001347 case TargetOpcode::G_EXTRACT:
1348 return widenScalarExtract(MI, TypeIdx, WideTy);
1349 case TargetOpcode::G_INSERT:
1350 return widenScalarInsert(MI, TypeIdx, WideTy);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001351 case TargetOpcode::G_MERGE_VALUES:
1352 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1353 case TargetOpcode::G_UNMERGE_VALUES:
1354 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001355 case TargetOpcode::G_UADDO:
1356 case TargetOpcode::G_USUBO: {
1357 if (TypeIdx == 1)
1358 return UnableToLegalize; // TODO
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001359 auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1360 {MI.getOperand(2).getReg()});
1361 auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1362 {MI.getOperand(3).getReg()});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001363 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1364 ? TargetOpcode::G_ADD
1365 : TargetOpcode::G_SUB;
1366 // Do the arithmetic in the larger type.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001367 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001368 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1369 APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
1370 auto AndOp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001371 TargetOpcode::G_AND, {WideTy},
1372 {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001373 // There is no overflow if the AndOp is the same as NewOp.
1374 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
1375 AndOp);
1376 // Now trunc the NewOp to the original result.
1377 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1378 MI.eraseFromParent();
1379 return Legalized;
1380 }
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001381 case TargetOpcode::G_CTTZ:
1382 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1383 case TargetOpcode::G_CTLZ:
1384 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1385 case TargetOpcode::G_CTPOP: {
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001386 if (TypeIdx == 0) {
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001387 Observer.changingInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001388 widenScalarDst(MI, WideTy, 0);
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001389 Observer.changedInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001390 return Legalized;
1391 }
1392
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001393 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001394
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001395 // First ZEXT the input.
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001396 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1397 LLT CurTy = MRI.getType(SrcReg);
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001398 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1399 // The count is the same in the larger type except if the original
1400 // value was zero. This can be handled by setting the bit just off
1401 // the top of the original type.
1402 auto TopBit =
1403 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001404 MIBSrc = MIRBuilder.buildOr(
1405 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001406 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001407
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001408 // Perform the operation at the larger size.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001409 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001410 // This is already the correct result for CTPOP and CTTZs
1411 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1412 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1413 // The correct result is NewOp - (Difference in widety and current ty).
1414 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001415 MIBNewOp = MIRBuilder.buildInstr(
1416 TargetOpcode::G_SUB, {WideTy},
1417 {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001418 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001419
1420 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1421 MI.eraseFromParent();
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001422 return Legalized;
1423 }
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001424 case TargetOpcode::G_BSWAP: {
1425 Observer.changingInstr(MI);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001426 Register DstReg = MI.getOperand(0).getReg();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001427
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001428 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1429 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1430 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001431 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1432
1433 MI.getOperand(0).setReg(DstExt);
1434
1435 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1436
1437 LLT Ty = MRI.getType(DstReg);
1438 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1439 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1440 MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1441 .addDef(ShrReg)
1442 .addUse(DstExt)
1443 .addUse(ShiftAmtReg);
1444
1445 MIRBuilder.buildTrunc(DstReg, ShrReg);
1446 Observer.changedInstr(MI);
1447 return Legalized;
1448 }
Tim Northover61c16142016-08-04 21:39:49 +00001449 case TargetOpcode::G_ADD:
1450 case TargetOpcode::G_AND:
1451 case TargetOpcode::G_MUL:
1452 case TargetOpcode::G_OR:
1453 case TargetOpcode::G_XOR:
Justin Bognerddb80ae2017-01-19 07:51:17 +00001454 case TargetOpcode::G_SUB:
Matt Arsenault1cf713662019-02-12 14:54:52 +00001455 // Perform operation at larger width (any extension is fines here, high bits
Tim Northover32335812016-08-04 18:35:11 +00001456 // don't affect the result) and then truncate the result back to the
1457 // original type.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001458 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001459 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1460 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1461 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001462 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001463 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001464
Roman Tereshin6d266382018-05-09 21:43:30 +00001465 case TargetOpcode::G_SHL:
Matt Arsenault012ecbb2019-05-16 04:08:46 +00001466 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001467
1468 if (TypeIdx == 0) {
1469 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1470 widenScalarDst(MI, WideTy);
1471 } else {
1472 assert(TypeIdx == 1);
1473 // The "number of bits to shift" operand must preserve its value as an
1474 // unsigned integer:
1475 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1476 }
1477
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001478 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001479 return Legalized;
1480
Tim Northover7a753d92016-08-26 17:46:06 +00001481 case TargetOpcode::G_SDIV:
Roman Tereshin27bba442018-05-09 01:43:12 +00001482 case TargetOpcode::G_SREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001483 case TargetOpcode::G_SMIN:
1484 case TargetOpcode::G_SMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001485 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001486 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1487 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1488 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001489 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001490 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001491
Roman Tereshin6d266382018-05-09 21:43:30 +00001492 case TargetOpcode::G_ASHR:
Matt Arsenault30989e42019-01-22 21:42:11 +00001493 case TargetOpcode::G_LSHR:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001494 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001495
1496 if (TypeIdx == 0) {
1497 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1498 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1499
1500 widenScalarSrc(MI, WideTy, 1, CvtOp);
1501 widenScalarDst(MI, WideTy);
1502 } else {
1503 assert(TypeIdx == 1);
1504 // The "number of bits to shift" operand must preserve its value as an
1505 // unsigned integer:
1506 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1507 }
1508
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001509 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001510 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001511 case TargetOpcode::G_UDIV:
1512 case TargetOpcode::G_UREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001513 case TargetOpcode::G_UMIN:
1514 case TargetOpcode::G_UMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001515 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001516 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1517 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1518 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001519 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001520 return Legalized;
1521
1522 case TargetOpcode::G_SELECT:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001523 Observer.changingInstr(MI);
Petar Avramovic09dff332018-12-25 14:42:30 +00001524 if (TypeIdx == 0) {
1525 // Perform operation at larger width (any extension is fine here, high
1526 // bits don't affect the result) and then truncate the result back to the
1527 // original type.
1528 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1529 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1530 widenScalarDst(MI, WideTy);
1531 } else {
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001532 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
Petar Avramovic09dff332018-12-25 14:42:30 +00001533 // Explicit extension is required here since high bits affect the result.
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001534 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
Petar Avramovic09dff332018-12-25 14:42:30 +00001535 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001536 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001537 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001538
Ahmed Bougachab6137062017-01-23 21:10:14 +00001539 case TargetOpcode::G_FPTOSI:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001540 case TargetOpcode::G_FPTOUI:
Ahmed Bougachab6137062017-01-23 21:10:14 +00001541 if (TypeIdx != 0)
1542 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001543 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001544 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001545 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001546 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001547
Ahmed Bougachad2948232017-01-20 01:37:24 +00001548 case TargetOpcode::G_SITOFP:
Ahmed Bougachad2948232017-01-20 01:37:24 +00001549 if (TypeIdx != 1)
1550 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001551 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001552 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001553 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001554 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001555
1556 case TargetOpcode::G_UITOFP:
1557 if (TypeIdx != 1)
1558 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001559 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001560 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001561 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001562 return Legalized;
1563
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001564 case TargetOpcode::G_LOAD:
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001565 case TargetOpcode::G_SEXTLOAD:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001566 case TargetOpcode::G_ZEXTLOAD:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001567 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001568 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001569 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001570 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001571
Tim Northover3c73e362016-08-23 18:20:09 +00001572 case TargetOpcode::G_STORE: {
Matt Arsenault92c50012019-01-30 02:04:31 +00001573 if (TypeIdx != 0)
1574 return UnableToLegalize;
1575
1576 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1577 if (!isPowerOf2_32(Ty.getSizeInBits()))
Tim Northover548feee2017-03-21 22:22:05 +00001578 return UnableToLegalize;
1579
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001580 Observer.changingInstr(MI);
Matt Arsenault92c50012019-01-30 02:04:31 +00001581
1582 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1583 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1584 widenScalarSrc(MI, WideTy, 0, ExtType);
1585
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001586 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001587 return Legalized;
1588 }
Tim Northoverea904f92016-08-19 22:40:00 +00001589 case TargetOpcode::G_CONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001590 MachineOperand &SrcMO = MI.getOperand(1);
1591 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1592 const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001593 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001594 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1595
1596 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001597 Observer.changedInstr(MI);
Tim Northoverea904f92016-08-19 22:40:00 +00001598 return Legalized;
1599 }
Tim Northovera11be042016-08-19 22:40:08 +00001600 case TargetOpcode::G_FCONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001601 MachineOperand &SrcMO = MI.getOperand(1);
Amara Emerson77a5c962018-01-27 07:07:20 +00001602 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001603 APFloat Val = SrcMO.getFPImm()->getValueAPF();
Amara Emerson77a5c962018-01-27 07:07:20 +00001604 bool LosesInfo;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001605 switch (WideTy.getSizeInBits()) {
1606 case 32:
Matt Arsenault996c6662019-02-12 14:54:54 +00001607 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1608 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001609 break;
1610 case 64:
Matt Arsenault996c6662019-02-12 14:54:54 +00001611 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1612 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001613 break;
1614 default:
Matt Arsenault996c6662019-02-12 14:54:54 +00001615 return UnableToLegalize;
Tim Northover6cd4b232016-08-23 21:01:26 +00001616 }
Matt Arsenault996c6662019-02-12 14:54:54 +00001617
1618 assert(!LosesInfo && "extend should always be lossless");
1619
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001620 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001621 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1622
1623 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001624 Observer.changedInstr(MI);
Roman Tereshin25cbfe62018-05-08 22:53:09 +00001625 return Legalized;
Roman Tereshin27bba442018-05-09 01:43:12 +00001626 }
Matt Arsenaultbefee402019-01-09 07:34:14 +00001627 case TargetOpcode::G_IMPLICIT_DEF: {
1628 Observer.changingInstr(MI);
1629 widenScalarDst(MI, WideTy);
1630 Observer.changedInstr(MI);
1631 return Legalized;
1632 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001633 case TargetOpcode::G_BRCOND:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001634 Observer.changingInstr(MI);
Petar Avramovic5d9b8ee2019-02-14 11:39:53 +00001635 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001636 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001637 return Legalized;
1638
1639 case TargetOpcode::G_FCMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001640 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001641 if (TypeIdx == 0)
1642 widenScalarDst(MI, WideTy);
1643 else {
1644 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1645 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
Roman Tereshin27bba442018-05-09 01:43:12 +00001646 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001647 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001648 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001649
1650 case TargetOpcode::G_ICMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001651 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001652 if (TypeIdx == 0)
1653 widenScalarDst(MI, WideTy);
1654 else {
1655 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1656 MI.getOperand(1).getPredicate()))
1657 ? TargetOpcode::G_SEXT
1658 : TargetOpcode::G_ZEXT;
1659 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1660 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1661 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001662 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001663 return Legalized;
1664
1665 case TargetOpcode::G_GEP:
Tim Northover22d82cf2016-09-15 11:02:19 +00001666 assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001667 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001668 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001669 Observer.changedInstr(MI);
Tim Northover22d82cf2016-09-15 11:02:19 +00001670 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001671
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001672 case TargetOpcode::G_PHI: {
1673 assert(TypeIdx == 0 && "Expecting only Idx 0");
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001674
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001675 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001676 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1677 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1678 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1679 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001680 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001681
1682 MachineBasicBlock &MBB = *MI.getParent();
1683 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1684 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001685 Observer.changedInstr(MI);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001686 return Legalized;
1687 }
Matt Arsenault63786292019-01-22 20:38:15 +00001688 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1689 if (TypeIdx == 0) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001690 Register VecReg = MI.getOperand(1).getReg();
Matt Arsenault63786292019-01-22 20:38:15 +00001691 LLT VecTy = MRI.getType(VecReg);
1692 Observer.changingInstr(MI);
1693
1694 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1695 WideTy.getSizeInBits()),
1696 1, TargetOpcode::G_SEXT);
1697
1698 widenScalarDst(MI, WideTy, 0);
1699 Observer.changedInstr(MI);
1700 return Legalized;
1701 }
1702
Amara Emersoncbd86d82018-10-25 14:04:54 +00001703 if (TypeIdx != 2)
1704 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001705 Observer.changingInstr(MI);
Amara Emersoncbd86d82018-10-25 14:04:54 +00001706 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001707 Observer.changedInstr(MI);
Amara Emersoncbd86d82018-10-25 14:04:54 +00001708 return Legalized;
Matt Arsenault63786292019-01-22 20:38:15 +00001709 }
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001710 case TargetOpcode::G_FADD:
1711 case TargetOpcode::G_FMUL:
1712 case TargetOpcode::G_FSUB:
1713 case TargetOpcode::G_FMA:
1714 case TargetOpcode::G_FNEG:
1715 case TargetOpcode::G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00001716 case TargetOpcode::G_FCANONICALIZE:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00001717 case TargetOpcode::G_FMINNUM:
1718 case TargetOpcode::G_FMAXNUM:
1719 case TargetOpcode::G_FMINNUM_IEEE:
1720 case TargetOpcode::G_FMAXNUM_IEEE:
1721 case TargetOpcode::G_FMINIMUM:
1722 case TargetOpcode::G_FMAXIMUM:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001723 case TargetOpcode::G_FDIV:
1724 case TargetOpcode::G_FREM:
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001725 case TargetOpcode::G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00001726 case TargetOpcode::G_FFLOOR:
Jessica Paquette7db82d72019-01-28 18:34:18 +00001727 case TargetOpcode::G_FCOS:
1728 case TargetOpcode::G_FSIN:
Jessica Paquettec49428a2019-01-28 19:53:14 +00001729 case TargetOpcode::G_FLOG10:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +00001730 case TargetOpcode::G_FLOG:
Jessica Paquette0154bd12019-01-30 21:16:04 +00001731 case TargetOpcode::G_FLOG2:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00001732 case TargetOpcode::G_FRINT:
Jessica Paquetteba557672019-04-25 16:44:40 +00001733 case TargetOpcode::G_FNEARBYINT:
Jessica Paquette22457f82019-01-30 21:03:52 +00001734 case TargetOpcode::G_FSQRT:
Jessica Paquette84bedac2019-01-30 23:46:15 +00001735 case TargetOpcode::G_FEXP:
Jessica Paquettee7941212019-04-03 16:58:32 +00001736 case TargetOpcode::G_FEXP2:
Jessica Paquettedfd87f62019-04-19 16:28:08 +00001737 case TargetOpcode::G_FPOW:
Jessica Paquette56342642019-04-23 18:20:44 +00001738 case TargetOpcode::G_INTRINSIC_TRUNC:
Jessica Paquette3cc6d1f2019-04-23 21:11:57 +00001739 case TargetOpcode::G_INTRINSIC_ROUND:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001740 assert(TypeIdx == 0);
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001741 Observer.changingInstr(MI);
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001742
1743 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1744 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1745
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001746 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1747 Observer.changedInstr(MI);
1748 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +00001749 case TargetOpcode::G_INTTOPTR:
1750 if (TypeIdx != 1)
1751 return UnableToLegalize;
1752
1753 Observer.changingInstr(MI);
1754 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1755 Observer.changedInstr(MI);
1756 return Legalized;
1757 case TargetOpcode::G_PTRTOINT:
1758 if (TypeIdx != 0)
1759 return UnableToLegalize;
1760
1761 Observer.changingInstr(MI);
1762 widenScalarDst(MI, WideTy, 0);
1763 Observer.changedInstr(MI);
1764 return Legalized;
Matt Arsenaultbd791b52019-07-08 13:48:06 +00001765 case TargetOpcode::G_BUILD_VECTOR: {
1766 Observer.changingInstr(MI);
1767
1768 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
1769 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
1770 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
1771
1772 // Avoid changing the result vector type if the source element type was
1773 // requested.
1774 if (TypeIdx == 1) {
1775 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
1776 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
1777 } else {
1778 widenScalarDst(MI, WideTy, 0);
1779 }
1780
1781 Observer.changedInstr(MI);
1782 return Legalized;
1783 }
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001784 case TargetOpcode::G_SEXT_INREG:
1785 if (TypeIdx != 0)
1786 return UnableToLegalize;
1787
1788 Observer.changingInstr(MI);
1789 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1790 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
1791 Observer.changedInstr(MI);
1792 return Legalized;
Tim Northover32335812016-08-04 18:35:11 +00001793 }
Tim Northover33b07d62016-07-22 20:03:43 +00001794}
1795
Tim Northover69fa84a2016-10-14 22:18:18 +00001796LegalizerHelper::LegalizeResult
1797LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Tim Northovercecee562016-08-26 17:46:13 +00001798 using namespace TargetOpcode;
Tim Northovercecee562016-08-26 17:46:13 +00001799 MIRBuilder.setInstr(MI);
1800
1801 switch(MI.getOpcode()) {
1802 default:
1803 return UnableToLegalize;
1804 case TargetOpcode::G_SREM:
1805 case TargetOpcode::G_UREM: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001806 Register QuotReg = MRI.createGenericVirtualRegister(Ty);
Tim Northover0f140c72016-09-09 11:46:34 +00001807 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
Tim Northovercecee562016-08-26 17:46:13 +00001808 .addDef(QuotReg)
1809 .addUse(MI.getOperand(1).getReg())
1810 .addUse(MI.getOperand(2).getReg());
1811
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001812 Register ProdReg = MRI.createGenericVirtualRegister(Ty);
Tim Northover0f140c72016-09-09 11:46:34 +00001813 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1814 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
1815 ProdReg);
Tim Northovercecee562016-08-26 17:46:13 +00001816 MI.eraseFromParent();
1817 return Legalized;
1818 }
Tim Northover0a9b2792017-02-08 21:22:15 +00001819 case TargetOpcode::G_SMULO:
1820 case TargetOpcode::G_UMULO: {
1821 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1822 // result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001823 Register Res = MI.getOperand(0).getReg();
1824 Register Overflow = MI.getOperand(1).getReg();
1825 Register LHS = MI.getOperand(2).getReg();
1826 Register RHS = MI.getOperand(3).getReg();
Tim Northover0a9b2792017-02-08 21:22:15 +00001827
1828 MIRBuilder.buildMul(Res, LHS, RHS);
1829
1830 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1831 ? TargetOpcode::G_SMULH
1832 : TargetOpcode::G_UMULH;
1833
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001834 Register HiPart = MRI.createGenericVirtualRegister(Ty);
Tim Northover0a9b2792017-02-08 21:22:15 +00001835 MIRBuilder.buildInstr(Opcode)
1836 .addDef(HiPart)
1837 .addUse(LHS)
1838 .addUse(RHS);
1839
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001840 Register Zero = MRI.createGenericVirtualRegister(Ty);
Tim Northover0a9b2792017-02-08 21:22:15 +00001841 MIRBuilder.buildConstant(Zero, 0);
Amara Emerson9de62132018-01-03 04:56:56 +00001842
1843 // For *signed* multiply, overflow is detected by checking:
1844 // (hi != (lo >> bitwidth-1))
1845 if (Opcode == TargetOpcode::G_SMULH) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001846 Register Shifted = MRI.createGenericVirtualRegister(Ty);
1847 Register ShiftAmt = MRI.createGenericVirtualRegister(Ty);
Amara Emerson9de62132018-01-03 04:56:56 +00001848 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1849 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1850 .addDef(Shifted)
1851 .addUse(Res)
1852 .addUse(ShiftAmt);
1853 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1854 } else {
1855 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1856 }
Tim Northover0a9b2792017-02-08 21:22:15 +00001857 MI.eraseFromParent();
1858 return Legalized;
1859 }
Volkan Keles5698b2a2017-03-08 18:09:14 +00001860 case TargetOpcode::G_FNEG: {
1861 // TODO: Handle vector types once we are able to
1862 // represent them.
1863 if (Ty.isVector())
1864 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001865 Register Res = MI.getOperand(0).getReg();
Volkan Keles5698b2a2017-03-08 18:09:14 +00001866 Type *ZeroTy;
Matthias Braunf1caa282017-12-15 22:22:58 +00001867 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Volkan Keles5698b2a2017-03-08 18:09:14 +00001868 switch (Ty.getSizeInBits()) {
1869 case 16:
1870 ZeroTy = Type::getHalfTy(Ctx);
1871 break;
1872 case 32:
1873 ZeroTy = Type::getFloatTy(Ctx);
1874 break;
1875 case 64:
1876 ZeroTy = Type::getDoubleTy(Ctx);
1877 break;
Amara Emersonb6ddbef2017-12-19 17:21:35 +00001878 case 128:
1879 ZeroTy = Type::getFP128Ty(Ctx);
1880 break;
Volkan Keles5698b2a2017-03-08 18:09:14 +00001881 default:
1882 llvm_unreachable("unexpected floating-point type");
1883 }
1884 ConstantFP &ZeroForNegation =
1885 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
Volkan Keles02bb1742018-02-14 19:58:36 +00001886 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001887 Register SubByReg = MI.getOperand(1).getReg();
1888 Register ZeroReg = Zero->getOperand(0).getReg();
Michael Bergd573aa02019-04-18 18:48:57 +00001889 MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
Matt Arsenault5a321b82019-06-17 23:48:43 +00001890 MI.getFlags());
Volkan Keles5698b2a2017-03-08 18:09:14 +00001891 MI.eraseFromParent();
1892 return Legalized;
1893 }
Volkan Keles225921a2017-03-10 21:25:09 +00001894 case TargetOpcode::G_FSUB: {
1895 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1896 // First, check if G_FNEG is marked as Lower. If so, we may
1897 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
Daniel Sanders9ade5592018-01-29 17:37:29 +00001898 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
Volkan Keles225921a2017-03-10 21:25:09 +00001899 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001900 Register Res = MI.getOperand(0).getReg();
1901 Register LHS = MI.getOperand(1).getReg();
1902 Register RHS = MI.getOperand(2).getReg();
1903 Register Neg = MRI.createGenericVirtualRegister(Ty);
Volkan Keles225921a2017-03-10 21:25:09 +00001904 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
Michael Bergd573aa02019-04-18 18:48:57 +00001905 MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
Volkan Keles225921a2017-03-10 21:25:09 +00001906 MI.eraseFromParent();
1907 return Legalized;
1908 }
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00001909 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001910 Register OldValRes = MI.getOperand(0).getReg();
1911 Register SuccessRes = MI.getOperand(1).getReg();
1912 Register Addr = MI.getOperand(2).getReg();
1913 Register CmpVal = MI.getOperand(3).getReg();
1914 Register NewVal = MI.getOperand(4).getReg();
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00001915 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1916 **MI.memoperands_begin());
1917 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1918 MI.eraseFromParent();
1919 return Legalized;
1920 }
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001921 case TargetOpcode::G_LOAD:
1922 case TargetOpcode::G_SEXTLOAD:
1923 case TargetOpcode::G_ZEXTLOAD: {
1924 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001925 Register DstReg = MI.getOperand(0).getReg();
1926 Register PtrReg = MI.getOperand(1).getReg();
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001927 LLT DstTy = MRI.getType(DstReg);
1928 auto &MMO = **MI.memoperands_begin();
1929
Amara Emersonc8351642019-08-02 23:44:24 +00001930 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
1931 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
1932 // This load needs splitting into power of 2 sized loads.
1933 if (DstTy.isVector())
Daniel Sanders2de9d4a2018-04-30 17:20:01 +00001934 return UnableToLegalize;
Amara Emersonc8351642019-08-02 23:44:24 +00001935 if (isPowerOf2_32(DstTy.getSizeInBits()))
1936 return UnableToLegalize; // Don't know what we're being asked to do.
1937
1938 // Our strategy here is to generate anyextending loads for the smaller
1939 // types up to next power-2 result type, and then combine the two larger
1940 // result values together, before truncating back down to the non-pow-2
1941 // type.
1942 // E.g. v1 = i24 load =>
1943 // v2 = i32 load (2 byte)
1944 // v3 = i32 load (1 byte)
1945 // v4 = i32 shl v3, 16
1946 // v5 = i32 or v4, v2
1947 // v1 = i24 trunc v5
1948 // By doing this we generate the correct truncate which should get
1949 // combined away as an artifact with a matching extend.
1950 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
1951 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
1952
1953 MachineFunction &MF = MIRBuilder.getMF();
1954 MachineMemOperand *LargeMMO =
1955 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
1956 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
1957 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
1958
1959 LLT PtrTy = MRI.getType(PtrReg);
1960 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
1961 LLT AnyExtTy = LLT::scalar(AnyExtSize);
1962 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
1963 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
1964 auto LargeLoad =
1965 MIRBuilder.buildLoad(LargeLdReg, PtrReg, *LargeMMO);
1966
1967 auto OffsetCst =
1968 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
1969 Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
1970 auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
1971 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
1972 *SmallMMO);
1973
1974 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
1975 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
1976 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
1977 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
1978 MI.eraseFromParent();
1979 return Legalized;
1980 }
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001981 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
1982 MI.eraseFromParent();
1983 return Legalized;
1984 }
1985
1986 if (DstTy.isScalar()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001987 Register TmpReg =
Amara Emersond51adf02019-04-17 22:21:05 +00001988 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001989 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1990 switch (MI.getOpcode()) {
1991 default:
1992 llvm_unreachable("Unexpected opcode");
1993 case TargetOpcode::G_LOAD:
1994 MIRBuilder.buildAnyExt(DstReg, TmpReg);
1995 break;
1996 case TargetOpcode::G_SEXTLOAD:
1997 MIRBuilder.buildSExt(DstReg, TmpReg);
1998 break;
1999 case TargetOpcode::G_ZEXTLOAD:
2000 MIRBuilder.buildZExt(DstReg, TmpReg);
2001 break;
2002 }
2003 MI.eraseFromParent();
2004 return Legalized;
2005 }
2006
2007 return UnableToLegalize;
2008 }
Amara Emersonc8351642019-08-02 23:44:24 +00002009 case TargetOpcode::G_STORE: {
2010 // Lower a non-power of 2 store into multiple pow-2 stores.
2011 // E.g. split an i24 store into an i16 store + i8 store.
2012 // We do this by first extending the stored value to the next largest power
2013 // of 2 type, and then using truncating stores to store the components.
2014 // By doing this, likewise with G_LOAD, generate an extend that can be
2015 // artifact-combined away instead of leaving behind extracts.
2016 Register SrcReg = MI.getOperand(0).getReg();
2017 Register PtrReg = MI.getOperand(1).getReg();
2018 LLT SrcTy = MRI.getType(SrcReg);
2019 MachineMemOperand &MMO = **MI.memoperands_begin();
2020 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2021 return UnableToLegalize;
2022 if (SrcTy.isVector())
2023 return UnableToLegalize;
2024 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2025 return UnableToLegalize; // Don't know what we're being asked to do.
2026
2027 // Extend to the next pow-2.
2028 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2029 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2030
2031 // Obtain the smaller value by shifting away the larger value.
2032 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2033 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2034 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2035 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2036
2037 // Generate the GEP and truncating stores.
2038 LLT PtrTy = MRI.getType(PtrReg);
2039 auto OffsetCst =
2040 MIRBuilder.buildConstant(LLT::scalar(64), LargeSplitSize / 8);
2041 Register GEPReg = MRI.createGenericVirtualRegister(PtrTy);
2042 auto SmallPtr = MIRBuilder.buildGEP(GEPReg, PtrReg, OffsetCst.getReg(0));
2043
2044 MachineFunction &MF = MIRBuilder.getMF();
2045 MachineMemOperand *LargeMMO =
2046 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2047 MachineMemOperand *SmallMMO =
2048 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2049 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2050 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2051 MI.eraseFromParent();
2052 return Legalized;
2053 }
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002054 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2055 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2056 case TargetOpcode::G_CTLZ:
2057 case TargetOpcode::G_CTTZ:
2058 case TargetOpcode::G_CTPOP:
2059 return lowerBitCount(MI, TypeIdx, Ty);
Petar Avramovicbd395692019-02-26 17:22:42 +00002060 case G_UADDO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002061 Register Res = MI.getOperand(0).getReg();
2062 Register CarryOut = MI.getOperand(1).getReg();
2063 Register LHS = MI.getOperand(2).getReg();
2064 Register RHS = MI.getOperand(3).getReg();
Petar Avramovicbd395692019-02-26 17:22:42 +00002065
2066 MIRBuilder.buildAdd(Res, LHS, RHS);
2067 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2068
2069 MI.eraseFromParent();
2070 return Legalized;
2071 }
Petar Avramovicb8276f22018-12-17 12:31:07 +00002072 case G_UADDE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002073 Register Res = MI.getOperand(0).getReg();
2074 Register CarryOut = MI.getOperand(1).getReg();
2075 Register LHS = MI.getOperand(2).getReg();
2076 Register RHS = MI.getOperand(3).getReg();
2077 Register CarryIn = MI.getOperand(4).getReg();
Petar Avramovicb8276f22018-12-17 12:31:07 +00002078
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002079 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2080 Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
Petar Avramovicb8276f22018-12-17 12:31:07 +00002081
2082 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
2083 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
2084 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
2085 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
2086
2087 MI.eraseFromParent();
2088 return Legalized;
2089 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002090 case G_USUBO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002091 Register Res = MI.getOperand(0).getReg();
2092 Register BorrowOut = MI.getOperand(1).getReg();
2093 Register LHS = MI.getOperand(2).getReg();
2094 Register RHS = MI.getOperand(3).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002095
2096 MIRBuilder.buildSub(Res, LHS, RHS);
2097 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
2098
2099 MI.eraseFromParent();
2100 return Legalized;
2101 }
2102 case G_USUBE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002103 Register Res = MI.getOperand(0).getReg();
2104 Register BorrowOut = MI.getOperand(1).getReg();
2105 Register LHS = MI.getOperand(2).getReg();
2106 Register RHS = MI.getOperand(3).getReg();
2107 Register BorrowIn = MI.getOperand(4).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002108
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002109 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2110 Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
2111 Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2112 Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002113
2114 MIRBuilder.buildSub(TmpRes, LHS, RHS);
2115 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
2116 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
2117 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
2118 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
2119 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
2120
2121 MI.eraseFromParent();
2122 return Legalized;
2123 }
Matt Arsenault02b5ca82019-05-17 23:05:13 +00002124 case G_UITOFP:
2125 return lowerUITOFP(MI, TypeIdx, Ty);
2126 case G_SITOFP:
2127 return lowerSITOFP(MI, TypeIdx, Ty);
Matt Arsenault6f74f552019-07-01 17:18:03 +00002128 case G_SMIN:
2129 case G_SMAX:
2130 case G_UMIN:
2131 case G_UMAX:
2132 return lowerMinMax(MI, TypeIdx, Ty);
Matt Arsenaultb1843e12019-07-09 23:34:29 +00002133 case G_FCOPYSIGN:
2134 return lowerFCopySign(MI, TypeIdx, Ty);
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00002135 case G_FMINNUM:
2136 case G_FMAXNUM:
2137 return lowerFMinNumMaxNum(MI);
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00002138 case G_UNMERGE_VALUES:
2139 return lowerUnmergeValues(MI);
Daniel Sanderse9a57c22019-08-09 21:11:20 +00002140 case TargetOpcode::G_SEXT_INREG: {
2141 assert(MI.getOperand(2).isImm() && "Expected immediate");
2142 int64_t SizeInBits = MI.getOperand(2).getImm();
2143
2144 Register DstReg = MI.getOperand(0).getReg();
2145 Register SrcReg = MI.getOperand(1).getReg();
2146 LLT DstTy = MRI.getType(DstReg);
2147 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
2148
2149 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
2150 MIRBuilder.buildInstr(TargetOpcode::G_SHL, {TmpRes}, {SrcReg, MIBSz->getOperand(0).getReg()});
2151 MIRBuilder.buildInstr(TargetOpcode::G_ASHR, {DstReg}, {TmpRes, MIBSz->getOperand(0).getReg()});
2152 MI.eraseFromParent();
2153 return Legalized;
2154 }
Matt Arsenault690645b2019-08-13 16:09:07 +00002155 case G_SHUFFLE_VECTOR:
2156 return lowerShuffleVector(MI);
Amara Emersone20b91c2019-08-27 19:54:27 +00002157 case G_DYN_STACKALLOC:
2158 return lowerDynStackAlloc(MI);
Tim Northovercecee562016-08-26 17:46:13 +00002159 }
2160}
2161
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002162LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
2163 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002164 SmallVector<Register, 2> DstRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002165
2166 unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002167 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002168 unsigned Size = MRI.getType(DstReg).getSizeInBits();
2169 int NumParts = Size / NarrowSize;
2170 // FIXME: Don't know how to handle the situation where the small vectors
2171 // aren't all the same size yet.
2172 if (Size % NarrowSize != 0)
2173 return UnableToLegalize;
2174
2175 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002176 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002177 MIRBuilder.buildUndef(TmpReg);
2178 DstRegs.push_back(TmpReg);
2179 }
2180
2181 if (NarrowTy.isVector())
2182 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2183 else
2184 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2185
2186 MI.eraseFromParent();
2187 return Legalized;
2188}
2189
2190LegalizerHelper::LegalizeResult
2191LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
2192 LLT NarrowTy) {
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002193 const unsigned Opc = MI.getOpcode();
2194 const unsigned NumOps = MI.getNumOperands() - 1;
2195 const unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenault3018d182019-06-28 01:47:44 +00002196 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002197 const unsigned Flags = MI.getFlags();
2198 const LLT DstTy = MRI.getType(DstReg);
2199 const unsigned Size = DstTy.getSizeInBits();
2200 const int NumParts = Size / NarrowSize;
2201 const LLT EltTy = DstTy.getElementType();
2202 const unsigned EltSize = EltTy.getSizeInBits();
2203 const unsigned BitsForNumParts = NarrowSize * NumParts;
2204
2205 // Check if we have any leftovers. If we do, then only handle the case where
2206 // the leftover is one element.
2207 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002208 return UnableToLegalize;
2209
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002210 if (BitsForNumParts != Size) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002211 Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002212 MIRBuilder.buildUndef(AccumDstReg);
2213
2214 // Handle the pieces which evenly divide into the requested type with
2215 // extract/op/insert sequence.
2216 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
2217 SmallVector<SrcOp, 4> SrcOps;
2218 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002219 Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002220 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
2221 SrcOps.push_back(PartOpReg);
2222 }
2223
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002224 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002225 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2226
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002227 Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002228 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
2229 AccumDstReg = PartInsertReg;
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002230 }
2231
2232 // Handle the remaining element sized leftover piece.
2233 SmallVector<SrcOp, 4> SrcOps;
2234 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002235 Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002236 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
2237 BitsForNumParts);
2238 SrcOps.push_back(PartOpReg);
2239 }
2240
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002241 Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002242 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2243 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
2244 MI.eraseFromParent();
2245
2246 return Legalized;
2247 }
2248
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002249 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002250
2251 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
2252
2253 if (NumOps >= 2)
2254 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
2255
2256 if (NumOps >= 3)
2257 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
2258
2259 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002260 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002261
2262 if (NumOps == 1)
2263 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
2264 else if (NumOps == 2) {
2265 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
2266 } else if (NumOps == 3) {
2267 MIRBuilder.buildInstr(Opc, {DstReg},
2268 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
2269 }
2270
2271 DstRegs.push_back(DstReg);
2272 }
2273
2274 if (NarrowTy.isVector())
2275 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2276 else
2277 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2278
2279 MI.eraseFromParent();
2280 return Legalized;
2281}
2282
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002283// Handle splitting vector operations which need to have the same number of
2284// elements in each type index, but each type index may have a different element
2285// type.
2286//
2287// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
2288// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2289// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2290//
2291// Also handles some irregular breakdown cases, e.g.
2292// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
2293// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2294// s64 = G_SHL s64, s32
2295LegalizerHelper::LegalizeResult
2296LegalizerHelper::fewerElementsVectorMultiEltType(
2297 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
2298 if (TypeIdx != 0)
2299 return UnableToLegalize;
2300
2301 const LLT NarrowTy0 = NarrowTyArg;
2302 const unsigned NewNumElts =
2303 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
2304
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002305 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002306 LLT DstTy = MRI.getType(DstReg);
2307 LLT LeftoverTy0;
2308
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002309 // All of the operands need to have the same number of elements, so if we can
2310 // determine a type breakdown for the result type, we can for all of the
2311 // source types.
Fangrui Songb251cc02019-07-12 14:58:15 +00002312 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002313 if (NumParts < 0)
2314 return UnableToLegalize;
2315
2316 SmallVector<MachineInstrBuilder, 4> NewInsts;
2317
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002318 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2319 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002320
2321 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2322 LLT LeftoverTy;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002323 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002324 LLT SrcTyI = MRI.getType(SrcReg);
2325 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
2326 LLT LeftoverTyI;
2327
2328 // Split this operand into the requested typed registers, and any leftover
2329 // required to reproduce the original type.
2330 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
2331 LeftoverRegs))
2332 return UnableToLegalize;
2333
2334 if (I == 1) {
2335 // For the first operand, create an instruction for each part and setup
2336 // the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002337 for (Register PartReg : PartRegs) {
2338 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002339 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2340 .addDef(PartDstReg)
2341 .addUse(PartReg));
2342 DstRegs.push_back(PartDstReg);
2343 }
2344
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002345 for (Register LeftoverReg : LeftoverRegs) {
2346 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002347 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2348 .addDef(PartDstReg)
2349 .addUse(LeftoverReg));
2350 LeftoverDstRegs.push_back(PartDstReg);
2351 }
2352 } else {
2353 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
2354
2355 // Add the newly created operand splits to the existing instructions. The
2356 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2357 // pieces.
2358 unsigned InstCount = 0;
2359 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
2360 NewInsts[InstCount++].addUse(PartRegs[J]);
2361 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
2362 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
2363 }
2364
2365 PartRegs.clear();
2366 LeftoverRegs.clear();
2367 }
2368
2369 // Insert the newly built operations and rebuild the result register.
2370 for (auto &MIB : NewInsts)
2371 MIRBuilder.insertInstr(MIB);
2372
2373 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
2374
2375 MI.eraseFromParent();
2376 return Legalized;
2377}
2378
Tim Northover69fa84a2016-10-14 22:18:18 +00002379LegalizerHelper::LegalizeResult
Matt Arsenaultca676342019-01-25 02:36:32 +00002380LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
2381 LLT NarrowTy) {
2382 if (TypeIdx != 0)
2383 return UnableToLegalize;
2384
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002385 Register DstReg = MI.getOperand(0).getReg();
2386 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenaultca676342019-01-25 02:36:32 +00002387 LLT DstTy = MRI.getType(DstReg);
2388 LLT SrcTy = MRI.getType(SrcReg);
2389
2390 LLT NarrowTy0 = NarrowTy;
2391 LLT NarrowTy1;
2392 unsigned NumParts;
2393
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002394 if (NarrowTy.isVector()) {
Matt Arsenaultca676342019-01-25 02:36:32 +00002395 // Uneven breakdown not handled.
2396 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
2397 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
2398 return UnableToLegalize;
2399
2400 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002401 } else {
2402 NumParts = DstTy.getNumElements();
2403 NarrowTy1 = SrcTy.getElementType();
Matt Arsenaultca676342019-01-25 02:36:32 +00002404 }
2405
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002406 SmallVector<Register, 4> SrcRegs, DstRegs;
Matt Arsenaultca676342019-01-25 02:36:32 +00002407 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
2408
2409 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002410 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultca676342019-01-25 02:36:32 +00002411 MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
2412 .addDef(DstReg)
2413 .addUse(SrcRegs[I]);
2414
2415 NewInst->setFlags(MI.getFlags());
2416 DstRegs.push_back(DstReg);
2417 }
2418
2419 if (NarrowTy.isVector())
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002420 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002421 else
2422 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2423
2424 MI.eraseFromParent();
2425 return Legalized;
2426}
2427
2428LegalizerHelper::LegalizeResult
2429LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
2430 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002431 Register DstReg = MI.getOperand(0).getReg();
2432 Register Src0Reg = MI.getOperand(2).getReg();
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002433 LLT DstTy = MRI.getType(DstReg);
2434 LLT SrcTy = MRI.getType(Src0Reg);
2435
2436 unsigned NumParts;
2437 LLT NarrowTy0, NarrowTy1;
2438
2439 if (TypeIdx == 0) {
2440 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2441 unsigned OldElts = DstTy.getNumElements();
2442
2443 NarrowTy0 = NarrowTy;
2444 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
2445 NarrowTy1 = NarrowTy.isVector() ?
2446 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
2447 SrcTy.getElementType();
2448
2449 } else {
2450 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2451 unsigned OldElts = SrcTy.getNumElements();
2452
2453 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2454 NarrowTy.getNumElements();
2455 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2456 DstTy.getScalarSizeInBits());
2457 NarrowTy1 = NarrowTy;
2458 }
2459
2460 // FIXME: Don't know how to handle the situation where the small vectors
2461 // aren't all the same size yet.
2462 if (NarrowTy1.isVector() &&
2463 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2464 return UnableToLegalize;
2465
2466 CmpInst::Predicate Pred
2467 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2468
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002469 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002470 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2471 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2472
2473 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002474 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002475 DstRegs.push_back(DstReg);
2476
2477 if (MI.getOpcode() == TargetOpcode::G_ICMP)
2478 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2479 else {
2480 MachineInstr *NewCmp
2481 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2482 NewCmp->setFlags(MI.getFlags());
2483 }
2484 }
2485
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002486 if (NarrowTy1.isVector())
Matt Arsenaultca676342019-01-25 02:36:32 +00002487 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2488 else
2489 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2490
2491 MI.eraseFromParent();
2492 return Legalized;
2493}
2494
2495LegalizerHelper::LegalizeResult
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002496LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
2497 LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002498 Register DstReg = MI.getOperand(0).getReg();
2499 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002500
2501 unsigned NumParts = 0;
2502 LLT NarrowTy0, NarrowTy1;
2503
2504 LLT DstTy = MRI.getType(DstReg);
2505 LLT CondTy = MRI.getType(CondReg);
2506 unsigned Size = DstTy.getSizeInBits();
2507
2508 assert(TypeIdx == 0 || CondTy.isVector());
2509
2510 if (TypeIdx == 0) {
2511 NarrowTy0 = NarrowTy;
2512 NarrowTy1 = CondTy;
2513
2514 unsigned NarrowSize = NarrowTy0.getSizeInBits();
2515 // FIXME: Don't know how to handle the situation where the small vectors
2516 // aren't all the same size yet.
2517 if (Size % NarrowSize != 0)
2518 return UnableToLegalize;
2519
2520 NumParts = Size / NarrowSize;
2521
2522 // Need to break down the condition type
2523 if (CondTy.isVector()) {
2524 if (CondTy.getNumElements() == NumParts)
2525 NarrowTy1 = CondTy.getElementType();
2526 else
2527 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2528 CondTy.getScalarSizeInBits());
2529 }
2530 } else {
2531 NumParts = CondTy.getNumElements();
2532 if (NarrowTy.isVector()) {
2533 // TODO: Handle uneven breakdown.
2534 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2535 return UnableToLegalize;
2536
2537 return UnableToLegalize;
2538 } else {
2539 NarrowTy0 = DstTy.getElementType();
2540 NarrowTy1 = NarrowTy;
2541 }
2542 }
2543
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002544 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002545 if (CondTy.isVector())
2546 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2547
2548 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2549 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2550
2551 for (unsigned i = 0; i < NumParts; ++i) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002552 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002553 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2554 Src1Regs[i], Src2Regs[i]);
2555 DstRegs.push_back(DstReg);
2556 }
2557
2558 if (NarrowTy0.isVector())
2559 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2560 else
2561 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2562
2563 MI.eraseFromParent();
2564 return Legalized;
2565}
2566
2567LegalizerHelper::LegalizeResult
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002568LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2569 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002570 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002571 LLT PhiTy = MRI.getType(DstReg);
2572 LLT LeftoverTy;
2573
2574 // All of the operands need to have the same number of elements, so if we can
2575 // determine a type breakdown for the result type, we can for all of the
2576 // source types.
2577 int NumParts, NumLeftover;
2578 std::tie(NumParts, NumLeftover)
2579 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2580 if (NumParts < 0)
2581 return UnableToLegalize;
2582
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002583 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002584 SmallVector<MachineInstrBuilder, 4> NewInsts;
2585
2586 const int TotalNumParts = NumParts + NumLeftover;
2587
2588 // Insert the new phis in the result block first.
2589 for (int I = 0; I != TotalNumParts; ++I) {
2590 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002591 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002592 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2593 .addDef(PartDstReg));
2594 if (I < NumParts)
2595 DstRegs.push_back(PartDstReg);
2596 else
2597 LeftoverDstRegs.push_back(PartDstReg);
2598 }
2599
2600 MachineBasicBlock *MBB = MI.getParent();
2601 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2602 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2603
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002604 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002605
2606 // Insert code to extract the incoming values in each predecessor block.
2607 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2608 PartRegs.clear();
2609 LeftoverRegs.clear();
2610
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002611 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002612 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2613 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2614
2615 LLT Unused;
2616 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2617 LeftoverRegs))
2618 return UnableToLegalize;
2619
2620 // Add the newly created operand splits to the existing instructions. The
2621 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2622 // pieces.
2623 for (int J = 0; J != TotalNumParts; ++J) {
2624 MachineInstrBuilder MIB = NewInsts[J];
2625 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2626 MIB.addMBB(&OpMBB);
2627 }
2628 }
2629
2630 MI.eraseFromParent();
2631 return Legalized;
2632}
2633
2634LegalizerHelper::LegalizeResult
Matt Arsenault28215ca2019-08-13 16:26:28 +00002635LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
2636 unsigned TypeIdx,
2637 LLT NarrowTy) {
2638 if (TypeIdx != 1)
2639 return UnableToLegalize;
2640
2641 const int NumDst = MI.getNumOperands() - 1;
2642 const Register SrcReg = MI.getOperand(NumDst).getReg();
2643 LLT SrcTy = MRI.getType(SrcReg);
2644
2645 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2646
2647 // TODO: Create sequence of extracts.
2648 if (DstTy == NarrowTy)
2649 return UnableToLegalize;
2650
2651 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
2652 if (DstTy == GCDTy) {
2653 // This would just be a copy of the same unmerge.
2654 // TODO: Create extracts, pad with undef and create intermediate merges.
2655 return UnableToLegalize;
2656 }
2657
2658 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
2659 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2660 const int PartsPerUnmerge = NumDst / NumUnmerge;
2661
2662 for (int I = 0; I != NumUnmerge; ++I) {
2663 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
2664
2665 for (int J = 0; J != PartsPerUnmerge; ++J)
2666 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
2667 MIB.addUse(Unmerge.getReg(I));
2668 }
2669
2670 MI.eraseFromParent();
2671 return Legalized;
2672}
2673
2674LegalizerHelper::LegalizeResult
Matt Arsenault7f09fd62019-02-05 00:26:12 +00002675LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2676 LLT NarrowTy) {
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002677 // FIXME: Don't know how to handle secondary types yet.
2678 if (TypeIdx != 0)
2679 return UnableToLegalize;
2680
Matt Arsenaultcfca2a72019-01-27 22:36:24 +00002681 MachineMemOperand *MMO = *MI.memoperands_begin();
2682
2683 // This implementation doesn't work for atomics. Give up instead of doing
2684 // something invalid.
2685 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2686 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
2687 return UnableToLegalize;
2688
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002689 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002690 Register ValReg = MI.getOperand(0).getReg();
2691 Register AddrReg = MI.getOperand(1).getReg();
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002692 LLT ValTy = MRI.getType(ValReg);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002693
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002694 int NumParts = -1;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002695 int NumLeftover = -1;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002696 LLT LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002697 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002698 if (IsLoad) {
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002699 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002700 } else {
2701 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002702 NarrowLeftoverRegs)) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002703 NumParts = NarrowRegs.size();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002704 NumLeftover = NarrowLeftoverRegs.size();
2705 }
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002706 }
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002707
2708 if (NumParts == -1)
2709 return UnableToLegalize;
2710
2711 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2712
2713 unsigned TotalSize = ValTy.getSizeInBits();
2714
2715 // Split the load/store into PartTy sized pieces starting at Offset. If this
2716 // is a load, return the new registers in ValRegs. For a store, each elements
2717 // of ValRegs should be PartTy. Returns the next offset that needs to be
2718 // handled.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002719 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002720 unsigned Offset) -> unsigned {
2721 MachineFunction &MF = MIRBuilder.getMF();
2722 unsigned PartSize = PartTy.getSizeInBits();
2723 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2724 Offset += PartSize, ++Idx) {
2725 unsigned ByteSize = PartSize / 8;
2726 unsigned ByteOffset = Offset / 8;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002727 Register NewAddrReg;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002728
2729 MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2730
2731 MachineMemOperand *NewMMO =
2732 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2733
2734 if (IsLoad) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002735 Register Dst = MRI.createGenericVirtualRegister(PartTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002736 ValRegs.push_back(Dst);
2737 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2738 } else {
2739 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2740 }
2741 }
2742
2743 return Offset;
2744 };
2745
2746 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2747
2748 // Handle the rest of the register if this isn't an even type breakdown.
2749 if (LeftoverTy.isValid())
2750 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2751
2752 if (IsLoad) {
2753 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2754 LeftoverTy, NarrowLeftoverRegs);
2755 }
2756
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002757 MI.eraseFromParent();
2758 return Legalized;
2759}
2760
2761LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00002762LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
2763 LLT NarrowTy) {
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002764 using namespace TargetOpcode;
Volkan Keles574d7372018-12-14 22:11:20 +00002765
2766 MIRBuilder.setInstr(MI);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002767 switch (MI.getOpcode()) {
2768 case G_IMPLICIT_DEF:
2769 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2770 case G_AND:
2771 case G_OR:
2772 case G_XOR:
2773 case G_ADD:
2774 case G_SUB:
2775 case G_MUL:
2776 case G_SMULH:
2777 case G_UMULH:
2778 case G_FADD:
2779 case G_FMUL:
2780 case G_FSUB:
2781 case G_FNEG:
2782 case G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00002783 case G_FCANONICALIZE:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002784 case G_FDIV:
2785 case G_FREM:
2786 case G_FMA:
2787 case G_FPOW:
2788 case G_FEXP:
2789 case G_FEXP2:
2790 case G_FLOG:
2791 case G_FLOG2:
2792 case G_FLOG10:
Jessica Paquetteba557672019-04-25 16:44:40 +00002793 case G_FNEARBYINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002794 case G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00002795 case G_FFLOOR:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00002796 case G_FRINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002797 case G_INTRINSIC_ROUND:
2798 case G_INTRINSIC_TRUNC:
Jessica Paquette7db82d72019-01-28 18:34:18 +00002799 case G_FCOS:
2800 case G_FSIN:
Jessica Paquette22457f82019-01-30 21:03:52 +00002801 case G_FSQRT:
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00002802 case G_BSWAP:
Amara Emersonae878da2019-04-10 23:06:08 +00002803 case G_SDIV:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00002804 case G_SMIN:
2805 case G_SMAX:
2806 case G_UMIN:
2807 case G_UMAX:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00002808 case G_FMINNUM:
2809 case G_FMAXNUM:
2810 case G_FMINNUM_IEEE:
2811 case G_FMAXNUM_IEEE:
2812 case G_FMINIMUM:
2813 case G_FMAXIMUM:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002814 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002815 case G_SHL:
2816 case G_LSHR:
2817 case G_ASHR:
Matt Arsenault75e30c42019-02-20 16:42:52 +00002818 case G_CTLZ:
2819 case G_CTLZ_ZERO_UNDEF:
2820 case G_CTTZ:
2821 case G_CTTZ_ZERO_UNDEF:
2822 case G_CTPOP:
Matt Arsenault1448f562019-05-17 12:19:52 +00002823 case G_FCOPYSIGN:
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002824 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002825 case G_ZEXT:
2826 case G_SEXT:
2827 case G_ANYEXT:
2828 case G_FPEXT:
2829 case G_FPTRUNC:
2830 case G_SITOFP:
2831 case G_UITOFP:
2832 case G_FPTOSI:
2833 case G_FPTOUI:
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002834 case G_INTTOPTR:
2835 case G_PTRTOINT:
Matt Arsenaulta8b43392019-02-08 02:40:47 +00002836 case G_ADDRSPACE_CAST:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002837 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2838 case G_ICMP:
2839 case G_FCMP:
2840 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002841 case G_SELECT:
2842 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002843 case G_PHI:
2844 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
Matt Arsenault28215ca2019-08-13 16:26:28 +00002845 case G_UNMERGE_VALUES:
2846 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002847 case G_LOAD:
2848 case G_STORE:
Matt Arsenault7f09fd62019-02-05 00:26:12 +00002849 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Tim Northover33b07d62016-07-22 20:03:43 +00002850 default:
2851 return UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +00002852 }
2853}
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002854
2855LegalizerHelper::LegalizeResult
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002856LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
2857 const LLT HalfTy, const LLT AmtTy) {
2858
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002859 Register InL = MRI.createGenericVirtualRegister(HalfTy);
2860 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002861 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2862
2863 if (Amt.isNullValue()) {
2864 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
2865 MI.eraseFromParent();
2866 return Legalized;
2867 }
2868
2869 LLT NVT = HalfTy;
2870 unsigned NVTBits = HalfTy.getSizeInBits();
2871 unsigned VTBits = 2 * NVTBits;
2872
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002873 SrcOp Lo(Register(0)), Hi(Register(0));
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002874 if (MI.getOpcode() == TargetOpcode::G_SHL) {
2875 if (Amt.ugt(VTBits)) {
2876 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2877 } else if (Amt.ugt(NVTBits)) {
2878 Lo = MIRBuilder.buildConstant(NVT, 0);
2879 Hi = MIRBuilder.buildShl(NVT, InL,
2880 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2881 } else if (Amt == NVTBits) {
2882 Lo = MIRBuilder.buildConstant(NVT, 0);
2883 Hi = InL;
2884 } else {
2885 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
Matt Arsenaulte98cab12019-02-07 20:44:08 +00002886 auto OrLHS =
2887 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
2888 auto OrRHS = MIRBuilder.buildLShr(
2889 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2890 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002891 }
2892 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2893 if (Amt.ugt(VTBits)) {
2894 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2895 } else if (Amt.ugt(NVTBits)) {
2896 Lo = MIRBuilder.buildLShr(NVT, InH,
2897 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2898 Hi = MIRBuilder.buildConstant(NVT, 0);
2899 } else if (Amt == NVTBits) {
2900 Lo = InH;
2901 Hi = MIRBuilder.buildConstant(NVT, 0);
2902 } else {
2903 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2904
2905 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2906 auto OrRHS = MIRBuilder.buildShl(
2907 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2908
2909 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2910 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
2911 }
2912 } else {
2913 if (Amt.ugt(VTBits)) {
2914 Hi = Lo = MIRBuilder.buildAShr(
2915 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2916 } else if (Amt.ugt(NVTBits)) {
2917 Lo = MIRBuilder.buildAShr(NVT, InH,
2918 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2919 Hi = MIRBuilder.buildAShr(NVT, InH,
2920 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2921 } else if (Amt == NVTBits) {
2922 Lo = InH;
2923 Hi = MIRBuilder.buildAShr(NVT, InH,
2924 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2925 } else {
2926 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2927
2928 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2929 auto OrRHS = MIRBuilder.buildShl(
2930 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2931
2932 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2933 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
2934 }
2935 }
2936
2937 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
2938 MI.eraseFromParent();
2939
2940 return Legalized;
2941}
2942
2943// TODO: Optimize if constant shift amount.
2944LegalizerHelper::LegalizeResult
2945LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
2946 LLT RequestedTy) {
2947 if (TypeIdx == 1) {
2948 Observer.changingInstr(MI);
2949 narrowScalarSrc(MI, RequestedTy, 2);
2950 Observer.changedInstr(MI);
2951 return Legalized;
2952 }
2953
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002954 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002955 LLT DstTy = MRI.getType(DstReg);
2956 if (DstTy.isVector())
2957 return UnableToLegalize;
2958
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002959 Register Amt = MI.getOperand(2).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002960 LLT ShiftAmtTy = MRI.getType(Amt);
2961 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
2962 if (DstEltSize % 2 != 0)
2963 return UnableToLegalize;
2964
2965 // Ignore the input type. We can only go to exactly half the size of the
2966 // input. If that isn't small enough, the resulting pieces will be further
2967 // legalized.
2968 const unsigned NewBitSize = DstEltSize / 2;
2969 const LLT HalfTy = LLT::scalar(NewBitSize);
2970 const LLT CondTy = LLT::scalar(1);
2971
2972 if (const MachineInstr *KShiftAmt =
2973 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
2974 return narrowScalarShiftByConstant(
2975 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
2976 }
2977
2978 // TODO: Expand with known bits.
2979
2980 // Handle the fully general expansion by an unknown amount.
2981 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
2982
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002983 Register InL = MRI.createGenericVirtualRegister(HalfTy);
2984 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002985 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2986
2987 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
2988 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
2989
2990 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
2991 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
2992 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
2993
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002994 Register ResultRegs[2];
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002995 switch (MI.getOpcode()) {
2996 case TargetOpcode::G_SHL: {
2997 // Short: ShAmt < NewBitSize
Petar Avramovicd568ed42019-08-27 14:22:32 +00002998 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002999
Petar Avramovicd568ed42019-08-27 14:22:32 +00003000 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
3001 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
3002 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003003
3004 // Long: ShAmt >= NewBitSize
3005 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
3006 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
3007
3008 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
3009 auto Hi = MIRBuilder.buildSelect(
3010 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
3011
3012 ResultRegs[0] = Lo.getReg(0);
3013 ResultRegs[1] = Hi.getReg(0);
3014 break;
3015 }
Petar Avramovica3932382019-08-27 14:33:05 +00003016 case TargetOpcode::G_LSHR:
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003017 case TargetOpcode::G_ASHR: {
3018 // Short: ShAmt < NewBitSize
Petar Avramovica3932382019-08-27 14:33:05 +00003019 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003020
Petar Avramovicd568ed42019-08-27 14:22:32 +00003021 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
3022 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
3023 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003024
3025 // Long: ShAmt >= NewBitSize
Petar Avramovica3932382019-08-27 14:33:05 +00003026 MachineInstrBuilder HiL;
3027 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3028 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
3029 } else {
3030 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
3031 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
3032 }
3033 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
3034 {InH, AmtExcess}); // Lo from Hi part.
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003035
3036 auto Lo = MIRBuilder.buildSelect(
3037 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
3038
3039 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
3040
3041 ResultRegs[0] = Lo.getReg(0);
3042 ResultRegs[1] = Hi.getReg(0);
3043 break;
3044 }
3045 default:
3046 llvm_unreachable("not a shift");
3047 }
3048
3049 MIRBuilder.buildMerge(DstReg, ResultRegs);
3050 MI.eraseFromParent();
3051 return Legalized;
3052}
3053
3054LegalizerHelper::LegalizeResult
Matt Arsenault72bcf152019-02-28 00:01:05 +00003055LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3056 LLT MoreTy) {
3057 assert(TypeIdx == 0 && "Expecting only Idx 0");
3058
3059 Observer.changingInstr(MI);
3060 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3061 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3062 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3063 moreElementsVectorSrc(MI, MoreTy, I);
3064 }
3065
3066 MachineBasicBlock &MBB = *MI.getParent();
3067 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
3068 moreElementsVectorDst(MI, MoreTy, 0);
3069 Observer.changedInstr(MI);
3070 return Legalized;
3071}
3072
3073LegalizerHelper::LegalizeResult
Matt Arsenault18ec3822019-02-11 22:00:39 +00003074LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
3075 LLT MoreTy) {
3076 MIRBuilder.setInstr(MI);
3077 unsigned Opc = MI.getOpcode();
3078 switch (Opc) {
Matt Arsenault7bedceb2019-08-01 01:44:22 +00003079 case TargetOpcode::G_IMPLICIT_DEF:
3080 case TargetOpcode::G_LOAD: {
3081 if (TypeIdx != 0)
3082 return UnableToLegalize;
Matt Arsenault18ec3822019-02-11 22:00:39 +00003083 Observer.changingInstr(MI);
3084 moreElementsVectorDst(MI, MoreTy, 0);
3085 Observer.changedInstr(MI);
3086 return Legalized;
3087 }
Matt Arsenault7bedceb2019-08-01 01:44:22 +00003088 case TargetOpcode::G_STORE:
3089 if (TypeIdx != 0)
3090 return UnableToLegalize;
3091 Observer.changingInstr(MI);
3092 moreElementsVectorSrc(MI, MoreTy, 0);
3093 Observer.changedInstr(MI);
3094 return Legalized;
Matt Arsenault26b7e852019-02-19 16:30:19 +00003095 case TargetOpcode::G_AND:
3096 case TargetOpcode::G_OR:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00003097 case TargetOpcode::G_XOR:
3098 case TargetOpcode::G_SMIN:
3099 case TargetOpcode::G_SMAX:
3100 case TargetOpcode::G_UMIN:
3101 case TargetOpcode::G_UMAX: {
Matt Arsenault26b7e852019-02-19 16:30:19 +00003102 Observer.changingInstr(MI);
3103 moreElementsVectorSrc(MI, MoreTy, 1);
3104 moreElementsVectorSrc(MI, MoreTy, 2);
3105 moreElementsVectorDst(MI, MoreTy, 0);
3106 Observer.changedInstr(MI);
3107 return Legalized;
3108 }
Matt Arsenault4d884272019-02-19 16:44:22 +00003109 case TargetOpcode::G_EXTRACT:
3110 if (TypeIdx != 1)
3111 return UnableToLegalize;
3112 Observer.changingInstr(MI);
3113 moreElementsVectorSrc(MI, MoreTy, 1);
3114 Observer.changedInstr(MI);
3115 return Legalized;
Matt Arsenaultc4d07552019-02-20 16:11:22 +00003116 case TargetOpcode::G_INSERT:
3117 if (TypeIdx != 0)
3118 return UnableToLegalize;
3119 Observer.changingInstr(MI);
3120 moreElementsVectorSrc(MI, MoreTy, 1);
3121 moreElementsVectorDst(MI, MoreTy, 0);
3122 Observer.changedInstr(MI);
3123 return Legalized;
Matt Arsenaultb4c95b32019-02-19 17:03:09 +00003124 case TargetOpcode::G_SELECT:
3125 if (TypeIdx != 0)
3126 return UnableToLegalize;
3127 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
3128 return UnableToLegalize;
3129
3130 Observer.changingInstr(MI);
3131 moreElementsVectorSrc(MI, MoreTy, 2);
3132 moreElementsVectorSrc(MI, MoreTy, 3);
3133 moreElementsVectorDst(MI, MoreTy, 0);
3134 Observer.changedInstr(MI);
3135 return Legalized;
Matt Arsenault954a0122019-08-21 16:59:10 +00003136 case TargetOpcode::G_UNMERGE_VALUES: {
3137 if (TypeIdx != 1)
3138 return UnableToLegalize;
3139
3140 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3141 int NumDst = MI.getNumOperands() - 1;
3142 moreElementsVectorSrc(MI, MoreTy, NumDst);
3143
3144 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3145 for (int I = 0; I != NumDst; ++I)
3146 MIB.addDef(MI.getOperand(I).getReg());
3147
3148 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
3149 for (int I = NumDst; I != NewNumDst; ++I)
3150 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
3151
3152 MIB.addUse(MI.getOperand(NumDst).getReg());
3153 MI.eraseFromParent();
3154 return Legalized;
3155 }
Matt Arsenault72bcf152019-02-28 00:01:05 +00003156 case TargetOpcode::G_PHI:
3157 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +00003158 default:
3159 return UnableToLegalize;
3160 }
3161}
3162
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003163void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
3164 ArrayRef<Register> Src1Regs,
3165 ArrayRef<Register> Src2Regs,
Petar Avramovic0b17e592019-03-11 10:00:17 +00003166 LLT NarrowTy) {
3167 MachineIRBuilder &B = MIRBuilder;
3168 unsigned SrcParts = Src1Regs.size();
3169 unsigned DstParts = DstRegs.size();
3170
3171 unsigned DstIdx = 0; // Low bits of the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003172 Register FactorSum =
Petar Avramovic0b17e592019-03-11 10:00:17 +00003173 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
3174 DstRegs[DstIdx] = FactorSum;
3175
3176 unsigned CarrySumPrevDstIdx;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003177 SmallVector<Register, 4> Factors;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003178
3179 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
3180 // Collect low parts of muls for DstIdx.
3181 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
3182 i <= std::min(DstIdx, SrcParts - 1); ++i) {
3183 MachineInstrBuilder Mul =
3184 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
3185 Factors.push_back(Mul.getReg(0));
3186 }
3187 // Collect high parts of muls from previous DstIdx.
3188 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
3189 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
3190 MachineInstrBuilder Umulh =
3191 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
3192 Factors.push_back(Umulh.getReg(0));
3193 }
3194 // Add CarrySum from additons calculated for previous DstIdx.
3195 if (DstIdx != 1) {
3196 Factors.push_back(CarrySumPrevDstIdx);
3197 }
3198
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003199 Register CarrySum;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003200 // Add all factors and accumulate all carries into CarrySum.
3201 if (DstIdx != DstParts - 1) {
3202 MachineInstrBuilder Uaddo =
3203 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
3204 FactorSum = Uaddo.getReg(0);
3205 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
3206 for (unsigned i = 2; i < Factors.size(); ++i) {
3207 MachineInstrBuilder Uaddo =
3208 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
3209 FactorSum = Uaddo.getReg(0);
3210 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
3211 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
3212 }
3213 } else {
3214 // Since value for the next index is not calculated, neither is CarrySum.
3215 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
3216 for (unsigned i = 2; i < Factors.size(); ++i)
3217 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
3218 }
3219
3220 CarrySumPrevDstIdx = CarrySum;
3221 DstRegs[DstIdx] = FactorSum;
3222 Factors.clear();
3223 }
3224}
3225
Matt Arsenault18ec3822019-02-11 22:00:39 +00003226LegalizerHelper::LegalizeResult
Petar Avramovic0b17e592019-03-11 10:00:17 +00003227LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003228 Register DstReg = MI.getOperand(0).getReg();
3229 Register Src1 = MI.getOperand(1).getReg();
3230 Register Src2 = MI.getOperand(2).getReg();
Petar Avramovic0b17e592019-03-11 10:00:17 +00003231
Matt Arsenault211e89d2019-01-27 00:52:51 +00003232 LLT Ty = MRI.getType(DstReg);
3233 if (Ty.isVector())
3234 return UnableToLegalize;
3235
Petar Avramovic0b17e592019-03-11 10:00:17 +00003236 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
3237 unsigned DstSize = Ty.getSizeInBits();
3238 unsigned NarrowSize = NarrowTy.getSizeInBits();
3239 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
Matt Arsenault211e89d2019-01-27 00:52:51 +00003240 return UnableToLegalize;
3241
Petar Avramovic0b17e592019-03-11 10:00:17 +00003242 unsigned NumDstParts = DstSize / NarrowSize;
3243 unsigned NumSrcParts = SrcSize / NarrowSize;
Petar Avramovic5229f472019-03-11 10:08:44 +00003244 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
3245 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003246
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003247 SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003248 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
3249 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
Petar Avramovic5229f472019-03-11 10:08:44 +00003250 DstTmpRegs.resize(DstTmpParts);
3251 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003252
Petar Avramovic5229f472019-03-11 10:08:44 +00003253 // Take only high half of registers if this is high mul.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003254 ArrayRef<Register> DstRegs(
Petar Avramovic5229f472019-03-11 10:08:44 +00003255 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
Petar Avramovic0b17e592019-03-11 10:00:17 +00003256 MIRBuilder.buildMerge(DstReg, DstRegs);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003257 MI.eraseFromParent();
3258 return Legalized;
3259}
3260
Matt Arsenault1cf713662019-02-12 14:54:52 +00003261LegalizerHelper::LegalizeResult
3262LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
3263 LLT NarrowTy) {
3264 if (TypeIdx != 1)
3265 return UnableToLegalize;
3266
3267 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3268
3269 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3270 // FIXME: add support for when SizeOp1 isn't an exact multiple of
3271 // NarrowSize.
3272 if (SizeOp1 % NarrowSize != 0)
3273 return UnableToLegalize;
3274 int NumParts = SizeOp1 / NarrowSize;
3275
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003276 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003277 SmallVector<uint64_t, 2> Indexes;
3278 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3279
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003280 Register OpReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003281 uint64_t OpStart = MI.getOperand(2).getImm();
3282 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3283 for (int i = 0; i < NumParts; ++i) {
3284 unsigned SrcStart = i * NarrowSize;
3285
3286 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
3287 // No part of the extract uses this subregister, ignore it.
3288 continue;
3289 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3290 // The entire subregister is extracted, forward the value.
3291 DstRegs.push_back(SrcRegs[i]);
3292 continue;
3293 }
3294
3295 // OpSegStart is where this destination segment would start in OpReg if it
3296 // extended infinitely in both directions.
3297 int64_t ExtractOffset;
3298 uint64_t SegSize;
3299 if (OpStart < SrcStart) {
3300 ExtractOffset = 0;
3301 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
3302 } else {
3303 ExtractOffset = OpStart - SrcStart;
3304 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
3305 }
3306
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003307 Register SegReg = SrcRegs[i];
Matt Arsenault1cf713662019-02-12 14:54:52 +00003308 if (ExtractOffset != 0 || SegSize != NarrowSize) {
3309 // A genuine extract is needed.
3310 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3311 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
3312 }
3313
3314 DstRegs.push_back(SegReg);
3315 }
3316
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003317 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003318 if(MRI.getType(DstReg).isVector())
3319 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3320 else
3321 MIRBuilder.buildMerge(DstReg, DstRegs);
3322 MI.eraseFromParent();
3323 return Legalized;
3324}
3325
3326LegalizerHelper::LegalizeResult
3327LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
3328 LLT NarrowTy) {
3329 // FIXME: Don't know how to handle secondary types yet.
3330 if (TypeIdx != 0)
3331 return UnableToLegalize;
3332
3333 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3334 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3335
3336 // FIXME: add support for when SizeOp0 isn't an exact multiple of
3337 // NarrowSize.
3338 if (SizeOp0 % NarrowSize != 0)
3339 return UnableToLegalize;
3340
3341 int NumParts = SizeOp0 / NarrowSize;
3342
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003343 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003344 SmallVector<uint64_t, 2> Indexes;
3345 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3346
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003347 Register OpReg = MI.getOperand(2).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003348 uint64_t OpStart = MI.getOperand(3).getImm();
3349 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3350 for (int i = 0; i < NumParts; ++i) {
3351 unsigned DstStart = i * NarrowSize;
3352
3353 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
3354 // No part of the insert affects this subregister, forward the original.
3355 DstRegs.push_back(SrcRegs[i]);
3356 continue;
3357 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3358 // The entire subregister is defined by this insert, forward the new
3359 // value.
3360 DstRegs.push_back(OpReg);
3361 continue;
3362 }
3363
3364 // OpSegStart is where this destination segment would start in OpReg if it
3365 // extended infinitely in both directions.
3366 int64_t ExtractOffset, InsertOffset;
3367 uint64_t SegSize;
3368 if (OpStart < DstStart) {
3369 InsertOffset = 0;
3370 ExtractOffset = DstStart - OpStart;
3371 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
3372 } else {
3373 InsertOffset = OpStart - DstStart;
3374 ExtractOffset = 0;
3375 SegSize =
3376 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
3377 }
3378
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003379 Register SegReg = OpReg;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003380 if (ExtractOffset != 0 || SegSize != OpSize) {
3381 // A genuine extract is needed.
3382 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3383 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
3384 }
3385
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003386 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +00003387 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
3388 DstRegs.push_back(DstReg);
3389 }
3390
3391 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003392 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003393 if(MRI.getType(DstReg).isVector())
3394 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3395 else
3396 MIRBuilder.buildMerge(DstReg, DstRegs);
3397 MI.eraseFromParent();
3398 return Legalized;
3399}
3400
Matt Arsenault211e89d2019-01-27 00:52:51 +00003401LegalizerHelper::LegalizeResult
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003402LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
3403 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003404 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003405 LLT DstTy = MRI.getType(DstReg);
3406
3407 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
3408
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003409 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3410 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
3411 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003412 LLT LeftoverTy;
3413 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
3414 Src0Regs, Src0LeftoverRegs))
3415 return UnableToLegalize;
3416
3417 LLT Unused;
3418 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
3419 Src1Regs, Src1LeftoverRegs))
3420 llvm_unreachable("inconsistent extractParts result");
3421
3422 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3423 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
3424 {Src0Regs[I], Src1Regs[I]});
3425 DstRegs.push_back(Inst->getOperand(0).getReg());
3426 }
3427
3428 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3429 auto Inst = MIRBuilder.buildInstr(
3430 MI.getOpcode(),
3431 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
3432 DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
3433 }
3434
3435 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3436 LeftoverTy, DstLeftoverRegs);
3437
3438 MI.eraseFromParent();
3439 return Legalized;
3440}
3441
3442LegalizerHelper::LegalizeResult
Matt Arsenault81511e52019-02-05 00:13:44 +00003443LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
3444 LLT NarrowTy) {
3445 if (TypeIdx != 0)
3446 return UnableToLegalize;
3447
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003448 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00003449 LLT CondTy = MRI.getType(CondReg);
3450 if (CondTy.isVector()) // TODO: Handle vselect
3451 return UnableToLegalize;
3452
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003453 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00003454 LLT DstTy = MRI.getType(DstReg);
3455
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003456 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3457 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3458 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
Matt Arsenault81511e52019-02-05 00:13:44 +00003459 LLT LeftoverTy;
3460 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
3461 Src1Regs, Src1LeftoverRegs))
3462 return UnableToLegalize;
3463
3464 LLT Unused;
3465 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
3466 Src2Regs, Src2LeftoverRegs))
3467 llvm_unreachable("inconsistent extractParts result");
3468
3469 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3470 auto Select = MIRBuilder.buildSelect(NarrowTy,
3471 CondReg, Src1Regs[I], Src2Regs[I]);
3472 DstRegs.push_back(Select->getOperand(0).getReg());
3473 }
3474
3475 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3476 auto Select = MIRBuilder.buildSelect(
3477 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
3478 DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
3479 }
3480
3481 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3482 LeftoverTy, DstLeftoverRegs);
3483
3484 MI.eraseFromParent();
3485 return Legalized;
3486}
3487
3488LegalizerHelper::LegalizeResult
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003489LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3490 unsigned Opc = MI.getOpcode();
3491 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
Diana Picus0528e2c2018-11-26 11:07:02 +00003492 auto isSupported = [this](const LegalityQuery &Q) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003493 auto QAction = LI.getAction(Q).Action;
Diana Picus0528e2c2018-11-26 11:07:02 +00003494 return QAction == Legal || QAction == Libcall || QAction == Custom;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003495 };
3496 switch (Opc) {
3497 default:
3498 return UnableToLegalize;
3499 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
3500 // This trivially expands to CTLZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00003501 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003502 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00003503 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003504 return Legalized;
3505 }
3506 case TargetOpcode::G_CTLZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003507 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003508 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003509 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
Diana Picus0528e2c2018-11-26 11:07:02 +00003510 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003511 auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
3512 {Ty}, {SrcReg});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003513 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3514 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3515 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3516 SrcReg, MIBZero);
3517 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3518 MIBCtlzZU);
3519 MI.eraseFromParent();
3520 return Legalized;
3521 }
3522 // for now, we do this:
3523 // NewLen = NextPowerOf2(Len);
3524 // x = x | (x >> 1);
3525 // x = x | (x >> 2);
3526 // ...
3527 // x = x | (x >>16);
3528 // x = x | (x >>32); // for 64-bit input
3529 // Upto NewLen/2
3530 // return Len - popcount(x);
3531 //
3532 // Ref: "Hacker's Delight" by Henry Warren
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003533 Register Op = SrcReg;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003534 unsigned NewLen = PowerOf2Ceil(Len);
3535 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
3536 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
3537 auto MIBOp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003538 TargetOpcode::G_OR, {Ty},
3539 {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
3540 {Op, MIBShiftAmt})});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003541 Op = MIBOp->getOperand(0).getReg();
3542 }
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003543 auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
3544 MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3545 {MIRBuilder.buildConstant(Ty, Len), MIBPop});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003546 MI.eraseFromParent();
3547 return Legalized;
3548 }
3549 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
3550 // This trivially expands to CTTZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00003551 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003552 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00003553 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003554 return Legalized;
3555 }
3556 case TargetOpcode::G_CTTZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003557 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003558 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003559 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003560 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
3561 // zero.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003562 auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
3563 {Ty}, {SrcReg});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003564 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3565 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3566 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3567 SrcReg, MIBZero);
3568 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3569 MIBCttzZU);
3570 MI.eraseFromParent();
3571 return Legalized;
3572 }
3573 // for now, we use: { return popcount(~x & (x - 1)); }
3574 // unless the target has ctlz but not ctpop, in which case we use:
3575 // { return 32 - nlz(~x & (x-1)); }
3576 // Ref: "Hacker's Delight" by Henry Warren
3577 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
3578 auto MIBNot =
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003579 MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003580 auto MIBTmp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003581 TargetOpcode::G_AND, {Ty},
3582 {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
3583 {SrcReg, MIBCstNeg1})});
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003584 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
3585 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003586 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
3587 MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003588 TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3589 {MIBCstLen,
3590 MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003591 MI.eraseFromParent();
3592 return Legalized;
3593 }
3594 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
3595 MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
3596 return Legalized;
3597 }
3598 }
3599}
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003600
3601// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
3602// representation.
3603LegalizerHelper::LegalizeResult
3604LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003605 Register Dst = MI.getOperand(0).getReg();
3606 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003607 const LLT S64 = LLT::scalar(64);
3608 const LLT S32 = LLT::scalar(32);
3609 const LLT S1 = LLT::scalar(1);
3610
3611 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
3612
3613 // unsigned cul2f(ulong u) {
3614 // uint lz = clz(u);
3615 // uint e = (u != 0) ? 127U + 63U - lz : 0;
3616 // u = (u << lz) & 0x7fffffffffffffffUL;
3617 // ulong t = u & 0xffffffffffUL;
3618 // uint v = (e << 23) | (uint)(u >> 40);
3619 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3620 // return as_float(v + r);
3621 // }
3622
3623 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3624 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3625
3626 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3627
3628 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3629 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3630
3631 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3632 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3633
3634 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3635 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3636
3637 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3638
3639 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3640 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3641
3642 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3643 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3644 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3645
3646 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3647 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3648 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3649 auto One = MIRBuilder.buildConstant(S32, 1);
3650
3651 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3652 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3653 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3654 MIRBuilder.buildAdd(Dst, V, R);
3655
3656 return Legalized;
3657}
3658
3659LegalizerHelper::LegalizeResult
3660LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003661 Register Dst = MI.getOperand(0).getReg();
3662 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003663 LLT DstTy = MRI.getType(Dst);
3664 LLT SrcTy = MRI.getType(Src);
3665
3666 if (SrcTy != LLT::scalar(64))
3667 return UnableToLegalize;
3668
3669 if (DstTy == LLT::scalar(32)) {
3670 // TODO: SelectionDAG has several alternative expansions to port which may
3671 // be more reasonble depending on the available instructions. If a target
3672 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3673 // intermediate type, this is probably worse.
3674 return lowerU64ToF32BitOps(MI);
3675 }
3676
3677 return UnableToLegalize;
3678}
3679
3680LegalizerHelper::LegalizeResult
3681LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003682 Register Dst = MI.getOperand(0).getReg();
3683 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003684 LLT DstTy = MRI.getType(Dst);
3685 LLT SrcTy = MRI.getType(Src);
3686
3687 const LLT S64 = LLT::scalar(64);
3688 const LLT S32 = LLT::scalar(32);
3689 const LLT S1 = LLT::scalar(1);
3690
3691 if (SrcTy != S64)
3692 return UnableToLegalize;
3693
3694 if (DstTy == S32) {
3695 // signed cl2f(long l) {
3696 // long s = l >> 63;
3697 // float r = cul2f((l + s) ^ s);
3698 // return s ? -r : r;
3699 // }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003700 Register L = Src;
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003701 auto SignBit = MIRBuilder.buildConstant(S64, 63);
3702 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3703
3704 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3705 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3706 auto R = MIRBuilder.buildUITOFP(S32, Xor);
3707
3708 auto RNeg = MIRBuilder.buildFNeg(S32, R);
3709 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3710 MIRBuilder.buildConstant(S64, 0));
3711 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3712 return Legalized;
3713 }
3714
3715 return UnableToLegalize;
3716}
Matt Arsenault6f74f552019-07-01 17:18:03 +00003717
3718static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
3719 switch (Opc) {
3720 case TargetOpcode::G_SMIN:
3721 return CmpInst::ICMP_SLT;
3722 case TargetOpcode::G_SMAX:
3723 return CmpInst::ICMP_SGT;
3724 case TargetOpcode::G_UMIN:
3725 return CmpInst::ICMP_ULT;
3726 case TargetOpcode::G_UMAX:
3727 return CmpInst::ICMP_UGT;
3728 default:
3729 llvm_unreachable("not in integer min/max");
3730 }
3731}
3732
3733LegalizerHelper::LegalizeResult
3734LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3735 Register Dst = MI.getOperand(0).getReg();
3736 Register Src0 = MI.getOperand(1).getReg();
3737 Register Src1 = MI.getOperand(2).getReg();
3738
3739 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
3740 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
3741
3742 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
3743 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
3744
3745 MI.eraseFromParent();
3746 return Legalized;
3747}
Matt Arsenaultb1843e12019-07-09 23:34:29 +00003748
3749LegalizerHelper::LegalizeResult
3750LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3751 Register Dst = MI.getOperand(0).getReg();
3752 Register Src0 = MI.getOperand(1).getReg();
3753 Register Src1 = MI.getOperand(2).getReg();
3754
3755 const LLT Src0Ty = MRI.getType(Src0);
3756 const LLT Src1Ty = MRI.getType(Src1);
3757
3758 const int Src0Size = Src0Ty.getScalarSizeInBits();
3759 const int Src1Size = Src1Ty.getScalarSizeInBits();
3760
3761 auto SignBitMask = MIRBuilder.buildConstant(
3762 Src0Ty, APInt::getSignMask(Src0Size));
3763
3764 auto NotSignBitMask = MIRBuilder.buildConstant(
3765 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
3766
3767 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
3768 MachineInstr *Or;
3769
3770 if (Src0Ty == Src1Ty) {
3771 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
3772 Or = MIRBuilder.buildOr(Dst, And0, And1);
3773 } else if (Src0Size > Src1Size) {
3774 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
3775 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
3776 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
3777 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
3778 Or = MIRBuilder.buildOr(Dst, And0, And1);
3779 } else {
3780 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
3781 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
3782 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
3783 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
3784 Or = MIRBuilder.buildOr(Dst, And0, And1);
3785 }
3786
3787 // Be careful about setting nsz/nnan/ninf on every instruction, since the
3788 // constants are a nan and -0.0, but the final result should preserve
3789 // everything.
3790 if (unsigned Flags = MI.getFlags())
3791 Or->setFlags(Flags);
3792
3793 MI.eraseFromParent();
3794 return Legalized;
3795}
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00003796
3797LegalizerHelper::LegalizeResult
3798LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
3799 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
3800 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
3801
3802 Register Dst = MI.getOperand(0).getReg();
3803 Register Src0 = MI.getOperand(1).getReg();
3804 Register Src1 = MI.getOperand(2).getReg();
3805 LLT Ty = MRI.getType(Dst);
3806
3807 if (!MI.getFlag(MachineInstr::FmNoNans)) {
3808 // Insert canonicalizes if it's possible we need to quiet to get correct
3809 // sNaN behavior.
3810
3811 // Note this must be done here, and not as an optimization combine in the
3812 // absence of a dedicate quiet-snan instruction as we're using an
3813 // omni-purpose G_FCANONICALIZE.
3814 if (!isKnownNeverSNaN(Src0, MRI))
3815 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
3816
3817 if (!isKnownNeverSNaN(Src1, MRI))
3818 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
3819 }
3820
3821 // If there are no nans, it's safe to simply replace this with the non-IEEE
3822 // version.
3823 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
3824 MI.eraseFromParent();
3825 return Legalized;
3826}
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00003827
3828LegalizerHelper::LegalizeResult
3829LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
3830 const unsigned NumDst = MI.getNumOperands() - 1;
3831 const Register SrcReg = MI.getOperand(NumDst).getReg();
3832 LLT SrcTy = MRI.getType(SrcReg);
3833
3834 Register Dst0Reg = MI.getOperand(0).getReg();
3835 LLT DstTy = MRI.getType(Dst0Reg);
3836
3837
3838 // Expand scalarizing unmerge as bitcast to integer and shift.
3839 if (!DstTy.isVector() && SrcTy.isVector() &&
3840 SrcTy.getElementType() == DstTy) {
3841 LLT IntTy = LLT::scalar(SrcTy.getSizeInBits());
3842 Register Cast = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0);
3843
3844 MIRBuilder.buildTrunc(Dst0Reg, Cast);
3845
3846 const unsigned DstSize = DstTy.getSizeInBits();
3847 unsigned Offset = DstSize;
3848 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
3849 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
3850 auto Shift = MIRBuilder.buildLShr(IntTy, Cast, ShiftAmt);
3851 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
3852 }
3853
3854 MI.eraseFromParent();
3855 return Legalized;
3856 }
3857
3858 return UnableToLegalize;
3859}
Matt Arsenault690645b2019-08-13 16:09:07 +00003860
3861LegalizerHelper::LegalizeResult
3862LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
3863 Register DstReg = MI.getOperand(0).getReg();
3864 Register Src0Reg = MI.getOperand(1).getReg();
3865 Register Src1Reg = MI.getOperand(2).getReg();
Aditya Nandakumar615eee62019-08-13 21:49:11 +00003866 LLT Src0Ty = MRI.getType(Src0Reg);
Matt Arsenault690645b2019-08-13 16:09:07 +00003867 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault690645b2019-08-13 16:09:07 +00003868 LLT IdxTy = LLT::scalar(32);
3869
3870 const Constant *ShufMask = MI.getOperand(3).getShuffleMask();
3871
3872 SmallVector<int, 32> Mask;
3873 ShuffleVectorInst::getShuffleMask(ShufMask, Mask);
3874
Amara Emersonc8092302019-08-16 18:06:53 +00003875 if (DstTy.isScalar()) {
3876 if (Src0Ty.isVector())
3877 return UnableToLegalize;
3878
3879 // This is just a SELECT.
3880 assert(Mask.size() == 1 && "Expected a single mask element");
3881 Register Val;
3882 if (Mask[0] < 0 || Mask[0] > 1)
3883 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
3884 else
3885 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
3886 MIRBuilder.buildCopy(DstReg, Val);
3887 MI.eraseFromParent();
3888 return Legalized;
3889 }
3890
Matt Arsenault690645b2019-08-13 16:09:07 +00003891 Register Undef;
3892 SmallVector<Register, 32> BuildVec;
Amara Emersonc8092302019-08-16 18:06:53 +00003893 LLT EltTy = DstTy.getElementType();
Matt Arsenault690645b2019-08-13 16:09:07 +00003894
3895 for (int Idx : Mask) {
3896 if (Idx < 0) {
3897 if (!Undef.isValid())
3898 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
3899 BuildVec.push_back(Undef);
3900 continue;
3901 }
3902
Aditya Nandakumar615eee62019-08-13 21:49:11 +00003903 if (Src0Ty.isScalar()) {
3904 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
3905 } else {
Aditya Nandakumarc65ac862019-08-14 01:23:33 +00003906 int NumElts = Src0Ty.getNumElements();
Aditya Nandakumar615eee62019-08-13 21:49:11 +00003907 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
3908 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
3909 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
3910 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
3911 BuildVec.push_back(Extract.getReg(0));
3912 }
Matt Arsenault690645b2019-08-13 16:09:07 +00003913 }
3914
3915 MIRBuilder.buildBuildVector(DstReg, BuildVec);
3916 MI.eraseFromParent();
3917 return Legalized;
3918}
Amara Emersone20b91c2019-08-27 19:54:27 +00003919
3920LegalizerHelper::LegalizeResult
3921LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
3922 Register Dst = MI.getOperand(0).getReg();
3923 Register AllocSize = MI.getOperand(1).getReg();
3924 unsigned Align = MI.getOperand(2).getImm();
3925
3926 const auto &MF = *MI.getMF();
3927 const auto &TLI = *MF.getSubtarget().getTargetLowering();
3928
3929 LLT PtrTy = MRI.getType(Dst);
3930 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
3931
3932 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
3933 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
3934 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
3935
3936 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
3937 // have to generate an extra instruction to negate the alloc and then use
3938 // G_GEP to add the negative offset.
3939 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
3940 if (Align) {
3941 APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
3942 AlignMask.negate();
3943 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
3944 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
3945 }
3946
3947 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
3948 MIRBuilder.buildCopy(SPReg, SPTmp);
3949 MIRBuilder.buildCopy(Dst, SPTmp);
3950
3951 MI.eraseFromParent();
3952 return Legalized;
3953}