blob: d9e097f1197aff68a0f3793fdbc5556df664746c [file] [log] [blame]
Tim Northover69fa84a2016-10-14 22:18:18 +00001//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
Tim Northover33b07d62016-07-22 20:03:43 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tim Northover33b07d62016-07-22 20:03:43 +00006//
7//===----------------------------------------------------------------------===//
8//
Tim Northover69fa84a2016-10-14 22:18:18 +00009/// \file This file implements the LegalizerHelper class to legalize
Tim Northover33b07d62016-07-22 20:03:43 +000010/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
Tim Northover69fa84a2016-10-14 22:18:18 +000015#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
Tim Northoveredb3c8c2016-08-29 19:07:16 +000016#include "llvm/CodeGen/GlobalISel/CallLowering.h"
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000017#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
Tim Northover69fa84a2016-10-14 22:18:18 +000018#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000019#include "llvm/CodeGen/MachineRegisterInfo.h"
Amara Emersone20b91c2019-08-27 19:54:27 +000020#include "llvm/CodeGen/TargetFrameLowering.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000021#include "llvm/CodeGen/TargetInstrInfo.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000022#include "llvm/CodeGen/TargetLowering.h"
23#include "llvm/CodeGen/TargetSubtargetInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000024#include "llvm/Support/Debug.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000025#include "llvm/Support/MathExtras.h"
Tim Northover33b07d62016-07-22 20:03:43 +000026#include "llvm/Support/raw_ostream.h"
Tim Northover33b07d62016-07-22 20:03:43 +000027
Daniel Sanders5377fb32017-04-20 15:46:12 +000028#define DEBUG_TYPE "legalizer"
Tim Northover33b07d62016-07-22 20:03:43 +000029
30using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000031using namespace LegalizeActions;
Tim Northover33b07d62016-07-22 20:03:43 +000032
Matt Arsenaultc83b8232019-02-07 17:38:00 +000033/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
34///
35/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
36/// with any leftover piece as type \p LeftoverTy
37///
Matt Arsenaultd3093c22019-02-28 00:16:32 +000038/// Returns -1 in the first element of the pair if the breakdown is not
39/// satisfiable.
40static std::pair<int, int>
41getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
Matt Arsenaultc83b8232019-02-07 17:38:00 +000042 assert(!LeftoverTy.isValid() && "this is an out argument");
43
44 unsigned Size = OrigTy.getSizeInBits();
45 unsigned NarrowSize = NarrowTy.getSizeInBits();
46 unsigned NumParts = Size / NarrowSize;
47 unsigned LeftoverSize = Size - NumParts * NarrowSize;
48 assert(Size > NarrowSize);
49
50 if (LeftoverSize == 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000051 return {NumParts, 0};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000052
53 if (NarrowTy.isVector()) {
54 unsigned EltSize = OrigTy.getScalarSizeInBits();
55 if (LeftoverSize % EltSize != 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000056 return {-1, -1};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000057 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
58 } else {
59 LeftoverTy = LLT::scalar(LeftoverSize);
60 }
61
Matt Arsenaultd3093c22019-02-28 00:16:32 +000062 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
63 return std::make_pair(NumParts, NumLeftover);
Matt Arsenaultc83b8232019-02-07 17:38:00 +000064}
65
Matt Arsenaulta66d2812020-01-10 10:41:29 -050066static LLT getGCDType(LLT OrigTy, LLT TargetTy) {
67 if (OrigTy.isVector() && TargetTy.isVector()) {
68 assert(OrigTy.getElementType() == TargetTy.getElementType());
69 int GCD = greatestCommonDivisor(OrigTy.getNumElements(),
70 TargetTy.getNumElements());
71 return LLT::scalarOrVector(GCD, OrigTy.getElementType());
72 }
73
74 if (OrigTy.isVector() && !TargetTy.isVector()) {
75 assert(OrigTy.getElementType() == TargetTy);
76 return TargetTy;
77 }
78
79 assert(!OrigTy.isVector() && !TargetTy.isVector() &&
80 "GCD type of vector and scalar not implemented");
81
82 int GCD = greatestCommonDivisor(OrigTy.getSizeInBits(),
83 TargetTy.getSizeInBits());
84 return LLT::scalar(GCD);
85}
86
87static LLT getLCMType(LLT Ty0, LLT Ty1) {
88 assert(Ty0.isScalar() && Ty1.isScalar() && "not yet handled");
89 unsigned Mul = Ty0.getSizeInBits() * Ty1.getSizeInBits();
90 int GCDSize = greatestCommonDivisor(Ty0.getSizeInBits(),
91 Ty1.getSizeInBits());
92 return LLT::scalar(Mul / GCDSize);
93}
94
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000095LegalizerHelper::LegalizerHelper(MachineFunction &MF,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +000096 GISelChangeObserver &Observer,
97 MachineIRBuilder &Builder)
98 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
99 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
Tim Northover33b07d62016-07-22 20:03:43 +0000100 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000101 MIRBuilder.setChangeObserver(Observer);
Tim Northover33b07d62016-07-22 20:03:43 +0000102}
103
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000104LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +0000105 GISelChangeObserver &Observer,
106 MachineIRBuilder &B)
107 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +0000108 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000109 MIRBuilder.setChangeObserver(Observer);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +0000110}
Tim Northover69fa84a2016-10-14 22:18:18 +0000111LegalizerHelper::LegalizeResult
Volkan Keles685fbda2017-03-10 18:34:57 +0000112LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000113 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
Daniel Sanders5377fb32017-04-20 15:46:12 +0000114
Aditya Nandakumar1023a2e2019-07-01 17:53:50 +0000115 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
116 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
Matt Arsenaultc5fffa42020-01-27 15:50:55 -0500117 return LI.legalizeIntrinsic(MI, MIRBuilder, Observer) ? Legalized
118 : UnableToLegalize;
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000119 auto Step = LI.getAction(MI, MRI);
120 switch (Step.Action) {
Daniel Sanders9ade5592018-01-29 17:37:29 +0000121 case Legal:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000122 LLVM_DEBUG(dbgs() << ".. Already legal\n");
Tim Northover33b07d62016-07-22 20:03:43 +0000123 return AlreadyLegal;
Daniel Sanders9ade5592018-01-29 17:37:29 +0000124 case Libcall:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000125 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000126 return libcall(MI);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000127 case NarrowScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000128 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000129 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000130 case WidenScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000131 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000132 return widenScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000133 case Lower:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000134 LLVM_DEBUG(dbgs() << ".. Lower\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000135 return lower(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000136 case FewerElements:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000137 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000138 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
Matt Arsenault18ec3822019-02-11 22:00:39 +0000139 case MoreElements:
140 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
141 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000142 case Custom:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000143 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000144 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
145 : UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +0000146 default:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000147 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
Tim Northover33b07d62016-07-22 20:03:43 +0000148 return UnableToLegalize;
149 }
150}
151
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000152void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
153 SmallVectorImpl<Register> &VRegs) {
Tim Northoverbf017292017-03-03 22:46:09 +0000154 for (int i = 0; i < NumParts; ++i)
Tim Northover0f140c72016-09-09 11:46:34 +0000155 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
Tim Northoverbf017292017-03-03 22:46:09 +0000156 MIRBuilder.buildUnmerge(VRegs, Reg);
Tim Northover33b07d62016-07-22 20:03:43 +0000157}
158
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000159bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000160 LLT MainTy, LLT &LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000161 SmallVectorImpl<Register> &VRegs,
162 SmallVectorImpl<Register> &LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000163 assert(!LeftoverTy.isValid() && "this is an out argument");
164
165 unsigned RegSize = RegTy.getSizeInBits();
166 unsigned MainSize = MainTy.getSizeInBits();
167 unsigned NumParts = RegSize / MainSize;
168 unsigned LeftoverSize = RegSize - NumParts * MainSize;
169
170 // Use an unmerge when possible.
171 if (LeftoverSize == 0) {
172 for (unsigned I = 0; I < NumParts; ++I)
173 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
174 MIRBuilder.buildUnmerge(VRegs, Reg);
175 return true;
176 }
177
178 if (MainTy.isVector()) {
179 unsigned EltSize = MainTy.getScalarSizeInBits();
180 if (LeftoverSize % EltSize != 0)
181 return false;
182 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
183 } else {
184 LeftoverTy = LLT::scalar(LeftoverSize);
185 }
186
187 // For irregular sizes, extract the individual parts.
188 for (unsigned I = 0; I != NumParts; ++I) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000189 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000190 VRegs.push_back(NewReg);
191 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
192 }
193
194 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
195 Offset += LeftoverSize) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000196 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000197 LeftoverRegs.push_back(NewReg);
198 MIRBuilder.buildExtract(NewReg, Reg, Offset);
199 }
200
201 return true;
202}
203
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000204void LegalizerHelper::insertParts(Register DstReg,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000205 LLT ResultTy, LLT PartTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000206 ArrayRef<Register> PartRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000207 LLT LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000208 ArrayRef<Register> LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000209 if (!LeftoverTy.isValid()) {
210 assert(LeftoverRegs.empty());
211
Matt Arsenault81511e52019-02-05 00:13:44 +0000212 if (!ResultTy.isVector()) {
213 MIRBuilder.buildMerge(DstReg, PartRegs);
214 return;
215 }
216
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000217 if (PartTy.isVector())
218 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
219 else
220 MIRBuilder.buildBuildVector(DstReg, PartRegs);
221 return;
222 }
223
224 unsigned PartSize = PartTy.getSizeInBits();
225 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
226
Matt Arsenault3018d182019-06-28 01:47:44 +0000227 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000228 MIRBuilder.buildUndef(CurResultReg);
229
230 unsigned Offset = 0;
Matt Arsenault3018d182019-06-28 01:47:44 +0000231 for (Register PartReg : PartRegs) {
232 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000233 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
234 CurResultReg = NewResultReg;
235 Offset += PartSize;
236 }
237
238 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
239 // Use the original output register for the final insert to avoid a copy.
Matt Arsenault3018d182019-06-28 01:47:44 +0000240 Register NewResultReg = (I + 1 == E) ?
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000241 DstReg : MRI.createGenericVirtualRegister(ResultTy);
242
243 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
244 CurResultReg = NewResultReg;
245 Offset += LeftoverPartSize;
246 }
247}
248
Matt Arsenaulta66d2812020-01-10 10:41:29 -0500249/// Return the result registers of G_UNMERGE_VALUES \p MI in \p Regs
250static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
251 const MachineInstr &MI) {
252 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
253
254 const int NumResults = MI.getNumOperands() - 1;
255 Regs.resize(NumResults);
256 for (int I = 0; I != NumResults; ++I)
257 Regs[I] = MI.getOperand(I).getReg();
258}
259
260LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
261 LLT NarrowTy, Register SrcReg) {
262 LLT SrcTy = MRI.getType(SrcReg);
263
264 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
265 if (SrcTy == GCDTy) {
266 // If the source already evenly divides the result type, we don't need to do
267 // anything.
268 Parts.push_back(SrcReg);
269 } else {
270 // Need to split into common type sized pieces.
271 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
272 getUnmergeResults(Parts, *Unmerge);
273 }
274
275 return GCDTy;
276}
277
278void LegalizerHelper::buildLCMMerge(Register DstReg, LLT NarrowTy, LLT GCDTy,
279 SmallVectorImpl<Register> &VRegs,
280 unsigned PadStrategy) {
281 LLT DstTy = MRI.getType(DstReg);
282 LLT LCMTy = getLCMType(DstTy, NarrowTy);
283
284 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
285 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
286 int NumOrigSrc = VRegs.size();
287
288 Register PadReg;
289
290 // Get a value we can use to pad the source value if the sources won't evenly
291 // cover the result type.
292 if (NumOrigSrc < NumParts * NumSubParts) {
293 if (PadStrategy == TargetOpcode::G_ZEXT)
294 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0);
295 else if (PadStrategy == TargetOpcode::G_ANYEXT)
296 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
297 else {
298 assert(PadStrategy == TargetOpcode::G_SEXT);
299
300 // Shift the sign bit of the low register through the high register.
301 auto ShiftAmt =
302 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1);
303 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0);
304 }
305 }
306
307 // Registers for the final merge to be produced.
308 SmallVector<Register, 4> Remerge;
309 Remerge.resize(NumParts);
310
311 // Registers needed for intermediate merges, which will be merged into a
312 // source for Remerge.
313 SmallVector<Register, 4> SubMerge;
314 SubMerge.resize(NumSubParts);
315
316 // Once we've fully read off the end of the original source bits, we can reuse
317 // the same high bits for remaining padding elements.
318 Register AllPadReg;
319
320 // Build merges to the LCM type to cover the original result type.
321 for (int I = 0; I != NumParts; ++I) {
322 bool AllMergePartsArePadding = true;
323
324 // Build the requested merges to the requested type.
325 for (int J = 0; J != NumSubParts; ++J) {
326 int Idx = I * NumSubParts + J;
327 if (Idx >= NumOrigSrc) {
328 SubMerge[J] = PadReg;
329 continue;
330 }
331
332 SubMerge[J] = VRegs[Idx];
333
334 // There are meaningful bits here we can't reuse later.
335 AllMergePartsArePadding = false;
336 }
337
338 // If we've filled up a complete piece with padding bits, we can directly
339 // emit the natural sized constant if applicable, rather than a merge of
340 // smaller constants.
341 if (AllMergePartsArePadding && !AllPadReg) {
342 if (PadStrategy == TargetOpcode::G_ANYEXT)
343 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0);
344 else if (PadStrategy == TargetOpcode::G_ZEXT)
345 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
346
347 // If this is a sign extension, we can't materialize a trivial constant
348 // with the right type and have to produce a merge.
349 }
350
351 if (AllPadReg) {
352 // Avoid creating additional instructions if we're just adding additional
353 // copies of padding bits.
354 Remerge[I] = AllPadReg;
355 continue;
356 }
357
358 if (NumSubParts == 1)
359 Remerge[I] = SubMerge[0];
360 else
361 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0);
362
363 // In the sign extend padding case, re-use the first all-signbit merge.
364 if (AllMergePartsArePadding && !AllPadReg)
365 AllPadReg = Remerge[I];
366 }
367
368 // Create the merge to the widened source, and extract the relevant bits into
369 // the result.
370 if (DstTy == LCMTy)
371 MIRBuilder.buildMerge(DstReg, Remerge);
372 else
373 MIRBuilder.buildTrunc(DstReg, MIRBuilder.buildMerge(LCMTy, Remerge));
374}
375
Tim Northovere0418412017-02-08 23:23:39 +0000376static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
377 switch (Opcode) {
Diana Picuse97822e2017-04-24 07:22:31 +0000378 case TargetOpcode::G_SDIV:
Amara Emerson2a2c25b2019-09-03 21:42:32 +0000379 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
380 switch (Size) {
381 case 32:
382 return RTLIB::SDIV_I32;
383 case 64:
384 return RTLIB::SDIV_I64;
385 case 128:
386 return RTLIB::SDIV_I128;
387 default:
388 llvm_unreachable("unexpected size");
389 }
Diana Picuse97822e2017-04-24 07:22:31 +0000390 case TargetOpcode::G_UDIV:
Amara Emerson2a2c25b2019-09-03 21:42:32 +0000391 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
392 switch (Size) {
393 case 32:
394 return RTLIB::UDIV_I32;
395 case 64:
396 return RTLIB::UDIV_I64;
397 case 128:
398 return RTLIB::UDIV_I128;
399 default:
400 llvm_unreachable("unexpected size");
401 }
Diana Picus02e11012017-06-15 10:53:31 +0000402 case TargetOpcode::G_SREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000403 assert((Size == 32 || Size == 64) && "Unsupported size");
404 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
Diana Picus02e11012017-06-15 10:53:31 +0000405 case TargetOpcode::G_UREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000406 assert((Size == 32 || Size == 64) && "Unsupported size");
407 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
Diana Picus0528e2c2018-11-26 11:07:02 +0000408 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
409 assert(Size == 32 && "Unsupported size");
410 return RTLIB::CTLZ_I32;
Diana Picus1314a282017-04-11 10:52:34 +0000411 case TargetOpcode::G_FADD:
412 assert((Size == 32 || Size == 64) && "Unsupported size");
413 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
Javed Absar5cde1cc2017-10-30 13:51:56 +0000414 case TargetOpcode::G_FSUB:
415 assert((Size == 32 || Size == 64) && "Unsupported size");
416 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
Diana Picus9faa09b2017-11-23 12:44:20 +0000417 case TargetOpcode::G_FMUL:
418 assert((Size == 32 || Size == 64) && "Unsupported size");
419 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
Diana Picusc01f7f12017-11-23 13:26:07 +0000420 case TargetOpcode::G_FDIV:
421 assert((Size == 32 || Size == 64) && "Unsupported size");
422 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
Jessica Paquette84bedac2019-01-30 23:46:15 +0000423 case TargetOpcode::G_FEXP:
424 assert((Size == 32 || Size == 64) && "Unsupported size");
425 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
Jessica Paquettee7941212019-04-03 16:58:32 +0000426 case TargetOpcode::G_FEXP2:
427 assert((Size == 32 || Size == 64) && "Unsupported size");
428 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000429 case TargetOpcode::G_FREM:
430 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
431 case TargetOpcode::G_FPOW:
432 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
Diana Picuse74243d2018-01-12 11:30:45 +0000433 case TargetOpcode::G_FMA:
434 assert((Size == 32 || Size == 64) && "Unsupported size");
435 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
Jessica Paquette7db82d72019-01-28 18:34:18 +0000436 case TargetOpcode::G_FSIN:
437 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
438 return Size == 128 ? RTLIB::SIN_F128
439 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
440 case TargetOpcode::G_FCOS:
441 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
442 return Size == 128 ? RTLIB::COS_F128
443 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
Jessica Paquettec49428a2019-01-28 19:53:14 +0000444 case TargetOpcode::G_FLOG10:
445 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
446 return Size == 128 ? RTLIB::LOG10_F128
447 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000448 case TargetOpcode::G_FLOG:
449 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
450 return Size == 128 ? RTLIB::LOG_F128
451 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
Jessica Paquette0154bd12019-01-30 21:16:04 +0000452 case TargetOpcode::G_FLOG2:
453 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
454 return Size == 128 ? RTLIB::LOG2_F128
455 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000456 case TargetOpcode::G_FCEIL:
457 assert((Size == 32 || Size == 64) && "Unsupported size");
458 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
459 case TargetOpcode::G_FFLOOR:
460 assert((Size == 32 || Size == 64) && "Unsupported size");
461 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000462 }
463 llvm_unreachable("Unknown libcall function");
464}
465
Jessica Paquette727328a2019-09-13 20:25:58 +0000466/// True if an instruction is in tail position in its caller. Intended for
467/// legalizing libcalls as tail calls when possible.
468static bool isLibCallInTailPosition(MachineInstr &MI) {
469 const Function &F = MI.getParent()->getParent()->getFunction();
470
471 // Conservatively require the attributes of the call to match those of
472 // the return. Ignore NoAlias and NonNull because they don't affect the
473 // call sequence.
474 AttributeList CallerAttrs = F.getAttributes();
475 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
476 .removeAttribute(Attribute::NoAlias)
477 .removeAttribute(Attribute::NonNull)
478 .hasAttributes())
479 return false;
480
481 // It's not safe to eliminate the sign / zero extension of the return value.
482 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
483 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
484 return false;
485
486 // Only tail call if the following instruction is a standard return.
487 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
488 MachineInstr *Next = MI.getNextNode();
489 if (!Next || TII.isTailCall(*Next) || !Next->isReturn())
490 return false;
491
492 return true;
493}
494
Diana Picusfc1675e2017-07-05 12:57:24 +0000495LegalizerHelper::LegalizeResult
496llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
497 const CallLowering::ArgInfo &Result,
498 ArrayRef<CallLowering::ArgInfo> Args) {
Diana Picuse97822e2017-04-24 07:22:31 +0000499 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
500 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
Diana Picuse97822e2017-04-24 07:22:31 +0000501 const char *Name = TLI.getLibcallName(Libcall);
Diana Picusd0104ea2017-07-06 09:09:33 +0000502
Tim Northovere1a5f662019-08-09 08:26:38 +0000503 CallLowering::CallLoweringInfo Info;
504 Info.CallConv = TLI.getLibcallCallingConv(Libcall);
505 Info.Callee = MachineOperand::CreateES(Name);
506 Info.OrigRet = Result;
507 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
508 if (!CLI.lowerCall(MIRBuilder, Info))
Diana Picus02e11012017-06-15 10:53:31 +0000509 return LegalizerHelper::UnableToLegalize;
Diana Picusd0104ea2017-07-06 09:09:33 +0000510
Diana Picuse97822e2017-04-24 07:22:31 +0000511 return LegalizerHelper::Legalized;
512}
513
Diana Picus65ed3642018-01-17 13:34:10 +0000514// Useful for libcalls where all operands have the same type.
Diana Picus02e11012017-06-15 10:53:31 +0000515static LegalizerHelper::LegalizeResult
516simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
517 Type *OpType) {
518 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
Diana Picuse74243d2018-01-12 11:30:45 +0000519
520 SmallVector<CallLowering::ArgInfo, 3> Args;
521 for (unsigned i = 1; i < MI.getNumOperands(); i++)
522 Args.push_back({MI.getOperand(i).getReg(), OpType});
Diana Picusfc1675e2017-07-05 12:57:24 +0000523 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
Diana Picuse74243d2018-01-12 11:30:45 +0000524 Args);
Diana Picus02e11012017-06-15 10:53:31 +0000525}
526
Amara Emersoncf12c782019-07-19 00:24:45 +0000527LegalizerHelper::LegalizeResult
528llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
529 MachineInstr &MI) {
530 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
531 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
532
533 SmallVector<CallLowering::ArgInfo, 3> Args;
Amara Emerson509a4942019-09-28 05:33:21 +0000534 // Add all the args, except for the last which is an imm denoting 'tail'.
535 for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
Amara Emersoncf12c782019-07-19 00:24:45 +0000536 Register Reg = MI.getOperand(i).getReg();
537
538 // Need derive an IR type for call lowering.
539 LLT OpLLT = MRI.getType(Reg);
540 Type *OpTy = nullptr;
541 if (OpLLT.isPointer())
542 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
543 else
544 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
545 Args.push_back({Reg, OpTy});
546 }
547
548 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
549 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
550 Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID();
551 RTLIB::Libcall RTLibcall;
552 switch (ID) {
553 case Intrinsic::memcpy:
554 RTLibcall = RTLIB::MEMCPY;
555 break;
556 case Intrinsic::memset:
557 RTLibcall = RTLIB::MEMSET;
558 break;
559 case Intrinsic::memmove:
560 RTLibcall = RTLIB::MEMMOVE;
561 break;
562 default:
563 return LegalizerHelper::UnableToLegalize;
564 }
565 const char *Name = TLI.getLibcallName(RTLibcall);
566
567 MIRBuilder.setInstr(MI);
Tim Northovere1a5f662019-08-09 08:26:38 +0000568
569 CallLowering::CallLoweringInfo Info;
570 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
571 Info.Callee = MachineOperand::CreateES(Name);
572 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
Amara Emerson509a4942019-09-28 05:33:21 +0000573 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
574 isLibCallInTailPosition(MI);
Jessica Paquette727328a2019-09-13 20:25:58 +0000575
Tim Northovere1a5f662019-08-09 08:26:38 +0000576 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
577 if (!CLI.lowerCall(MIRBuilder, Info))
Amara Emersoncf12c782019-07-19 00:24:45 +0000578 return LegalizerHelper::UnableToLegalize;
579
Jessica Paquette727328a2019-09-13 20:25:58 +0000580 if (Info.LoweredTailCall) {
581 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
582 // We must have a return following the call to get past
583 // isLibCallInTailPosition.
584 assert(MI.getNextNode() && MI.getNextNode()->isReturn() &&
585 "Expected instr following MI to be a return?");
586
587 // We lowered a tail call, so the call is now the return from the block.
588 // Delete the old return.
589 MI.getNextNode()->eraseFromParent();
590 }
591
Amara Emersoncf12c782019-07-19 00:24:45 +0000592 return LegalizerHelper::Legalized;
593}
594
Diana Picus65ed3642018-01-17 13:34:10 +0000595static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
596 Type *FromType) {
597 auto ToMVT = MVT::getVT(ToType);
598 auto FromMVT = MVT::getVT(FromType);
599
600 switch (Opcode) {
601 case TargetOpcode::G_FPEXT:
602 return RTLIB::getFPEXT(FromMVT, ToMVT);
603 case TargetOpcode::G_FPTRUNC:
604 return RTLIB::getFPROUND(FromMVT, ToMVT);
Diana Picus4ed0ee72018-01-30 07:54:52 +0000605 case TargetOpcode::G_FPTOSI:
606 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
607 case TargetOpcode::G_FPTOUI:
608 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
Diana Picus517531e2018-01-30 09:15:17 +0000609 case TargetOpcode::G_SITOFP:
610 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
611 case TargetOpcode::G_UITOFP:
612 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
Diana Picus65ed3642018-01-17 13:34:10 +0000613 }
614 llvm_unreachable("Unsupported libcall function");
615}
616
617static LegalizerHelper::LegalizeResult
618conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
619 Type *FromType) {
620 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
621 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
622 {{MI.getOperand(1).getReg(), FromType}});
623}
624
Tim Northover69fa84a2016-10-14 22:18:18 +0000625LegalizerHelper::LegalizeResult
626LegalizerHelper::libcall(MachineInstr &MI) {
Diana Picus02e11012017-06-15 10:53:31 +0000627 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
628 unsigned Size = LLTy.getSizeInBits();
Matthias Braunf1caa282017-12-15 22:22:58 +0000629 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000630
Diana Picusfc1675e2017-07-05 12:57:24 +0000631 MIRBuilder.setInstr(MI);
632
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000633 switch (MI.getOpcode()) {
634 default:
635 return UnableToLegalize;
Diana Picuse97822e2017-04-24 07:22:31 +0000636 case TargetOpcode::G_SDIV:
Diana Picus02e11012017-06-15 10:53:31 +0000637 case TargetOpcode::G_UDIV:
638 case TargetOpcode::G_SREM:
Diana Picus0528e2c2018-11-26 11:07:02 +0000639 case TargetOpcode::G_UREM:
640 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000641 Type *HLTy = IntegerType::get(Ctx, Size);
Diana Picusfc1675e2017-07-05 12:57:24 +0000642 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
643 if (Status != Legalized)
644 return Status;
645 break;
Diana Picuse97822e2017-04-24 07:22:31 +0000646 }
Diana Picus1314a282017-04-11 10:52:34 +0000647 case TargetOpcode::G_FADD:
Javed Absar5cde1cc2017-10-30 13:51:56 +0000648 case TargetOpcode::G_FSUB:
Diana Picus9faa09b2017-11-23 12:44:20 +0000649 case TargetOpcode::G_FMUL:
Diana Picusc01f7f12017-11-23 13:26:07 +0000650 case TargetOpcode::G_FDIV:
Diana Picuse74243d2018-01-12 11:30:45 +0000651 case TargetOpcode::G_FMA:
Tim Northovere0418412017-02-08 23:23:39 +0000652 case TargetOpcode::G_FPOW:
Jessica Paquette7db82d72019-01-28 18:34:18 +0000653 case TargetOpcode::G_FREM:
654 case TargetOpcode::G_FCOS:
Jessica Paquettec49428a2019-01-28 19:53:14 +0000655 case TargetOpcode::G_FSIN:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000656 case TargetOpcode::G_FLOG10:
Jessica Paquette0154bd12019-01-30 21:16:04 +0000657 case TargetOpcode::G_FLOG:
Jessica Paquette84bedac2019-01-30 23:46:15 +0000658 case TargetOpcode::G_FLOG2:
Jessica Paquettee7941212019-04-03 16:58:32 +0000659 case TargetOpcode::G_FEXP:
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000660 case TargetOpcode::G_FEXP2:
661 case TargetOpcode::G_FCEIL:
662 case TargetOpcode::G_FFLOOR: {
Jessica Paquette7db82d72019-01-28 18:34:18 +0000663 if (Size > 64) {
664 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
665 return UnableToLegalize;
666 }
Diana Picus02e11012017-06-15 10:53:31 +0000667 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
Diana Picusfc1675e2017-07-05 12:57:24 +0000668 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
669 if (Status != Legalized)
670 return Status;
671 break;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000672 }
Diana Picus65ed3642018-01-17 13:34:10 +0000673 case TargetOpcode::G_FPEXT: {
674 // FIXME: Support other floating point types (half, fp128 etc)
675 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
676 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
677 if (ToSize != 64 || FromSize != 32)
678 return UnableToLegalize;
679 LegalizeResult Status = conversionLibcall(
680 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
681 if (Status != Legalized)
682 return Status;
683 break;
684 }
685 case TargetOpcode::G_FPTRUNC: {
686 // FIXME: Support other floating point types (half, fp128 etc)
687 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
688 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
689 if (ToSize != 32 || FromSize != 64)
690 return UnableToLegalize;
691 LegalizeResult Status = conversionLibcall(
692 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
693 if (Status != Legalized)
694 return Status;
695 break;
696 }
Diana Picus4ed0ee72018-01-30 07:54:52 +0000697 case TargetOpcode::G_FPTOSI:
698 case TargetOpcode::G_FPTOUI: {
699 // FIXME: Support other types
700 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
701 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000702 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
Diana Picus4ed0ee72018-01-30 07:54:52 +0000703 return UnableToLegalize;
704 LegalizeResult Status = conversionLibcall(
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000705 MI, MIRBuilder,
706 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
Diana Picus4ed0ee72018-01-30 07:54:52 +0000707 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
708 if (Status != Legalized)
709 return Status;
710 break;
711 }
Diana Picus517531e2018-01-30 09:15:17 +0000712 case TargetOpcode::G_SITOFP:
713 case TargetOpcode::G_UITOFP: {
714 // FIXME: Support other types
715 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
716 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic153bd242019-06-20 09:05:02 +0000717 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
Diana Picus517531e2018-01-30 09:15:17 +0000718 return UnableToLegalize;
719 LegalizeResult Status = conversionLibcall(
720 MI, MIRBuilder,
721 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
Petar Avramovic153bd242019-06-20 09:05:02 +0000722 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
Diana Picus517531e2018-01-30 09:15:17 +0000723 if (Status != Legalized)
724 return Status;
725 break;
726 }
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000727 }
Diana Picusfc1675e2017-07-05 12:57:24 +0000728
729 MI.eraseFromParent();
730 return Legalized;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000731}
732
Tim Northover69fa84a2016-10-14 22:18:18 +0000733LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
734 unsigned TypeIdx,
735 LLT NarrowTy) {
Justin Bognerfde01042017-01-18 17:29:54 +0000736 MIRBuilder.setInstr(MI);
737
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000738 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
739 uint64_t NarrowSize = NarrowTy.getSizeInBits();
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000740
Tim Northover9656f142016-08-04 20:54:13 +0000741 switch (MI.getOpcode()) {
742 default:
743 return UnableToLegalize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000744 case TargetOpcode::G_IMPLICIT_DEF: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000745 // FIXME: add support for when SizeOp0 isn't an exact multiple of
746 // NarrowSize.
747 if (SizeOp0 % NarrowSize != 0)
748 return UnableToLegalize;
749 int NumParts = SizeOp0 / NarrowSize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000750
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000751 SmallVector<Register, 2> DstRegs;
Volkan Keles02bb1742018-02-14 19:58:36 +0000752 for (int i = 0; i < NumParts; ++i)
753 DstRegs.push_back(
Jay Foadb482e1b2020-01-23 11:51:35 +0000754 MIRBuilder.buildUndef(NarrowTy).getReg(0));
Amara Emerson5ec146042018-12-10 18:44:58 +0000755
Matt Arsenault3018d182019-06-28 01:47:44 +0000756 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000757 if(MRI.getType(DstReg).isVector())
758 MIRBuilder.buildBuildVector(DstReg, DstRegs);
759 else
760 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northoverff5e7e12017-06-30 20:27:36 +0000761 MI.eraseFromParent();
762 return Legalized;
763 }
Matt Arsenault71872722019-04-10 17:27:53 +0000764 case TargetOpcode::G_CONSTANT: {
765 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
766 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
767 unsigned TotalSize = Ty.getSizeInBits();
768 unsigned NarrowSize = NarrowTy.getSizeInBits();
769 int NumParts = TotalSize / NarrowSize;
770
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000771 SmallVector<Register, 4> PartRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000772 for (int I = 0; I != NumParts; ++I) {
773 unsigned Offset = I * NarrowSize;
774 auto K = MIRBuilder.buildConstant(NarrowTy,
775 Val.lshr(Offset).trunc(NarrowSize));
776 PartRegs.push_back(K.getReg(0));
777 }
778
779 LLT LeftoverTy;
780 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000781 SmallVector<Register, 1> LeftoverRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000782 if (LeftoverBits != 0) {
783 LeftoverTy = LLT::scalar(LeftoverBits);
784 auto K = MIRBuilder.buildConstant(
785 LeftoverTy,
786 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
787 LeftoverRegs.push_back(K.getReg(0));
788 }
789
790 insertParts(MI.getOperand(0).getReg(),
791 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
792
793 MI.eraseFromParent();
794 return Legalized;
795 }
Matt Arsenault25e99382020-01-10 10:07:24 -0500796 case TargetOpcode::G_SEXT:
Matt Arsenault917156172020-01-10 09:47:17 -0500797 case TargetOpcode::G_ZEXT:
Matt Arsenaultbe31a7b2020-01-10 11:02:18 -0500798 case TargetOpcode::G_ANYEXT:
799 return narrowScalarExt(MI, TypeIdx, NarrowTy);
Petar Avramovic5b4c5c22019-08-21 09:26:39 +0000800 case TargetOpcode::G_TRUNC: {
801 if (TypeIdx != 1)
802 return UnableToLegalize;
803
804 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
805 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
806 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
807 return UnableToLegalize;
808 }
809
Jay Foad63f73542020-01-16 12:37:00 +0000810 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
811 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
Petar Avramovic5b4c5c22019-08-21 09:26:39 +0000812 MI.eraseFromParent();
813 return Legalized;
814 }
Amara Emerson7bc4fad2019-07-26 23:46:38 +0000815
Tim Northover9656f142016-08-04 20:54:13 +0000816 case TargetOpcode::G_ADD: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000817 // FIXME: add support for when SizeOp0 isn't an exact multiple of
818 // NarrowSize.
819 if (SizeOp0 % NarrowSize != 0)
820 return UnableToLegalize;
Tim Northover9656f142016-08-04 20:54:13 +0000821 // Expand in terms of carry-setting/consuming G_ADDE instructions.
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000822 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
Tim Northover9656f142016-08-04 20:54:13 +0000823
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000824 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Tim Northover9656f142016-08-04 20:54:13 +0000825 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
826 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
827
Matt Arsenaultfba82852019-08-22 17:29:17 +0000828 Register CarryIn;
Tim Northover9656f142016-08-04 20:54:13 +0000829 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000830 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
831 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Tim Northover9656f142016-08-04 20:54:13 +0000832
Matt Arsenaultfba82852019-08-22 17:29:17 +0000833 if (i == 0)
834 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
835 else {
836 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
837 Src2Regs[i], CarryIn);
838 }
Tim Northover9656f142016-08-04 20:54:13 +0000839
840 DstRegs.push_back(DstReg);
841 CarryIn = CarryOut;
842 }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000843 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000844 if(MRI.getType(DstReg).isVector())
845 MIRBuilder.buildBuildVector(DstReg, DstRegs);
846 else
847 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northover9656f142016-08-04 20:54:13 +0000848 MI.eraseFromParent();
849 return Legalized;
850 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000851 case TargetOpcode::G_SUB: {
852 // FIXME: add support for when SizeOp0 isn't an exact multiple of
853 // NarrowSize.
854 if (SizeOp0 % NarrowSize != 0)
855 return UnableToLegalize;
856
857 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
858
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000859 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000860 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
861 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
862
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000863 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
864 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000865 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
866 {Src1Regs[0], Src2Regs[0]});
867 DstRegs.push_back(DstReg);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000868 Register BorrowIn = BorrowOut;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000869 for (int i = 1; i < NumParts; ++i) {
870 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
871 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
872
873 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
874 {Src1Regs[i], Src2Regs[i], BorrowIn});
875
876 DstRegs.push_back(DstReg);
877 BorrowIn = BorrowOut;
878 }
Jay Foad63f73542020-01-16 12:37:00 +0000879 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000880 MI.eraseFromParent();
881 return Legalized;
882 }
Matt Arsenault211e89d2019-01-27 00:52:51 +0000883 case TargetOpcode::G_MUL:
Petar Avramovic5229f472019-03-11 10:08:44 +0000884 case TargetOpcode::G_UMULH:
Petar Avramovic0b17e592019-03-11 10:00:17 +0000885 return narrowScalarMul(MI, NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +0000886 case TargetOpcode::G_EXTRACT:
887 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
888 case TargetOpcode::G_INSERT:
889 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000890 case TargetOpcode::G_LOAD: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000891 const auto &MMO = **MI.memoperands_begin();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000892 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000893 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000894 if (DstTy.isVector())
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000895 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000896
897 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000898 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000899 auto &MMO = **MI.memoperands_begin();
Jay Foad63f73542020-01-16 12:37:00 +0000900 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO);
Matt Arsenault18619af2019-01-29 18:13:02 +0000901 MIRBuilder.buildAnyExt(DstReg, TmpReg);
902 MI.eraseFromParent();
903 return Legalized;
904 }
905
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000906 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000907 }
Matt Arsenault6614f852019-01-22 19:02:10 +0000908 case TargetOpcode::G_ZEXTLOAD:
909 case TargetOpcode::G_SEXTLOAD: {
910 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000911 Register DstReg = MI.getOperand(0).getReg();
912 Register PtrReg = MI.getOperand(1).getReg();
Matt Arsenault6614f852019-01-22 19:02:10 +0000913
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000914 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault6614f852019-01-22 19:02:10 +0000915 auto &MMO = **MI.memoperands_begin();
Amara Emersond51adf02019-04-17 22:21:05 +0000916 if (MMO.getSizeInBits() == NarrowSize) {
Matt Arsenault6614f852019-01-22 19:02:10 +0000917 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
918 } else {
Jay Foad28bb43b2020-01-16 12:09:48 +0000919 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO);
Matt Arsenault6614f852019-01-22 19:02:10 +0000920 }
921
922 if (ZExt)
923 MIRBuilder.buildZExt(DstReg, TmpReg);
924 else
925 MIRBuilder.buildSExt(DstReg, TmpReg);
926
927 MI.eraseFromParent();
928 return Legalized;
929 }
Justin Bognerfde01042017-01-18 17:29:54 +0000930 case TargetOpcode::G_STORE: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000931 const auto &MMO = **MI.memoperands_begin();
Matt Arsenault18619af2019-01-29 18:13:02 +0000932
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000933 Register SrcReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000934 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000935 if (SrcTy.isVector())
936 return UnableToLegalize;
937
938 int NumParts = SizeOp0 / NarrowSize;
939 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
940 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
941 if (SrcTy.isVector() && LeftoverBits != 0)
942 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000943
944 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000945 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000946 auto &MMO = **MI.memoperands_begin();
947 MIRBuilder.buildTrunc(TmpReg, SrcReg);
Jay Foad63f73542020-01-16 12:37:00 +0000948 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO);
Matt Arsenault18619af2019-01-29 18:13:02 +0000949 MI.eraseFromParent();
950 return Legalized;
951 }
952
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000953 return reduceLoadStoreWidth(MI, 0, NarrowTy);
Justin Bognerfde01042017-01-18 17:29:54 +0000954 }
Matt Arsenault81511e52019-02-05 00:13:44 +0000955 case TargetOpcode::G_SELECT:
956 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
Petar Avramovic150fd432018-12-18 11:36:14 +0000957 case TargetOpcode::G_AND:
958 case TargetOpcode::G_OR:
959 case TargetOpcode::G_XOR: {
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000960 // Legalize bitwise operation:
961 // A = BinOp<Ty> B, C
962 // into:
963 // B1, ..., BN = G_UNMERGE_VALUES B
964 // C1, ..., CN = G_UNMERGE_VALUES C
965 // A1 = BinOp<Ty/N> B1, C2
966 // ...
967 // AN = BinOp<Ty/N> BN, CN
968 // A = G_MERGE_VALUES A1, ..., AN
Matt Arsenault9e0eeba2019-04-10 17:07:56 +0000969 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000970 }
Matt Arsenault30989e42019-01-22 21:42:11 +0000971 case TargetOpcode::G_SHL:
972 case TargetOpcode::G_LSHR:
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000973 case TargetOpcode::G_ASHR:
974 return narrowScalarShift(MI, TypeIdx, NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000975 case TargetOpcode::G_CTLZ:
976 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
977 case TargetOpcode::G_CTTZ:
978 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
979 case TargetOpcode::G_CTPOP:
Petar Avramovic2b66d322020-01-27 09:43:38 +0100980 if (TypeIdx == 1)
981 switch (MI.getOpcode()) {
982 case TargetOpcode::G_CTLZ:
983 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy);
Petar Avramovic8bc7ba52020-01-27 09:51:06 +0100984 case TargetOpcode::G_CTTZ:
985 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy);
Petar Avramoviccbf03aee2020-01-27 09:59:50 +0100986 case TargetOpcode::G_CTPOP:
987 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy);
Petar Avramovic2b66d322020-01-27 09:43:38 +0100988 default:
989 return UnableToLegalize;
990 }
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000991
992 Observer.changingInstr(MI);
993 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
994 Observer.changedInstr(MI);
995 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000996 case TargetOpcode::G_INTTOPTR:
997 if (TypeIdx != 1)
998 return UnableToLegalize;
999
1000 Observer.changingInstr(MI);
1001 narrowScalarSrc(MI, NarrowTy, 1);
1002 Observer.changedInstr(MI);
1003 return Legalized;
1004 case TargetOpcode::G_PTRTOINT:
1005 if (TypeIdx != 0)
1006 return UnableToLegalize;
1007
1008 Observer.changingInstr(MI);
1009 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1010 Observer.changedInstr(MI);
1011 return Legalized;
Petar Avramovicbe20e362019-07-09 14:36:17 +00001012 case TargetOpcode::G_PHI: {
1013 unsigned NumParts = SizeOp0 / NarrowSize;
1014 SmallVector<Register, 2> DstRegs;
1015 SmallVector<SmallVector<Register, 2>, 2> SrcRegs;
1016 DstRegs.resize(NumParts);
1017 SrcRegs.resize(MI.getNumOperands() / 2);
1018 Observer.changingInstr(MI);
1019 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1020 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
1021 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1022 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
1023 SrcRegs[i / 2]);
1024 }
1025 MachineBasicBlock &MBB = *MI.getParent();
1026 MIRBuilder.setInsertPt(MBB, MI);
1027 for (unsigned i = 0; i < NumParts; ++i) {
1028 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
1029 MachineInstrBuilder MIB =
1030 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
1031 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1032 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
1033 }
Amara Emerson02bcc862019-09-13 21:49:24 +00001034 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
Jay Foad63f73542020-01-16 12:37:00 +00001035 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
Petar Avramovicbe20e362019-07-09 14:36:17 +00001036 Observer.changedInstr(MI);
1037 MI.eraseFromParent();
1038 return Legalized;
1039 }
Matt Arsenault434d6642019-07-15 19:37:34 +00001040 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1041 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1042 if (TypeIdx != 2)
1043 return UnableToLegalize;
1044
1045 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1046 Observer.changingInstr(MI);
1047 narrowScalarSrc(MI, NarrowTy, OpIdx);
1048 Observer.changedInstr(MI);
1049 return Legalized;
1050 }
Petar Avramovic1e626352019-07-17 12:08:01 +00001051 case TargetOpcode::G_ICMP: {
1052 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
1053 if (NarrowSize * 2 != SrcSize)
1054 return UnableToLegalize;
1055
1056 Observer.changingInstr(MI);
1057 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
1058 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
Jay Foad63f73542020-01-16 12:37:00 +00001059 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2));
Petar Avramovic1e626352019-07-17 12:08:01 +00001060
1061 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
1062 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
Jay Foad63f73542020-01-16 12:37:00 +00001063 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3));
Petar Avramovic1e626352019-07-17 12:08:01 +00001064
1065 CmpInst::Predicate Pred =
1066 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
Amara Emersona1997ce2019-07-24 20:46:42 +00001067 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
Petar Avramovic1e626352019-07-17 12:08:01 +00001068
1069 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
1070 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
1071 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
1072 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
1073 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
Jay Foad63f73542020-01-16 12:37:00 +00001074 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero);
Petar Avramovic1e626352019-07-17 12:08:01 +00001075 } else {
Amara Emersona1997ce2019-07-24 20:46:42 +00001076 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
Petar Avramovic1e626352019-07-17 12:08:01 +00001077 MachineInstrBuilder CmpHEQ =
Amara Emersona1997ce2019-07-24 20:46:42 +00001078 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
Petar Avramovic1e626352019-07-17 12:08:01 +00001079 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
Amara Emersona1997ce2019-07-24 20:46:42 +00001080 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
Jay Foad63f73542020-01-16 12:37:00 +00001081 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH);
Petar Avramovic1e626352019-07-17 12:08:01 +00001082 }
1083 Observer.changedInstr(MI);
1084 MI.eraseFromParent();
1085 return Legalized;
1086 }
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001087 case TargetOpcode::G_SEXT_INREG: {
1088 if (TypeIdx != 0)
1089 return UnableToLegalize;
1090
1091 if (!MI.getOperand(2).isImm())
1092 return UnableToLegalize;
1093 int64_t SizeInBits = MI.getOperand(2).getImm();
1094
1095 // So long as the new type has more bits than the bits we're extending we
1096 // don't need to break it apart.
1097 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
1098 Observer.changingInstr(MI);
1099 // We don't lose any non-extension bits by truncating the src and
1100 // sign-extending the dst.
1101 MachineOperand &MO1 = MI.getOperand(1);
Jay Foad63f73542020-01-16 12:37:00 +00001102 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1);
Jay Foadb482e1b2020-01-23 11:51:35 +00001103 MO1.setReg(TruncMIB.getReg(0));
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001104
1105 MachineOperand &MO2 = MI.getOperand(0);
1106 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1107 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Jay Foad63f73542020-01-16 12:37:00 +00001108 MIRBuilder.buildSExt(MO2, DstExt);
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001109 MO2.setReg(DstExt);
1110 Observer.changedInstr(MI);
1111 return Legalized;
1112 }
1113
1114 // Break it apart. Components below the extension point are unmodified. The
1115 // component containing the extension point becomes a narrower SEXT_INREG.
1116 // Components above it are ashr'd from the component containing the
1117 // extension point.
1118 if (SizeOp0 % NarrowSize != 0)
1119 return UnableToLegalize;
1120 int NumParts = SizeOp0 / NarrowSize;
1121
1122 // List the registers where the destination will be scattered.
1123 SmallVector<Register, 2> DstRegs;
1124 // List the registers where the source will be split.
1125 SmallVector<Register, 2> SrcRegs;
1126
1127 // Create all the temporary registers.
1128 for (int i = 0; i < NumParts; ++i) {
1129 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1130
1131 SrcRegs.push_back(SrcReg);
1132 }
1133
1134 // Explode the big arguments into smaller chunks.
Jay Foad63f73542020-01-16 12:37:00 +00001135 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1));
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001136
1137 Register AshrCstReg =
1138 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
Jay Foadb482e1b2020-01-23 11:51:35 +00001139 .getReg(0);
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001140 Register FullExtensionReg = 0;
1141 Register PartialExtensionReg = 0;
1142
1143 // Do the operation on each small part.
1144 for (int i = 0; i < NumParts; ++i) {
1145 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1146 DstRegs.push_back(SrcRegs[i]);
1147 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1148 assert(PartialExtensionReg &&
1149 "Expected to visit partial extension before full");
1150 if (FullExtensionReg) {
1151 DstRegs.push_back(FullExtensionReg);
1152 continue;
1153 }
Jay Foad28bb43b2020-01-16 12:09:48 +00001154 DstRegs.push_back(
1155 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg)
Jay Foadb482e1b2020-01-23 11:51:35 +00001156 .getReg(0));
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001157 FullExtensionReg = DstRegs.back();
1158 } else {
1159 DstRegs.push_back(
1160 MIRBuilder
1161 .buildInstr(
1162 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1163 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
Jay Foadb482e1b2020-01-23 11:51:35 +00001164 .getReg(0));
Daniel Sanderse9a57c22019-08-09 21:11:20 +00001165 PartialExtensionReg = DstRegs.back();
1166 }
1167 }
1168
1169 // Gather the destination registers into the final destination.
1170 Register DstReg = MI.getOperand(0).getReg();
1171 MIRBuilder.buildMerge(DstReg, DstRegs);
1172 MI.eraseFromParent();
1173 return Legalized;
1174 }
Petar Avramovic98f72a52019-12-30 18:06:29 +01001175 case TargetOpcode::G_BSWAP:
1176 case TargetOpcode::G_BITREVERSE: {
Petar Avramovic94a24e72019-12-30 11:13:22 +01001177 if (SizeOp0 % NarrowSize != 0)
1178 return UnableToLegalize;
1179
1180 Observer.changingInstr(MI);
1181 SmallVector<Register, 2> SrcRegs, DstRegs;
1182 unsigned NumParts = SizeOp0 / NarrowSize;
1183 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
1184
1185 for (unsigned i = 0; i < NumParts; ++i) {
1186 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
1187 {SrcRegs[NumParts - 1 - i]});
1188 DstRegs.push_back(DstPart.getReg(0));
1189 }
1190
Jay Foad63f73542020-01-16 12:37:00 +00001191 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
Petar Avramovic94a24e72019-12-30 11:13:22 +01001192
1193 Observer.changedInstr(MI);
1194 MI.eraseFromParent();
1195 return Legalized;
1196 }
Tim Northover9656f142016-08-04 20:54:13 +00001197 }
Tim Northover33b07d62016-07-22 20:03:43 +00001198}
1199
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001200void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1201 unsigned OpIdx, unsigned ExtOpcode) {
1202 MachineOperand &MO = MI.getOperand(OpIdx);
Jay Foad63f73542020-01-16 12:37:00 +00001203 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
Jay Foadb482e1b2020-01-23 11:51:35 +00001204 MO.setReg(ExtB.getReg(0));
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001205}
1206
Matt Arsenault30989e42019-01-22 21:42:11 +00001207void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1208 unsigned OpIdx) {
1209 MachineOperand &MO = MI.getOperand(OpIdx);
Jay Foad63f73542020-01-16 12:37:00 +00001210 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO);
Jay Foadb482e1b2020-01-23 11:51:35 +00001211 MO.setReg(ExtB.getReg(0));
Matt Arsenault30989e42019-01-22 21:42:11 +00001212}
1213
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001214void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1215 unsigned OpIdx, unsigned TruncOpcode) {
1216 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001217 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001218 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Jay Foad63f73542020-01-16 12:37:00 +00001219 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001220 MO.setReg(DstExt);
1221}
1222
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001223void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1224 unsigned OpIdx, unsigned ExtOpcode) {
1225 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001226 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001227 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Jay Foad63f73542020-01-16 12:37:00 +00001228 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc});
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001229 MO.setReg(DstTrunc);
1230}
1231
Matt Arsenault18ec3822019-02-11 22:00:39 +00001232void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1233 unsigned OpIdx) {
1234 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001235 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +00001236 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Jay Foad63f73542020-01-16 12:37:00 +00001237 MIRBuilder.buildExtract(MO, DstExt, 0);
Matt Arsenault18ec3822019-02-11 22:00:39 +00001238 MO.setReg(DstExt);
1239}
1240
Matt Arsenault26b7e852019-02-19 16:30:19 +00001241void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1242 unsigned OpIdx) {
1243 MachineOperand &MO = MI.getOperand(OpIdx);
1244
1245 LLT OldTy = MRI.getType(MO.getReg());
1246 unsigned OldElts = OldTy.getNumElements();
1247 unsigned NewElts = MoreTy.getNumElements();
1248
1249 unsigned NumParts = NewElts / OldElts;
1250
1251 // Use concat_vectors if the result is a multiple of the number of elements.
1252 if (NumParts * OldElts == NewElts) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001253 SmallVector<Register, 8> Parts;
Matt Arsenault26b7e852019-02-19 16:30:19 +00001254 Parts.push_back(MO.getReg());
1255
Matt Arsenault3018d182019-06-28 01:47:44 +00001256 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +00001257 for (unsigned I = 1; I != NumParts; ++I)
1258 Parts.push_back(ImpDef);
1259
1260 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1261 MO.setReg(Concat.getReg(0));
1262 return;
1263 }
1264
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001265 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1266 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +00001267 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1268 MO.setReg(MoreReg);
1269}
1270
Tim Northover69fa84a2016-10-14 22:18:18 +00001271LegalizerHelper::LegalizeResult
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001272LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1273 LLT WideTy) {
1274 if (TypeIdx != 1)
1275 return UnableToLegalize;
1276
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001277 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001278 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault43cbca52019-07-03 23:08:06 +00001279 if (DstTy.isVector())
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001280 return UnableToLegalize;
1281
Matt Arsenaultc9f14f22019-07-01 19:36:10 +00001282 Register Src1 = MI.getOperand(1).getReg();
1283 LLT SrcTy = MRI.getType(Src1);
Matt Arsenault0966dd02019-07-17 20:22:44 +00001284 const int DstSize = DstTy.getSizeInBits();
1285 const int SrcSize = SrcTy.getSizeInBits();
1286 const int WideSize = WideTy.getSizeInBits();
1287 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
Matt Arsenaultc9f14f22019-07-01 19:36:10 +00001288
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001289 unsigned NumOps = MI.getNumOperands();
1290 unsigned NumSrc = MI.getNumOperands() - 1;
1291 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1292
Matt Arsenault0966dd02019-07-17 20:22:44 +00001293 if (WideSize >= DstSize) {
1294 // Directly pack the bits in the target type.
1295 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001296
Matt Arsenault0966dd02019-07-17 20:22:44 +00001297 for (unsigned I = 2; I != NumOps; ++I) {
1298 const unsigned Offset = (I - 1) * PartSize;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001299
Matt Arsenault0966dd02019-07-17 20:22:44 +00001300 Register SrcReg = MI.getOperand(I).getReg();
1301 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
1302
1303 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1304
Matt Arsenault5faa5332019-08-01 18:13:16 +00001305 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
Matt Arsenault0966dd02019-07-17 20:22:44 +00001306 MRI.createGenericVirtualRegister(WideTy);
1307
1308 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1309 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1310 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1311 ResultReg = NextResult;
1312 }
1313
1314 if (WideSize > DstSize)
1315 MIRBuilder.buildTrunc(DstReg, ResultReg);
Matt Arsenault5faa5332019-08-01 18:13:16 +00001316 else if (DstTy.isPointer())
1317 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
Matt Arsenault0966dd02019-07-17 20:22:44 +00001318
1319 MI.eraseFromParent();
1320 return Legalized;
1321 }
1322
1323 // Unmerge the original values to the GCD type, and recombine to the next
1324 // multiple greater than the original type.
1325 //
1326 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1327 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1328 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1329 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1330 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1331 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1332 // %12:_(s12) = G_MERGE_VALUES %10, %11
1333 //
1334 // Padding with undef if necessary:
1335 //
1336 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1337 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1338 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1339 // %7:_(s2) = G_IMPLICIT_DEF
1340 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1341 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1342 // %10:_(s12) = G_MERGE_VALUES %8, %9
1343
1344 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1345 LLT GCDTy = LLT::scalar(GCD);
1346
1347 SmallVector<Register, 8> Parts;
1348 SmallVector<Register, 8> NewMergeRegs;
1349 SmallVector<Register, 8> Unmerges;
1350 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1351
1352 // Decompose the original operands if they don't evenly divide.
1353 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001354 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenault0966dd02019-07-17 20:22:44 +00001355 if (GCD == SrcSize) {
1356 Unmerges.push_back(SrcReg);
1357 } else {
1358 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1359 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1360 Unmerges.push_back(Unmerge.getReg(J));
1361 }
1362 }
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001363
Matt Arsenault0966dd02019-07-17 20:22:44 +00001364 // Pad with undef to the next size that is a multiple of the requested size.
1365 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1366 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1367 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1368 Unmerges.push_back(UndefReg);
1369 }
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001370
Matt Arsenault0966dd02019-07-17 20:22:44 +00001371 const int PartsPerGCD = WideSize / GCD;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001372
Matt Arsenault0966dd02019-07-17 20:22:44 +00001373 // Build merges of each piece.
1374 ArrayRef<Register> Slicer(Unmerges);
1375 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1376 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1377 NewMergeRegs.push_back(Merge.getReg(0));
1378 }
1379
1380 // A truncate may be necessary if the requested type doesn't evenly divide the
1381 // original result type.
1382 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1383 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1384 } else {
1385 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1386 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001387 }
1388
1389 MI.eraseFromParent();
1390 return Legalized;
1391}
1392
1393LegalizerHelper::LegalizeResult
1394LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1395 LLT WideTy) {
1396 if (TypeIdx != 0)
1397 return UnableToLegalize;
1398
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001399 int NumDst = MI.getNumOperands() - 1;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001400 Register SrcReg = MI.getOperand(NumDst).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001401 LLT SrcTy = MRI.getType(SrcReg);
1402 if (!SrcTy.isScalar())
1403 return UnableToLegalize;
1404
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001405 Register Dst0Reg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001406 LLT DstTy = MRI.getType(Dst0Reg);
1407 if (!DstTy.isScalar())
1408 return UnableToLegalize;
1409
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001410 if (WideTy == SrcTy) {
1411 // Theres no unmerge type to target. Directly extract the bits from the
1412 // source type
1413 unsigned DstSize = DstTy.getSizeInBits();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001414
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001415 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
1416 for (int I = 1; I != NumDst; ++I) {
1417 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I);
1418 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt);
1419 MIRBuilder.buildTrunc(MI.getOperand(I), Shr);
1420 }
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001421
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001422 MI.eraseFromParent();
1423 return Legalized;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001424 }
1425
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001426 // TODO
1427 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
1428 return UnableToLegalize;
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001429
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001430 // Extend the source to a wider type.
1431 LLT LCMTy = getLCMType(SrcTy, WideTy);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001432
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001433 Register WideSrc = SrcReg;
1434 if (LCMTy != SrcTy)
1435 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0);
1436 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001437
Matt Arsenault2a160ba2020-01-21 09:02:42 -05001438 // Create a sequence of unmerges to the original results. since we may have
1439 // widened the source, we will need to pad the results with dead defs to cover
1440 // the source register.
1441 // e.g. widen s16 to s32:
1442 // %1:_(s16), %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0:_(s48)
1443 //
1444 // =>
1445 // %4:_(s64) = G_ANYEXT %0:_(s48)
1446 // %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4 ; Requested unmerge
1447 // %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %5 ; unpack to original regs
1448 // %3:_(s16), dead %7 = G_UNMERGE_VALUES %6 ; original reg + extra dead def
1449
1450 const int NumUnmerge = Unmerge->getNumOperands() - 1;
1451 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
1452
1453 for (int I = 0; I != NumUnmerge; ++I) {
1454 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
1455
1456 for (int J = 0; J != PartsPerUnmerge; ++J) {
1457 int Idx = I * PartsPerUnmerge + J;
1458 if (Idx < NumDst)
1459 MIB.addDef(MI.getOperand(Idx).getReg());
1460 else {
1461 // Create dead def for excess components.
1462 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
1463 }
1464 }
1465
1466 MIB.addUse(Unmerge.getReg(I));
1467 }
1468
1469 MI.eraseFromParent();
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001470 return Legalized;
1471}
1472
1473LegalizerHelper::LegalizeResult
Matt Arsenault1cf713662019-02-12 14:54:52 +00001474LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1475 LLT WideTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001476 Register DstReg = MI.getOperand(0).getReg();
1477 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00001478 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenaultfbe92a52019-02-18 22:39:27 +00001479
1480 LLT DstTy = MRI.getType(DstReg);
1481 unsigned Offset = MI.getOperand(2).getImm();
1482
1483 if (TypeIdx == 0) {
1484 if (SrcTy.isVector() || DstTy.isVector())
1485 return UnableToLegalize;
1486
1487 SrcOp Src(SrcReg);
1488 if (SrcTy.isPointer()) {
1489 // Extracts from pointers can be handled only if they are really just
1490 // simple integers.
1491 const DataLayout &DL = MIRBuilder.getDataLayout();
1492 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1493 return UnableToLegalize;
1494
1495 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1496 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1497 SrcTy = SrcAsIntTy;
1498 }
1499
1500 if (DstTy.isPointer())
1501 return UnableToLegalize;
1502
1503 if (Offset == 0) {
1504 // Avoid a shift in the degenerate case.
1505 MIRBuilder.buildTrunc(DstReg,
1506 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1507 MI.eraseFromParent();
1508 return Legalized;
1509 }
1510
1511 // Do a shift in the source type.
1512 LLT ShiftTy = SrcTy;
1513 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1514 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1515 ShiftTy = WideTy;
1516 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
1517 return UnableToLegalize;
1518
1519 auto LShr = MIRBuilder.buildLShr(
1520 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1521 MIRBuilder.buildTrunc(DstReg, LShr);
1522 MI.eraseFromParent();
1523 return Legalized;
1524 }
1525
Matt Arsenault8f624ab2019-04-22 15:10:42 +00001526 if (SrcTy.isScalar()) {
1527 Observer.changingInstr(MI);
1528 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1529 Observer.changedInstr(MI);
1530 return Legalized;
1531 }
1532
Matt Arsenault1cf713662019-02-12 14:54:52 +00001533 if (!SrcTy.isVector())
1534 return UnableToLegalize;
1535
Matt Arsenault1cf713662019-02-12 14:54:52 +00001536 if (DstTy != SrcTy.getElementType())
1537 return UnableToLegalize;
1538
Matt Arsenault1cf713662019-02-12 14:54:52 +00001539 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1540 return UnableToLegalize;
1541
1542 Observer.changingInstr(MI);
1543 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1544
1545 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1546 Offset);
1547 widenScalarDst(MI, WideTy.getScalarType(), 0);
1548 Observer.changedInstr(MI);
1549 return Legalized;
1550}
1551
1552LegalizerHelper::LegalizeResult
1553LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1554 LLT WideTy) {
1555 if (TypeIdx != 0)
1556 return UnableToLegalize;
1557 Observer.changingInstr(MI);
1558 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1559 widenScalarDst(MI, WideTy);
1560 Observer.changedInstr(MI);
1561 return Legalized;
1562}
1563
1564LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00001565LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Tim Northover3c73e362016-08-23 18:20:09 +00001566 MIRBuilder.setInstr(MI);
1567
Tim Northover32335812016-08-04 18:35:11 +00001568 switch (MI.getOpcode()) {
1569 default:
1570 return UnableToLegalize;
Matt Arsenault1cf713662019-02-12 14:54:52 +00001571 case TargetOpcode::G_EXTRACT:
1572 return widenScalarExtract(MI, TypeIdx, WideTy);
1573 case TargetOpcode::G_INSERT:
1574 return widenScalarInsert(MI, TypeIdx, WideTy);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001575 case TargetOpcode::G_MERGE_VALUES:
1576 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1577 case TargetOpcode::G_UNMERGE_VALUES:
1578 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001579 case TargetOpcode::G_UADDO:
1580 case TargetOpcode::G_USUBO: {
1581 if (TypeIdx == 1)
1582 return UnableToLegalize; // TODO
Jay Foad63f73542020-01-16 12:37:00 +00001583 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2));
1584 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3));
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001585 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1586 ? TargetOpcode::G_ADD
1587 : TargetOpcode::G_SUB;
1588 // Do the arithmetic in the larger type.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001589 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001590 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
Jay Foad885260d2020-01-16 14:36:41 +00001591 APInt Mask =
1592 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits());
Jay Foad28bb43b2020-01-16 12:09:48 +00001593 auto AndOp = MIRBuilder.buildAnd(
Jay Foad885260d2020-01-16 14:36:41 +00001594 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask));
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001595 // There is no overflow if the AndOp is the same as NewOp.
Jay Foad63f73542020-01-16 12:37:00 +00001596 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp);
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001597 // Now trunc the NewOp to the original result.
Jay Foad63f73542020-01-16 12:37:00 +00001598 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp);
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001599 MI.eraseFromParent();
1600 return Legalized;
1601 }
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001602 case TargetOpcode::G_CTTZ:
1603 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1604 case TargetOpcode::G_CTLZ:
1605 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1606 case TargetOpcode::G_CTPOP: {
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001607 if (TypeIdx == 0) {
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001608 Observer.changingInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001609 widenScalarDst(MI, WideTy, 0);
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001610 Observer.changedInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001611 return Legalized;
1612 }
1613
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001614 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001615
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001616 // First ZEXT the input.
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001617 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1618 LLT CurTy = MRI.getType(SrcReg);
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001619 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1620 // The count is the same in the larger type except if the original
1621 // value was zero. This can be handled by setting the bit just off
1622 // the top of the original type.
1623 auto TopBit =
1624 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001625 MIBSrc = MIRBuilder.buildOr(
1626 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001627 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001628
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001629 // Perform the operation at the larger size.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001630 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001631 // This is already the correct result for CTPOP and CTTZs
1632 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1633 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1634 // The correct result is NewOp - (Difference in widety and current ty).
1635 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
Jay Foad28bb43b2020-01-16 12:09:48 +00001636 MIBNewOp = MIRBuilder.buildSub(
1637 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001638 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001639
1640 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1641 MI.eraseFromParent();
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001642 return Legalized;
1643 }
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001644 case TargetOpcode::G_BSWAP: {
1645 Observer.changingInstr(MI);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001646 Register DstReg = MI.getOperand(0).getReg();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001647
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001648 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1649 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1650 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001651 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1652
1653 MI.getOperand(0).setReg(DstExt);
1654
1655 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1656
1657 LLT Ty = MRI.getType(DstReg);
1658 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1659 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
Jay Foad28bb43b2020-01-16 12:09:48 +00001660 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg);
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001661
1662 MIRBuilder.buildTrunc(DstReg, ShrReg);
1663 Observer.changedInstr(MI);
1664 return Legalized;
1665 }
Matt Arsenault5ff310e2019-09-04 20:46:15 +00001666 case TargetOpcode::G_BITREVERSE: {
1667 Observer.changingInstr(MI);
1668
1669 Register DstReg = MI.getOperand(0).getReg();
1670 LLT Ty = MRI.getType(DstReg);
1671 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1672
1673 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1674 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1675 MI.getOperand(0).setReg(DstExt);
1676 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1677
1678 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
1679 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
1680 MIRBuilder.buildTrunc(DstReg, Shift);
1681 Observer.changedInstr(MI);
1682 return Legalized;
1683 }
Tim Northover61c16142016-08-04 21:39:49 +00001684 case TargetOpcode::G_ADD:
1685 case TargetOpcode::G_AND:
1686 case TargetOpcode::G_MUL:
1687 case TargetOpcode::G_OR:
1688 case TargetOpcode::G_XOR:
Justin Bognerddb80ae2017-01-19 07:51:17 +00001689 case TargetOpcode::G_SUB:
Matt Arsenault1cf713662019-02-12 14:54:52 +00001690 // Perform operation at larger width (any extension is fines here, high bits
Tim Northover32335812016-08-04 18:35:11 +00001691 // don't affect the result) and then truncate the result back to the
1692 // original type.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001693 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001694 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1695 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1696 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001697 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001698 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001699
Roman Tereshin6d266382018-05-09 21:43:30 +00001700 case TargetOpcode::G_SHL:
Matt Arsenault012ecbb2019-05-16 04:08:46 +00001701 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001702
1703 if (TypeIdx == 0) {
1704 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1705 widenScalarDst(MI, WideTy);
1706 } else {
1707 assert(TypeIdx == 1);
1708 // The "number of bits to shift" operand must preserve its value as an
1709 // unsigned integer:
1710 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1711 }
1712
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001713 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001714 return Legalized;
1715
Tim Northover7a753d92016-08-26 17:46:06 +00001716 case TargetOpcode::G_SDIV:
Roman Tereshin27bba442018-05-09 01:43:12 +00001717 case TargetOpcode::G_SREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001718 case TargetOpcode::G_SMIN:
1719 case TargetOpcode::G_SMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001720 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001721 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1722 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1723 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001724 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001725 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001726
Roman Tereshin6d266382018-05-09 21:43:30 +00001727 case TargetOpcode::G_ASHR:
Matt Arsenault30989e42019-01-22 21:42:11 +00001728 case TargetOpcode::G_LSHR:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001729 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001730
1731 if (TypeIdx == 0) {
1732 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1733 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1734
1735 widenScalarSrc(MI, WideTy, 1, CvtOp);
1736 widenScalarDst(MI, WideTy);
1737 } else {
1738 assert(TypeIdx == 1);
1739 // The "number of bits to shift" operand must preserve its value as an
1740 // unsigned integer:
1741 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1742 }
1743
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001744 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001745 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001746 case TargetOpcode::G_UDIV:
1747 case TargetOpcode::G_UREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001748 case TargetOpcode::G_UMIN:
1749 case TargetOpcode::G_UMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001750 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001751 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1752 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1753 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001754 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001755 return Legalized;
1756
1757 case TargetOpcode::G_SELECT:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001758 Observer.changingInstr(MI);
Petar Avramovic09dff332018-12-25 14:42:30 +00001759 if (TypeIdx == 0) {
1760 // Perform operation at larger width (any extension is fine here, high
1761 // bits don't affect the result) and then truncate the result back to the
1762 // original type.
1763 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1764 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1765 widenScalarDst(MI, WideTy);
1766 } else {
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001767 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
Petar Avramovic09dff332018-12-25 14:42:30 +00001768 // Explicit extension is required here since high bits affect the result.
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001769 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
Petar Avramovic09dff332018-12-25 14:42:30 +00001770 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001771 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001772 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001773
Ahmed Bougachab6137062017-01-23 21:10:14 +00001774 case TargetOpcode::G_FPTOSI:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001775 case TargetOpcode::G_FPTOUI:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001776 Observer.changingInstr(MI);
Matt Arsenaulted85b0c2019-10-01 01:06:48 +00001777
1778 if (TypeIdx == 0)
1779 widenScalarDst(MI, WideTy);
1780 else
1781 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
1782
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001783 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001784 return Legalized;
Ahmed Bougachad2948232017-01-20 01:37:24 +00001785 case TargetOpcode::G_SITOFP:
Ahmed Bougachad2948232017-01-20 01:37:24 +00001786 if (TypeIdx != 1)
1787 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001788 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001789 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001790 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001791 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001792
1793 case TargetOpcode::G_UITOFP:
1794 if (TypeIdx != 1)
1795 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001796 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001797 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001798 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001799 return Legalized;
1800
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001801 case TargetOpcode::G_LOAD:
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001802 case TargetOpcode::G_SEXTLOAD:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001803 case TargetOpcode::G_ZEXTLOAD:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001804 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001805 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001806 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001807 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001808
Tim Northover3c73e362016-08-23 18:20:09 +00001809 case TargetOpcode::G_STORE: {
Matt Arsenault92c50012019-01-30 02:04:31 +00001810 if (TypeIdx != 0)
1811 return UnableToLegalize;
1812
1813 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1814 if (!isPowerOf2_32(Ty.getSizeInBits()))
Tim Northover548feee2017-03-21 22:22:05 +00001815 return UnableToLegalize;
1816
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001817 Observer.changingInstr(MI);
Matt Arsenault92c50012019-01-30 02:04:31 +00001818
1819 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1820 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1821 widenScalarSrc(MI, WideTy, 0, ExtType);
1822
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001823 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001824 return Legalized;
1825 }
Tim Northoverea904f92016-08-19 22:40:00 +00001826 case TargetOpcode::G_CONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001827 MachineOperand &SrcMO = MI.getOperand(1);
1828 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Aditya Nandakumar6da7dbb2019-12-03 10:40:03 -08001829 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
1830 MRI.getType(MI.getOperand(0).getReg()));
1831 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
1832 ExtOpc == TargetOpcode::G_ANYEXT) &&
1833 "Illegal Extend");
1834 const APInt &SrcVal = SrcMO.getCImm()->getValue();
1835 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
1836 ? SrcVal.sext(WideTy.getSizeInBits())
1837 : SrcVal.zext(WideTy.getSizeInBits());
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001838 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001839 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1840
1841 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001842 Observer.changedInstr(MI);
Tim Northoverea904f92016-08-19 22:40:00 +00001843 return Legalized;
1844 }
Tim Northovera11be042016-08-19 22:40:08 +00001845 case TargetOpcode::G_FCONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001846 MachineOperand &SrcMO = MI.getOperand(1);
Amara Emerson77a5c962018-01-27 07:07:20 +00001847 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001848 APFloat Val = SrcMO.getFPImm()->getValueAPF();
Amara Emerson77a5c962018-01-27 07:07:20 +00001849 bool LosesInfo;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001850 switch (WideTy.getSizeInBits()) {
1851 case 32:
Matt Arsenault996c6662019-02-12 14:54:54 +00001852 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1853 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001854 break;
1855 case 64:
Matt Arsenault996c6662019-02-12 14:54:54 +00001856 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1857 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001858 break;
1859 default:
Matt Arsenault996c6662019-02-12 14:54:54 +00001860 return UnableToLegalize;
Tim Northover6cd4b232016-08-23 21:01:26 +00001861 }
Matt Arsenault996c6662019-02-12 14:54:54 +00001862
1863 assert(!LosesInfo && "extend should always be lossless");
1864
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001865 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001866 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1867
1868 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001869 Observer.changedInstr(MI);
Roman Tereshin25cbfe62018-05-08 22:53:09 +00001870 return Legalized;
Roman Tereshin27bba442018-05-09 01:43:12 +00001871 }
Matt Arsenaultbefee402019-01-09 07:34:14 +00001872 case TargetOpcode::G_IMPLICIT_DEF: {
1873 Observer.changingInstr(MI);
1874 widenScalarDst(MI, WideTy);
1875 Observer.changedInstr(MI);
1876 return Legalized;
1877 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001878 case TargetOpcode::G_BRCOND:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001879 Observer.changingInstr(MI);
Petar Avramovic5d9b8ee2019-02-14 11:39:53 +00001880 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001881 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001882 return Legalized;
1883
1884 case TargetOpcode::G_FCMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001885 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001886 if (TypeIdx == 0)
1887 widenScalarDst(MI, WideTy);
1888 else {
1889 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1890 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
Roman Tereshin27bba442018-05-09 01:43:12 +00001891 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001892 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001893 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001894
1895 case TargetOpcode::G_ICMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001896 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001897 if (TypeIdx == 0)
1898 widenScalarDst(MI, WideTy);
1899 else {
1900 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1901 MI.getOperand(1).getPredicate()))
1902 ? TargetOpcode::G_SEXT
1903 : TargetOpcode::G_ZEXT;
1904 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1905 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1906 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001907 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001908 return Legalized;
1909
Daniel Sanderse74c5b92019-11-01 13:18:00 -07001910 case TargetOpcode::G_PTR_ADD:
1911 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001912 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001913 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001914 Observer.changedInstr(MI);
Tim Northover22d82cf2016-09-15 11:02:19 +00001915 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001916
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001917 case TargetOpcode::G_PHI: {
1918 assert(TypeIdx == 0 && "Expecting only Idx 0");
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001919
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001920 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001921 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1922 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1923 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1924 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001925 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001926
1927 MachineBasicBlock &MBB = *MI.getParent();
Amara Emerson9d647212019-09-16 23:46:03 +00001928 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001929 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001930 Observer.changedInstr(MI);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001931 return Legalized;
1932 }
Matt Arsenault63786292019-01-22 20:38:15 +00001933 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1934 if (TypeIdx == 0) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001935 Register VecReg = MI.getOperand(1).getReg();
Matt Arsenault63786292019-01-22 20:38:15 +00001936 LLT VecTy = MRI.getType(VecReg);
1937 Observer.changingInstr(MI);
1938
1939 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1940 WideTy.getSizeInBits()),
1941 1, TargetOpcode::G_SEXT);
1942
1943 widenScalarDst(MI, WideTy, 0);
1944 Observer.changedInstr(MI);
1945 return Legalized;
1946 }
1947
Amara Emersoncbd86d82018-10-25 14:04:54 +00001948 if (TypeIdx != 2)
1949 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001950 Observer.changingInstr(MI);
Matt Arsenault1a276d12019-10-01 15:51:37 -04001951 // TODO: Probably should be zext
Amara Emersoncbd86d82018-10-25 14:04:54 +00001952 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001953 Observer.changedInstr(MI);
Amara Emersoncbd86d82018-10-25 14:04:54 +00001954 return Legalized;
Matt Arsenault63786292019-01-22 20:38:15 +00001955 }
Matt Arsenault1a276d12019-10-01 15:51:37 -04001956 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1957 if (TypeIdx == 1) {
1958 Observer.changingInstr(MI);
1959
1960 Register VecReg = MI.getOperand(1).getReg();
1961 LLT VecTy = MRI.getType(VecReg);
1962 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy);
1963
1964 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
1965 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1966 widenScalarDst(MI, WideVecTy, 0);
1967 Observer.changedInstr(MI);
1968 return Legalized;
1969 }
1970
1971 if (TypeIdx == 2) {
1972 Observer.changingInstr(MI);
1973 // TODO: Probably should be zext
1974 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
1975 Observer.changedInstr(MI);
1976 }
1977
1978 return Legalized;
1979 }
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001980 case TargetOpcode::G_FADD:
1981 case TargetOpcode::G_FMUL:
1982 case TargetOpcode::G_FSUB:
1983 case TargetOpcode::G_FMA:
Matt Arsenaultcf103722019-09-06 20:49:10 +00001984 case TargetOpcode::G_FMAD:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001985 case TargetOpcode::G_FNEG:
1986 case TargetOpcode::G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00001987 case TargetOpcode::G_FCANONICALIZE:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00001988 case TargetOpcode::G_FMINNUM:
1989 case TargetOpcode::G_FMAXNUM:
1990 case TargetOpcode::G_FMINNUM_IEEE:
1991 case TargetOpcode::G_FMAXNUM_IEEE:
1992 case TargetOpcode::G_FMINIMUM:
1993 case TargetOpcode::G_FMAXIMUM:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001994 case TargetOpcode::G_FDIV:
1995 case TargetOpcode::G_FREM:
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001996 case TargetOpcode::G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00001997 case TargetOpcode::G_FFLOOR:
Jessica Paquette7db82d72019-01-28 18:34:18 +00001998 case TargetOpcode::G_FCOS:
1999 case TargetOpcode::G_FSIN:
Jessica Paquettec49428a2019-01-28 19:53:14 +00002000 case TargetOpcode::G_FLOG10:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +00002001 case TargetOpcode::G_FLOG:
Jessica Paquette0154bd12019-01-30 21:16:04 +00002002 case TargetOpcode::G_FLOG2:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00002003 case TargetOpcode::G_FRINT:
Jessica Paquetteba557672019-04-25 16:44:40 +00002004 case TargetOpcode::G_FNEARBYINT:
Jessica Paquette22457f82019-01-30 21:03:52 +00002005 case TargetOpcode::G_FSQRT:
Jessica Paquette84bedac2019-01-30 23:46:15 +00002006 case TargetOpcode::G_FEXP:
Jessica Paquettee7941212019-04-03 16:58:32 +00002007 case TargetOpcode::G_FEXP2:
Jessica Paquettedfd87f62019-04-19 16:28:08 +00002008 case TargetOpcode::G_FPOW:
Jessica Paquette56342642019-04-23 18:20:44 +00002009 case TargetOpcode::G_INTRINSIC_TRUNC:
Jessica Paquette3cc6d1f2019-04-23 21:11:57 +00002010 case TargetOpcode::G_INTRINSIC_ROUND:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00002011 assert(TypeIdx == 0);
Jessica Paquette453ab1d2018-12-21 17:05:26 +00002012 Observer.changingInstr(MI);
Matt Arsenault745fd9f2019-01-20 19:10:31 +00002013
2014 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
2015 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
2016
Jessica Paquette453ab1d2018-12-21 17:05:26 +00002017 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2018 Observer.changedInstr(MI);
2019 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002020 case TargetOpcode::G_INTTOPTR:
2021 if (TypeIdx != 1)
2022 return UnableToLegalize;
2023
2024 Observer.changingInstr(MI);
2025 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2026 Observer.changedInstr(MI);
2027 return Legalized;
2028 case TargetOpcode::G_PTRTOINT:
2029 if (TypeIdx != 0)
2030 return UnableToLegalize;
2031
2032 Observer.changingInstr(MI);
2033 widenScalarDst(MI, WideTy, 0);
2034 Observer.changedInstr(MI);
2035 return Legalized;
Matt Arsenaultbd791b52019-07-08 13:48:06 +00002036 case TargetOpcode::G_BUILD_VECTOR: {
2037 Observer.changingInstr(MI);
2038
2039 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
2040 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
2041 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
2042
2043 // Avoid changing the result vector type if the source element type was
2044 // requested.
2045 if (TypeIdx == 1) {
2046 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
2047 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
2048 } else {
2049 widenScalarDst(MI, WideTy, 0);
2050 }
2051
2052 Observer.changedInstr(MI);
2053 return Legalized;
2054 }
Daniel Sanderse9a57c22019-08-09 21:11:20 +00002055 case TargetOpcode::G_SEXT_INREG:
2056 if (TypeIdx != 0)
2057 return UnableToLegalize;
2058
2059 Observer.changingInstr(MI);
2060 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2061 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
2062 Observer.changedInstr(MI);
2063 return Legalized;
Tim Northover32335812016-08-04 18:35:11 +00002064 }
Tim Northover33b07d62016-07-22 20:03:43 +00002065}
2066
Matt Arsenault936483f2020-01-09 21:53:28 -05002067static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
2068 MachineIRBuilder &B, Register Src, LLT Ty) {
2069 auto Unmerge = B.buildUnmerge(Ty, Src);
2070 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
2071 Pieces.push_back(Unmerge.getReg(I));
2072}
2073
2074LegalizerHelper::LegalizeResult
2075LegalizerHelper::lowerBitcast(MachineInstr &MI) {
2076 Register Dst = MI.getOperand(0).getReg();
2077 Register Src = MI.getOperand(1).getReg();
2078 LLT DstTy = MRI.getType(Dst);
2079 LLT SrcTy = MRI.getType(Src);
2080
2081 if (SrcTy.isVector() && !DstTy.isVector()) {
2082 SmallVector<Register, 8> SrcRegs;
2083 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcTy.getElementType());
2084 MIRBuilder.buildMerge(Dst, SrcRegs);
2085 MI.eraseFromParent();
2086 return Legalized;
2087 }
2088
2089 if (DstTy.isVector() && !SrcTy.isVector()) {
2090 SmallVector<Register, 8> SrcRegs;
2091 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType());
2092 MIRBuilder.buildMerge(Dst, SrcRegs);
2093 MI.eraseFromParent();
2094 return Legalized;
2095 }
2096
2097 return UnableToLegalize;
2098}
2099
Tim Northover69fa84a2016-10-14 22:18:18 +00002100LegalizerHelper::LegalizeResult
2101LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Tim Northovercecee562016-08-26 17:46:13 +00002102 using namespace TargetOpcode;
Tim Northovercecee562016-08-26 17:46:13 +00002103 MIRBuilder.setInstr(MI);
2104
2105 switch(MI.getOpcode()) {
2106 default:
2107 return UnableToLegalize;
Matt Arsenault936483f2020-01-09 21:53:28 -05002108 case TargetOpcode::G_BITCAST:
2109 return lowerBitcast(MI);
Tim Northovercecee562016-08-26 17:46:13 +00002110 case TargetOpcode::G_SREM:
2111 case TargetOpcode::G_UREM: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002112 Register QuotReg = MRI.createGenericVirtualRegister(Ty);
Jay Foad63f73542020-01-16 12:37:00 +00002113 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {QuotReg},
2114 {MI.getOperand(1), MI.getOperand(2)});
Tim Northovercecee562016-08-26 17:46:13 +00002115
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002116 Register ProdReg = MRI.createGenericVirtualRegister(Ty);
Jay Foad63f73542020-01-16 12:37:00 +00002117 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2));
2118 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), ProdReg);
Tim Northovercecee562016-08-26 17:46:13 +00002119 MI.eraseFromParent();
2120 return Legalized;
2121 }
Matt Arsenault34ed76e2019-10-16 20:46:32 +00002122 case TargetOpcode::G_SADDO:
2123 case TargetOpcode::G_SSUBO:
2124 return lowerSADDO_SSUBO(MI);
Tim Northover0a9b2792017-02-08 21:22:15 +00002125 case TargetOpcode::G_SMULO:
2126 case TargetOpcode::G_UMULO: {
2127 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
2128 // result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002129 Register Res = MI.getOperand(0).getReg();
2130 Register Overflow = MI.getOperand(1).getReg();
2131 Register LHS = MI.getOperand(2).getReg();
2132 Register RHS = MI.getOperand(3).getReg();
Tim Northover0a9b2792017-02-08 21:22:15 +00002133
Tim Northover0a9b2792017-02-08 21:22:15 +00002134 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
2135 ? TargetOpcode::G_SMULH
2136 : TargetOpcode::G_UMULH;
2137
Jay Foadf465b1a2020-01-16 14:46:36 +00002138 Observer.changingInstr(MI);
2139 const auto &TII = MIRBuilder.getTII();
2140 MI.setDesc(TII.get(TargetOpcode::G_MUL));
2141 MI.RemoveOperand(1);
2142 Observer.changedInstr(MI);
2143
2144 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
2145
2146 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS});
Tim Northover0a9b2792017-02-08 21:22:15 +00002147
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002148 Register Zero = MRI.createGenericVirtualRegister(Ty);
Tim Northover0a9b2792017-02-08 21:22:15 +00002149 MIRBuilder.buildConstant(Zero, 0);
Amara Emerson9de62132018-01-03 04:56:56 +00002150
2151 // For *signed* multiply, overflow is detected by checking:
2152 // (hi != (lo >> bitwidth-1))
2153 if (Opcode == TargetOpcode::G_SMULH) {
Jay Foadf465b1a2020-01-16 14:46:36 +00002154 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1);
2155 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt);
Amara Emerson9de62132018-01-03 04:56:56 +00002156 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
2157 } else {
2158 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
2159 }
Tim Northover0a9b2792017-02-08 21:22:15 +00002160 return Legalized;
2161 }
Volkan Keles5698b2a2017-03-08 18:09:14 +00002162 case TargetOpcode::G_FNEG: {
2163 // TODO: Handle vector types once we are able to
2164 // represent them.
2165 if (Ty.isVector())
2166 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002167 Register Res = MI.getOperand(0).getReg();
Volkan Keles5698b2a2017-03-08 18:09:14 +00002168 Type *ZeroTy;
Matthias Braunf1caa282017-12-15 22:22:58 +00002169 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Volkan Keles5698b2a2017-03-08 18:09:14 +00002170 switch (Ty.getSizeInBits()) {
2171 case 16:
2172 ZeroTy = Type::getHalfTy(Ctx);
2173 break;
2174 case 32:
2175 ZeroTy = Type::getFloatTy(Ctx);
2176 break;
2177 case 64:
2178 ZeroTy = Type::getDoubleTy(Ctx);
2179 break;
Amara Emersonb6ddbef2017-12-19 17:21:35 +00002180 case 128:
2181 ZeroTy = Type::getFP128Ty(Ctx);
2182 break;
Volkan Keles5698b2a2017-03-08 18:09:14 +00002183 default:
2184 llvm_unreachable("unexpected floating-point type");
2185 }
2186 ConstantFP &ZeroForNegation =
2187 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
Volkan Keles02bb1742018-02-14 19:58:36 +00002188 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002189 Register SubByReg = MI.getOperand(1).getReg();
Jay Foadb482e1b2020-01-23 11:51:35 +00002190 Register ZeroReg = Zero.getReg(0);
Jay Foad28bb43b2020-01-16 12:09:48 +00002191 MIRBuilder.buildFSub(Res, ZeroReg, SubByReg, MI.getFlags());
Volkan Keles5698b2a2017-03-08 18:09:14 +00002192 MI.eraseFromParent();
2193 return Legalized;
2194 }
Volkan Keles225921a2017-03-10 21:25:09 +00002195 case TargetOpcode::G_FSUB: {
2196 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
2197 // First, check if G_FNEG is marked as Lower. If so, we may
2198 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
Daniel Sanders9ade5592018-01-29 17:37:29 +00002199 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
Volkan Keles225921a2017-03-10 21:25:09 +00002200 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002201 Register Res = MI.getOperand(0).getReg();
2202 Register LHS = MI.getOperand(1).getReg();
2203 Register RHS = MI.getOperand(2).getReg();
2204 Register Neg = MRI.createGenericVirtualRegister(Ty);
Jay Foad28bb43b2020-01-16 12:09:48 +00002205 MIRBuilder.buildFNeg(Neg, RHS);
2206 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags());
Volkan Keles225921a2017-03-10 21:25:09 +00002207 MI.eraseFromParent();
2208 return Legalized;
2209 }
Matt Arsenault4d339182019-09-13 00:44:35 +00002210 case TargetOpcode::G_FMAD:
2211 return lowerFMad(MI);
Matt Arsenaultf3de8ab2019-12-24 14:49:31 -05002212 case TargetOpcode::G_INTRINSIC_ROUND:
2213 return lowerIntrinsicRound(MI);
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00002214 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002215 Register OldValRes = MI.getOperand(0).getReg();
2216 Register SuccessRes = MI.getOperand(1).getReg();
2217 Register Addr = MI.getOperand(2).getReg();
2218 Register CmpVal = MI.getOperand(3).getReg();
2219 Register NewVal = MI.getOperand(4).getReg();
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00002220 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
2221 **MI.memoperands_begin());
2222 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
2223 MI.eraseFromParent();
2224 return Legalized;
2225 }
Daniel Sanders5eb9f582018-04-28 18:14:50 +00002226 case TargetOpcode::G_LOAD:
2227 case TargetOpcode::G_SEXTLOAD:
2228 case TargetOpcode::G_ZEXTLOAD: {
2229 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002230 Register DstReg = MI.getOperand(0).getReg();
2231 Register PtrReg = MI.getOperand(1).getReg();
Daniel Sanders5eb9f582018-04-28 18:14:50 +00002232 LLT DstTy = MRI.getType(DstReg);
2233 auto &MMO = **MI.memoperands_begin();
2234
Amara Emersonc8351642019-08-02 23:44:24 +00002235 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
2236 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
2237 // This load needs splitting into power of 2 sized loads.
2238 if (DstTy.isVector())
Daniel Sanders2de9d4a2018-04-30 17:20:01 +00002239 return UnableToLegalize;
Amara Emersonc8351642019-08-02 23:44:24 +00002240 if (isPowerOf2_32(DstTy.getSizeInBits()))
2241 return UnableToLegalize; // Don't know what we're being asked to do.
2242
2243 // Our strategy here is to generate anyextending loads for the smaller
2244 // types up to next power-2 result type, and then combine the two larger
2245 // result values together, before truncating back down to the non-pow-2
2246 // type.
2247 // E.g. v1 = i24 load =>
2248 // v2 = i32 load (2 byte)
2249 // v3 = i32 load (1 byte)
2250 // v4 = i32 shl v3, 16
2251 // v5 = i32 or v4, v2
2252 // v1 = i24 trunc v5
2253 // By doing this we generate the correct truncate which should get
2254 // combined away as an artifact with a matching extend.
2255 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
2256 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
2257
2258 MachineFunction &MF = MIRBuilder.getMF();
2259 MachineMemOperand *LargeMMO =
2260 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2261 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
2262 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2263
2264 LLT PtrTy = MRI.getType(PtrReg);
2265 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
2266 LLT AnyExtTy = LLT::scalar(AnyExtSize);
2267 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2268 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2269 auto LargeLoad =
2270 MIRBuilder.buildLoad(LargeLdReg, PtrReg, *LargeMMO);
2271
Dominik Montada9965b122020-01-27 09:35:59 -05002272 auto OffsetCst = MIRBuilder.buildConstant(
2273 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
Daniel Sanderse74c5b92019-11-01 13:18:00 -07002274 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2275 auto SmallPtr =
2276 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
Amara Emersonc8351642019-08-02 23:44:24 +00002277 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
2278 *SmallMMO);
2279
2280 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
2281 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
2282 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
2283 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
2284 MI.eraseFromParent();
2285 return Legalized;
2286 }
Daniel Sanders5eb9f582018-04-28 18:14:50 +00002287 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
2288 MI.eraseFromParent();
2289 return Legalized;
2290 }
2291
2292 if (DstTy.isScalar()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002293 Register TmpReg =
Amara Emersond51adf02019-04-17 22:21:05 +00002294 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
Daniel Sanders5eb9f582018-04-28 18:14:50 +00002295 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
2296 switch (MI.getOpcode()) {
2297 default:
2298 llvm_unreachable("Unexpected opcode");
2299 case TargetOpcode::G_LOAD:
Amara Emerson28f5ad52019-12-04 17:01:07 -08002300 MIRBuilder.buildExtOrTrunc(TargetOpcode::G_ANYEXT, DstReg, TmpReg);
Daniel Sanders5eb9f582018-04-28 18:14:50 +00002301 break;
2302 case TargetOpcode::G_SEXTLOAD:
2303 MIRBuilder.buildSExt(DstReg, TmpReg);
2304 break;
2305 case TargetOpcode::G_ZEXTLOAD:
2306 MIRBuilder.buildZExt(DstReg, TmpReg);
2307 break;
2308 }
2309 MI.eraseFromParent();
2310 return Legalized;
2311 }
2312
2313 return UnableToLegalize;
2314 }
Amara Emersonc8351642019-08-02 23:44:24 +00002315 case TargetOpcode::G_STORE: {
2316 // Lower a non-power of 2 store into multiple pow-2 stores.
2317 // E.g. split an i24 store into an i16 store + i8 store.
2318 // We do this by first extending the stored value to the next largest power
2319 // of 2 type, and then using truncating stores to store the components.
2320 // By doing this, likewise with G_LOAD, generate an extend that can be
2321 // artifact-combined away instead of leaving behind extracts.
2322 Register SrcReg = MI.getOperand(0).getReg();
2323 Register PtrReg = MI.getOperand(1).getReg();
2324 LLT SrcTy = MRI.getType(SrcReg);
2325 MachineMemOperand &MMO = **MI.memoperands_begin();
2326 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2327 return UnableToLegalize;
2328 if (SrcTy.isVector())
2329 return UnableToLegalize;
2330 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2331 return UnableToLegalize; // Don't know what we're being asked to do.
2332
2333 // Extend to the next pow-2.
2334 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2335 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2336
2337 // Obtain the smaller value by shifting away the larger value.
2338 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2339 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2340 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2341 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2342
Daniel Sanderse74c5b92019-11-01 13:18:00 -07002343 // Generate the PtrAdd and truncating stores.
Amara Emersonc8351642019-08-02 23:44:24 +00002344 LLT PtrTy = MRI.getType(PtrReg);
Dominik Montadadc141af2020-01-30 08:25:10 -05002345 auto OffsetCst = MIRBuilder.buildConstant(
2346 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
Daniel Sanderse74c5b92019-11-01 13:18:00 -07002347 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2348 auto SmallPtr =
2349 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
Amara Emersonc8351642019-08-02 23:44:24 +00002350
2351 MachineFunction &MF = MIRBuilder.getMF();
2352 MachineMemOperand *LargeMMO =
2353 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2354 MachineMemOperand *SmallMMO =
2355 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2356 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2357 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2358 MI.eraseFromParent();
2359 return Legalized;
2360 }
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002361 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2362 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2363 case TargetOpcode::G_CTLZ:
2364 case TargetOpcode::G_CTTZ:
2365 case TargetOpcode::G_CTPOP:
2366 return lowerBitCount(MI, TypeIdx, Ty);
Petar Avramovicbd395692019-02-26 17:22:42 +00002367 case G_UADDO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002368 Register Res = MI.getOperand(0).getReg();
2369 Register CarryOut = MI.getOperand(1).getReg();
2370 Register LHS = MI.getOperand(2).getReg();
2371 Register RHS = MI.getOperand(3).getReg();
Petar Avramovicbd395692019-02-26 17:22:42 +00002372
2373 MIRBuilder.buildAdd(Res, LHS, RHS);
2374 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2375
2376 MI.eraseFromParent();
2377 return Legalized;
2378 }
Petar Avramovicb8276f22018-12-17 12:31:07 +00002379 case G_UADDE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002380 Register Res = MI.getOperand(0).getReg();
2381 Register CarryOut = MI.getOperand(1).getReg();
2382 Register LHS = MI.getOperand(2).getReg();
2383 Register RHS = MI.getOperand(3).getReg();
2384 Register CarryIn = MI.getOperand(4).getReg();
Petar Avramovicb8276f22018-12-17 12:31:07 +00002385
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002386 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2387 Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
Petar Avramovicb8276f22018-12-17 12:31:07 +00002388
2389 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
2390 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
2391 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
2392 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
2393
2394 MI.eraseFromParent();
2395 return Legalized;
2396 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002397 case G_USUBO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002398 Register Res = MI.getOperand(0).getReg();
2399 Register BorrowOut = MI.getOperand(1).getReg();
2400 Register LHS = MI.getOperand(2).getReg();
2401 Register RHS = MI.getOperand(3).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002402
2403 MIRBuilder.buildSub(Res, LHS, RHS);
2404 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
2405
2406 MI.eraseFromParent();
2407 return Legalized;
2408 }
2409 case G_USUBE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002410 Register Res = MI.getOperand(0).getReg();
2411 Register BorrowOut = MI.getOperand(1).getReg();
2412 Register LHS = MI.getOperand(2).getReg();
2413 Register RHS = MI.getOperand(3).getReg();
2414 Register BorrowIn = MI.getOperand(4).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002415
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002416 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
2417 Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
2418 Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
2419 Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +00002420
2421 MIRBuilder.buildSub(TmpRes, LHS, RHS);
2422 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
2423 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
2424 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
2425 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
2426 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
2427
2428 MI.eraseFromParent();
2429 return Legalized;
2430 }
Matt Arsenault02b5ca82019-05-17 23:05:13 +00002431 case G_UITOFP:
2432 return lowerUITOFP(MI, TypeIdx, Ty);
2433 case G_SITOFP:
2434 return lowerSITOFP(MI, TypeIdx, Ty);
Petar Avramovic6412b562019-08-30 05:44:02 +00002435 case G_FPTOUI:
2436 return lowerFPTOUI(MI, TypeIdx, Ty);
Matt Arsenaultea956682020-01-04 17:09:48 -05002437 case G_FPTOSI:
2438 return lowerFPTOSI(MI);
Matt Arsenault6f74f552019-07-01 17:18:03 +00002439 case G_SMIN:
2440 case G_SMAX:
2441 case G_UMIN:
2442 case G_UMAX:
2443 return lowerMinMax(MI, TypeIdx, Ty);
Matt Arsenaultb1843e12019-07-09 23:34:29 +00002444 case G_FCOPYSIGN:
2445 return lowerFCopySign(MI, TypeIdx, Ty);
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00002446 case G_FMINNUM:
2447 case G_FMAXNUM:
2448 return lowerFMinNumMaxNum(MI);
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00002449 case G_UNMERGE_VALUES:
2450 return lowerUnmergeValues(MI);
Daniel Sanderse9a57c22019-08-09 21:11:20 +00002451 case TargetOpcode::G_SEXT_INREG: {
2452 assert(MI.getOperand(2).isImm() && "Expected immediate");
2453 int64_t SizeInBits = MI.getOperand(2).getImm();
2454
2455 Register DstReg = MI.getOperand(0).getReg();
2456 Register SrcReg = MI.getOperand(1).getReg();
2457 LLT DstTy = MRI.getType(DstReg);
2458 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
2459
2460 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
Jay Foad63f73542020-01-16 12:37:00 +00002461 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0));
2462 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0));
Daniel Sanderse9a57c22019-08-09 21:11:20 +00002463 MI.eraseFromParent();
2464 return Legalized;
2465 }
Matt Arsenault690645b2019-08-13 16:09:07 +00002466 case G_SHUFFLE_VECTOR:
2467 return lowerShuffleVector(MI);
Amara Emersone20b91c2019-08-27 19:54:27 +00002468 case G_DYN_STACKALLOC:
2469 return lowerDynStackAlloc(MI);
Matt Arsenaulta5b9c752019-10-06 01:37:35 +00002470 case G_EXTRACT:
2471 return lowerExtract(MI);
Matt Arsenault4bcdcad2019-10-07 19:13:27 +00002472 case G_INSERT:
2473 return lowerInsert(MI);
Petar Avramovic94a24e72019-12-30 11:13:22 +01002474 case G_BSWAP:
2475 return lowerBswap(MI);
Petar Avramovic98f72a52019-12-30 18:06:29 +01002476 case G_BITREVERSE:
2477 return lowerBitreverse(MI);
Matt Arsenault0ea3c722019-12-27 19:26:51 -05002478 case G_READ_REGISTER:
Matt Arsenaultc5c1bb32020-01-12 13:29:44 -05002479 case G_WRITE_REGISTER:
2480 return lowerReadWriteRegister(MI);
Tim Northovercecee562016-08-26 17:46:13 +00002481 }
2482}
2483
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002484LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
2485 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002486 SmallVector<Register, 2> DstRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002487
2488 unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002489 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002490 unsigned Size = MRI.getType(DstReg).getSizeInBits();
2491 int NumParts = Size / NarrowSize;
2492 // FIXME: Don't know how to handle the situation where the small vectors
2493 // aren't all the same size yet.
2494 if (Size % NarrowSize != 0)
2495 return UnableToLegalize;
2496
2497 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002498 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002499 MIRBuilder.buildUndef(TmpReg);
2500 DstRegs.push_back(TmpReg);
2501 }
2502
2503 if (NarrowTy.isVector())
2504 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2505 else
2506 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2507
2508 MI.eraseFromParent();
2509 return Legalized;
2510}
2511
2512LegalizerHelper::LegalizeResult
2513LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
2514 LLT NarrowTy) {
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002515 const unsigned Opc = MI.getOpcode();
2516 const unsigned NumOps = MI.getNumOperands() - 1;
2517 const unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenault3018d182019-06-28 01:47:44 +00002518 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002519 const unsigned Flags = MI.getFlags();
2520 const LLT DstTy = MRI.getType(DstReg);
2521 const unsigned Size = DstTy.getSizeInBits();
2522 const int NumParts = Size / NarrowSize;
2523 const LLT EltTy = DstTy.getElementType();
2524 const unsigned EltSize = EltTy.getSizeInBits();
2525 const unsigned BitsForNumParts = NarrowSize * NumParts;
2526
2527 // Check if we have any leftovers. If we do, then only handle the case where
2528 // the leftover is one element.
2529 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002530 return UnableToLegalize;
2531
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002532 if (BitsForNumParts != Size) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002533 Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002534 MIRBuilder.buildUndef(AccumDstReg);
2535
2536 // Handle the pieces which evenly divide into the requested type with
2537 // extract/op/insert sequence.
2538 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
2539 SmallVector<SrcOp, 4> SrcOps;
2540 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002541 Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
Jay Foad63f73542020-01-16 12:37:00 +00002542 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I), Offset);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002543 SrcOps.push_back(PartOpReg);
2544 }
2545
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002546 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002547 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2548
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002549 Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002550 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
2551 AccumDstReg = PartInsertReg;
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002552 }
2553
2554 // Handle the remaining element sized leftover piece.
2555 SmallVector<SrcOp, 4> SrcOps;
2556 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002557 Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
Jay Foad63f73542020-01-16 12:37:00 +00002558 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I), BitsForNumParts);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002559 SrcOps.push_back(PartOpReg);
2560 }
2561
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002562 Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00002563 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
2564 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
2565 MI.eraseFromParent();
2566
2567 return Legalized;
2568 }
2569
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002570 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002571
2572 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
2573
2574 if (NumOps >= 2)
2575 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
2576
2577 if (NumOps >= 3)
2578 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
2579
2580 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002581 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002582
2583 if (NumOps == 1)
2584 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
2585 else if (NumOps == 2) {
2586 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
2587 } else if (NumOps == 3) {
2588 MIRBuilder.buildInstr(Opc, {DstReg},
2589 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
2590 }
2591
2592 DstRegs.push_back(DstReg);
2593 }
2594
2595 if (NarrowTy.isVector())
2596 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2597 else
2598 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2599
2600 MI.eraseFromParent();
2601 return Legalized;
2602}
2603
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002604// Handle splitting vector operations which need to have the same number of
2605// elements in each type index, but each type index may have a different element
2606// type.
2607//
2608// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
2609// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2610// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2611//
2612// Also handles some irregular breakdown cases, e.g.
2613// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
2614// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
2615// s64 = G_SHL s64, s32
2616LegalizerHelper::LegalizeResult
2617LegalizerHelper::fewerElementsVectorMultiEltType(
2618 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
2619 if (TypeIdx != 0)
2620 return UnableToLegalize;
2621
2622 const LLT NarrowTy0 = NarrowTyArg;
2623 const unsigned NewNumElts =
2624 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
2625
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002626 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002627 LLT DstTy = MRI.getType(DstReg);
2628 LLT LeftoverTy0;
2629
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002630 // All of the operands need to have the same number of elements, so if we can
2631 // determine a type breakdown for the result type, we can for all of the
2632 // source types.
Fangrui Songb251cc02019-07-12 14:58:15 +00002633 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002634 if (NumParts < 0)
2635 return UnableToLegalize;
2636
2637 SmallVector<MachineInstrBuilder, 4> NewInsts;
2638
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002639 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
2640 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002641
2642 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
2643 LLT LeftoverTy;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002644 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002645 LLT SrcTyI = MRI.getType(SrcReg);
2646 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
2647 LLT LeftoverTyI;
2648
2649 // Split this operand into the requested typed registers, and any leftover
2650 // required to reproduce the original type.
2651 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
2652 LeftoverRegs))
2653 return UnableToLegalize;
2654
2655 if (I == 1) {
2656 // For the first operand, create an instruction for each part and setup
2657 // the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002658 for (Register PartReg : PartRegs) {
2659 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002660 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2661 .addDef(PartDstReg)
2662 .addUse(PartReg));
2663 DstRegs.push_back(PartDstReg);
2664 }
2665
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002666 for (Register LeftoverReg : LeftoverRegs) {
2667 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002668 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
2669 .addDef(PartDstReg)
2670 .addUse(LeftoverReg));
2671 LeftoverDstRegs.push_back(PartDstReg);
2672 }
2673 } else {
2674 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
2675
2676 // Add the newly created operand splits to the existing instructions. The
2677 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2678 // pieces.
2679 unsigned InstCount = 0;
2680 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
2681 NewInsts[InstCount++].addUse(PartRegs[J]);
2682 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
2683 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
2684 }
2685
2686 PartRegs.clear();
2687 LeftoverRegs.clear();
2688 }
2689
2690 // Insert the newly built operations and rebuild the result register.
2691 for (auto &MIB : NewInsts)
2692 MIRBuilder.insertInstr(MIB);
2693
2694 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
2695
2696 MI.eraseFromParent();
2697 return Legalized;
2698}
2699
Tim Northover69fa84a2016-10-14 22:18:18 +00002700LegalizerHelper::LegalizeResult
Matt Arsenaultca676342019-01-25 02:36:32 +00002701LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
2702 LLT NarrowTy) {
2703 if (TypeIdx != 0)
2704 return UnableToLegalize;
2705
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002706 Register DstReg = MI.getOperand(0).getReg();
2707 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenaultca676342019-01-25 02:36:32 +00002708 LLT DstTy = MRI.getType(DstReg);
2709 LLT SrcTy = MRI.getType(SrcReg);
2710
2711 LLT NarrowTy0 = NarrowTy;
2712 LLT NarrowTy1;
2713 unsigned NumParts;
2714
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002715 if (NarrowTy.isVector()) {
Matt Arsenaultca676342019-01-25 02:36:32 +00002716 // Uneven breakdown not handled.
2717 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
2718 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
2719 return UnableToLegalize;
2720
2721 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002722 } else {
2723 NumParts = DstTy.getNumElements();
2724 NarrowTy1 = SrcTy.getElementType();
Matt Arsenaultca676342019-01-25 02:36:32 +00002725 }
2726
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002727 SmallVector<Register, 4> SrcRegs, DstRegs;
Matt Arsenaultca676342019-01-25 02:36:32 +00002728 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
2729
2730 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002731 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Jay Foad28bb43b2020-01-16 12:09:48 +00002732 MachineInstr *NewInst =
2733 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]});
Matt Arsenaultca676342019-01-25 02:36:32 +00002734
2735 NewInst->setFlags(MI.getFlags());
2736 DstRegs.push_back(DstReg);
2737 }
2738
2739 if (NarrowTy.isVector())
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002740 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002741 else
2742 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2743
2744 MI.eraseFromParent();
2745 return Legalized;
2746}
2747
2748LegalizerHelper::LegalizeResult
2749LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
2750 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002751 Register DstReg = MI.getOperand(0).getReg();
2752 Register Src0Reg = MI.getOperand(2).getReg();
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002753 LLT DstTy = MRI.getType(DstReg);
2754 LLT SrcTy = MRI.getType(Src0Reg);
2755
2756 unsigned NumParts;
2757 LLT NarrowTy0, NarrowTy1;
2758
2759 if (TypeIdx == 0) {
2760 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2761 unsigned OldElts = DstTy.getNumElements();
2762
2763 NarrowTy0 = NarrowTy;
2764 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
2765 NarrowTy1 = NarrowTy.isVector() ?
2766 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
2767 SrcTy.getElementType();
2768
2769 } else {
2770 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2771 unsigned OldElts = SrcTy.getNumElements();
2772
2773 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2774 NarrowTy.getNumElements();
2775 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2776 DstTy.getScalarSizeInBits());
2777 NarrowTy1 = NarrowTy;
2778 }
2779
2780 // FIXME: Don't know how to handle the situation where the small vectors
2781 // aren't all the same size yet.
2782 if (NarrowTy1.isVector() &&
2783 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2784 return UnableToLegalize;
2785
2786 CmpInst::Predicate Pred
2787 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2788
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002789 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002790 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2791 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2792
2793 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002794 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002795 DstRegs.push_back(DstReg);
2796
2797 if (MI.getOpcode() == TargetOpcode::G_ICMP)
2798 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2799 else {
2800 MachineInstr *NewCmp
2801 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2802 NewCmp->setFlags(MI.getFlags());
2803 }
2804 }
2805
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002806 if (NarrowTy1.isVector())
Matt Arsenaultca676342019-01-25 02:36:32 +00002807 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2808 else
2809 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2810
2811 MI.eraseFromParent();
2812 return Legalized;
2813}
2814
2815LegalizerHelper::LegalizeResult
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002816LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
2817 LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002818 Register DstReg = MI.getOperand(0).getReg();
2819 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002820
2821 unsigned NumParts = 0;
2822 LLT NarrowTy0, NarrowTy1;
2823
2824 LLT DstTy = MRI.getType(DstReg);
2825 LLT CondTy = MRI.getType(CondReg);
2826 unsigned Size = DstTy.getSizeInBits();
2827
2828 assert(TypeIdx == 0 || CondTy.isVector());
2829
2830 if (TypeIdx == 0) {
2831 NarrowTy0 = NarrowTy;
2832 NarrowTy1 = CondTy;
2833
2834 unsigned NarrowSize = NarrowTy0.getSizeInBits();
2835 // FIXME: Don't know how to handle the situation where the small vectors
2836 // aren't all the same size yet.
2837 if (Size % NarrowSize != 0)
2838 return UnableToLegalize;
2839
2840 NumParts = Size / NarrowSize;
2841
2842 // Need to break down the condition type
2843 if (CondTy.isVector()) {
2844 if (CondTy.getNumElements() == NumParts)
2845 NarrowTy1 = CondTy.getElementType();
2846 else
2847 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2848 CondTy.getScalarSizeInBits());
2849 }
2850 } else {
2851 NumParts = CondTy.getNumElements();
2852 if (NarrowTy.isVector()) {
2853 // TODO: Handle uneven breakdown.
2854 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2855 return UnableToLegalize;
2856
2857 return UnableToLegalize;
2858 } else {
2859 NarrowTy0 = DstTy.getElementType();
2860 NarrowTy1 = NarrowTy;
2861 }
2862 }
2863
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002864 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002865 if (CondTy.isVector())
2866 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2867
2868 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2869 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2870
2871 for (unsigned i = 0; i < NumParts; ++i) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002872 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002873 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2874 Src1Regs[i], Src2Regs[i]);
2875 DstRegs.push_back(DstReg);
2876 }
2877
2878 if (NarrowTy0.isVector())
2879 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2880 else
2881 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2882
2883 MI.eraseFromParent();
2884 return Legalized;
2885}
2886
2887LegalizerHelper::LegalizeResult
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002888LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2889 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002890 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002891 LLT PhiTy = MRI.getType(DstReg);
2892 LLT LeftoverTy;
2893
2894 // All of the operands need to have the same number of elements, so if we can
2895 // determine a type breakdown for the result type, we can for all of the
2896 // source types.
2897 int NumParts, NumLeftover;
2898 std::tie(NumParts, NumLeftover)
2899 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2900 if (NumParts < 0)
2901 return UnableToLegalize;
2902
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002903 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002904 SmallVector<MachineInstrBuilder, 4> NewInsts;
2905
2906 const int TotalNumParts = NumParts + NumLeftover;
2907
2908 // Insert the new phis in the result block first.
2909 for (int I = 0; I != TotalNumParts; ++I) {
2910 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002911 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002912 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2913 .addDef(PartDstReg));
2914 if (I < NumParts)
2915 DstRegs.push_back(PartDstReg);
2916 else
2917 LeftoverDstRegs.push_back(PartDstReg);
2918 }
2919
2920 MachineBasicBlock *MBB = MI.getParent();
2921 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2922 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2923
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002924 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002925
2926 // Insert code to extract the incoming values in each predecessor block.
2927 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2928 PartRegs.clear();
2929 LeftoverRegs.clear();
2930
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002931 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002932 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2933 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2934
2935 LLT Unused;
2936 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2937 LeftoverRegs))
2938 return UnableToLegalize;
2939
2940 // Add the newly created operand splits to the existing instructions. The
2941 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2942 // pieces.
2943 for (int J = 0; J != TotalNumParts; ++J) {
2944 MachineInstrBuilder MIB = NewInsts[J];
2945 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2946 MIB.addMBB(&OpMBB);
2947 }
2948 }
2949
2950 MI.eraseFromParent();
2951 return Legalized;
2952}
2953
2954LegalizerHelper::LegalizeResult
Matt Arsenault28215ca2019-08-13 16:26:28 +00002955LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
2956 unsigned TypeIdx,
2957 LLT NarrowTy) {
2958 if (TypeIdx != 1)
2959 return UnableToLegalize;
2960
2961 const int NumDst = MI.getNumOperands() - 1;
2962 const Register SrcReg = MI.getOperand(NumDst).getReg();
2963 LLT SrcTy = MRI.getType(SrcReg);
2964
2965 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
2966
2967 // TODO: Create sequence of extracts.
2968 if (DstTy == NarrowTy)
2969 return UnableToLegalize;
2970
2971 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
2972 if (DstTy == GCDTy) {
2973 // This would just be a copy of the same unmerge.
2974 // TODO: Create extracts, pad with undef and create intermediate merges.
2975 return UnableToLegalize;
2976 }
2977
2978 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
2979 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2980 const int PartsPerUnmerge = NumDst / NumUnmerge;
2981
2982 for (int I = 0; I != NumUnmerge; ++I) {
2983 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
2984
2985 for (int J = 0; J != PartsPerUnmerge; ++J)
2986 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
2987 MIB.addUse(Unmerge.getReg(I));
2988 }
2989
2990 MI.eraseFromParent();
2991 return Legalized;
2992}
2993
2994LegalizerHelper::LegalizeResult
Matt Arsenault3cd39592019-10-09 22:44:43 +00002995LegalizerHelper::fewerElementsVectorBuildVector(MachineInstr &MI,
2996 unsigned TypeIdx,
2997 LLT NarrowTy) {
2998 assert(TypeIdx == 0 && "not a vector type index");
2999 Register DstReg = MI.getOperand(0).getReg();
3000 LLT DstTy = MRI.getType(DstReg);
3001 LLT SrcTy = DstTy.getElementType();
3002
3003 int DstNumElts = DstTy.getNumElements();
3004 int NarrowNumElts = NarrowTy.getNumElements();
3005 int NumConcat = (DstNumElts + NarrowNumElts - 1) / NarrowNumElts;
3006 LLT WidenedDstTy = LLT::vector(NarrowNumElts * NumConcat, SrcTy);
3007
3008 SmallVector<Register, 8> ConcatOps;
3009 SmallVector<Register, 8> SubBuildVector;
3010
3011 Register UndefReg;
3012 if (WidenedDstTy != DstTy)
3013 UndefReg = MIRBuilder.buildUndef(SrcTy).getReg(0);
3014
3015 // Create a G_CONCAT_VECTORS of NarrowTy pieces, padding with undef as
3016 // necessary.
3017 //
3018 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
3019 // -> <2 x s16>
3020 //
3021 // %4:_(s16) = G_IMPLICIT_DEF
3022 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
3023 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
3024 // %7:_(<4 x s16>) = G_CONCAT_VECTORS %5, %6
3025 // %3:_(<3 x s16>) = G_EXTRACT %7, 0
3026 for (int I = 0; I != NumConcat; ++I) {
3027 for (int J = 0; J != NarrowNumElts; ++J) {
3028 int SrcIdx = NarrowNumElts * I + J;
3029
3030 if (SrcIdx < DstNumElts) {
3031 Register SrcReg = MI.getOperand(SrcIdx + 1).getReg();
3032 SubBuildVector.push_back(SrcReg);
3033 } else
3034 SubBuildVector.push_back(UndefReg);
3035 }
3036
3037 auto BuildVec = MIRBuilder.buildBuildVector(NarrowTy, SubBuildVector);
3038 ConcatOps.push_back(BuildVec.getReg(0));
3039 SubBuildVector.clear();
3040 }
3041
3042 if (DstTy == WidenedDstTy)
3043 MIRBuilder.buildConcatVectors(DstReg, ConcatOps);
3044 else {
3045 auto Concat = MIRBuilder.buildConcatVectors(WidenedDstTy, ConcatOps);
3046 MIRBuilder.buildExtract(DstReg, Concat, 0);
3047 }
3048
3049 MI.eraseFromParent();
3050 return Legalized;
3051}
3052
3053LegalizerHelper::LegalizeResult
Matt Arsenault7f09fd62019-02-05 00:26:12 +00003054LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
3055 LLT NarrowTy) {
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003056 // FIXME: Don't know how to handle secondary types yet.
3057 if (TypeIdx != 0)
3058 return UnableToLegalize;
3059
Matt Arsenaultcfca2a72019-01-27 22:36:24 +00003060 MachineMemOperand *MMO = *MI.memoperands_begin();
3061
3062 // This implementation doesn't work for atomics. Give up instead of doing
3063 // something invalid.
3064 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
3065 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
3066 return UnableToLegalize;
3067
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003068 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003069 Register ValReg = MI.getOperand(0).getReg();
3070 Register AddrReg = MI.getOperand(1).getReg();
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003071 LLT ValTy = MRI.getType(ValReg);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003072
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003073 int NumParts = -1;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00003074 int NumLeftover = -1;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003075 LLT LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003076 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003077 if (IsLoad) {
Matt Arsenaultd3093c22019-02-28 00:16:32 +00003078 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003079 } else {
3080 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
Matt Arsenaultd3093c22019-02-28 00:16:32 +00003081 NarrowLeftoverRegs)) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003082 NumParts = NarrowRegs.size();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00003083 NumLeftover = NarrowLeftoverRegs.size();
3084 }
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003085 }
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003086
3087 if (NumParts == -1)
3088 return UnableToLegalize;
3089
3090 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
3091
3092 unsigned TotalSize = ValTy.getSizeInBits();
3093
3094 // Split the load/store into PartTy sized pieces starting at Offset. If this
3095 // is a load, return the new registers in ValRegs. For a store, each elements
3096 // of ValRegs should be PartTy. Returns the next offset that needs to be
3097 // handled.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003098 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003099 unsigned Offset) -> unsigned {
3100 MachineFunction &MF = MIRBuilder.getMF();
3101 unsigned PartSize = PartTy.getSizeInBits();
3102 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
3103 Offset += PartSize, ++Idx) {
3104 unsigned ByteSize = PartSize / 8;
3105 unsigned ByteOffset = Offset / 8;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003106 Register NewAddrReg;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003107
Daniel Sanderse74c5b92019-11-01 13:18:00 -07003108 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003109
3110 MachineMemOperand *NewMMO =
3111 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
3112
3113 if (IsLoad) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003114 Register Dst = MRI.createGenericVirtualRegister(PartTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00003115 ValRegs.push_back(Dst);
3116 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
3117 } else {
3118 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
3119 }
3120 }
3121
3122 return Offset;
3123 };
3124
3125 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
3126
3127 // Handle the rest of the register if this isn't an even type breakdown.
3128 if (LeftoverTy.isValid())
3129 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
3130
3131 if (IsLoad) {
3132 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
3133 LeftoverTy, NarrowLeftoverRegs);
3134 }
3135
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003136 MI.eraseFromParent();
3137 return Legalized;
3138}
3139
3140LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00003141LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
3142 LLT NarrowTy) {
Matt Arsenault1b1e6852019-01-25 02:59:34 +00003143 using namespace TargetOpcode;
Volkan Keles574d7372018-12-14 22:11:20 +00003144
3145 MIRBuilder.setInstr(MI);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003146 switch (MI.getOpcode()) {
3147 case G_IMPLICIT_DEF:
3148 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
3149 case G_AND:
3150 case G_OR:
3151 case G_XOR:
3152 case G_ADD:
3153 case G_SUB:
3154 case G_MUL:
3155 case G_SMULH:
3156 case G_UMULH:
3157 case G_FADD:
3158 case G_FMUL:
3159 case G_FSUB:
3160 case G_FNEG:
3161 case G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00003162 case G_FCANONICALIZE:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003163 case G_FDIV:
3164 case G_FREM:
3165 case G_FMA:
Matt Arsenaultcf103722019-09-06 20:49:10 +00003166 case G_FMAD:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003167 case G_FPOW:
3168 case G_FEXP:
3169 case G_FEXP2:
3170 case G_FLOG:
3171 case G_FLOG2:
3172 case G_FLOG10:
Jessica Paquetteba557672019-04-25 16:44:40 +00003173 case G_FNEARBYINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003174 case G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00003175 case G_FFLOOR:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00003176 case G_FRINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003177 case G_INTRINSIC_ROUND:
3178 case G_INTRINSIC_TRUNC:
Jessica Paquette7db82d72019-01-28 18:34:18 +00003179 case G_FCOS:
3180 case G_FSIN:
Jessica Paquette22457f82019-01-30 21:03:52 +00003181 case G_FSQRT:
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00003182 case G_BSWAP:
Matt Arsenault5ff310e2019-09-04 20:46:15 +00003183 case G_BITREVERSE:
Amara Emersonae878da2019-04-10 23:06:08 +00003184 case G_SDIV:
Matt Arsenaultd12f2a22020-01-04 13:24:09 -05003185 case G_UDIV:
3186 case G_SREM:
3187 case G_UREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00003188 case G_SMIN:
3189 case G_SMAX:
3190 case G_UMIN:
3191 case G_UMAX:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00003192 case G_FMINNUM:
3193 case G_FMAXNUM:
3194 case G_FMINNUM_IEEE:
3195 case G_FMAXNUM_IEEE:
3196 case G_FMINIMUM:
3197 case G_FMAXIMUM:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003198 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00003199 case G_SHL:
3200 case G_LSHR:
3201 case G_ASHR:
Matt Arsenault75e30c42019-02-20 16:42:52 +00003202 case G_CTLZ:
3203 case G_CTLZ_ZERO_UNDEF:
3204 case G_CTTZ:
3205 case G_CTTZ_ZERO_UNDEF:
3206 case G_CTPOP:
Matt Arsenault1448f562019-05-17 12:19:52 +00003207 case G_FCOPYSIGN:
Matt Arsenaultc83b8232019-02-07 17:38:00 +00003208 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003209 case G_ZEXT:
3210 case G_SEXT:
3211 case G_ANYEXT:
3212 case G_FPEXT:
3213 case G_FPTRUNC:
3214 case G_SITOFP:
3215 case G_UITOFP:
3216 case G_FPTOSI:
3217 case G_FPTOUI:
Matt Arsenaultcbaada62019-02-02 23:29:55 +00003218 case G_INTTOPTR:
3219 case G_PTRTOINT:
Matt Arsenaulta8b43392019-02-08 02:40:47 +00003220 case G_ADDRSPACE_CAST:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003221 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
3222 case G_ICMP:
3223 case G_FCMP:
3224 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00003225 case G_SELECT:
3226 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00003227 case G_PHI:
3228 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
Matt Arsenault28215ca2019-08-13 16:26:28 +00003229 case G_UNMERGE_VALUES:
3230 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
Matt Arsenault3cd39592019-10-09 22:44:43 +00003231 case G_BUILD_VECTOR:
3232 return fewerElementsVectorBuildVector(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00003233 case G_LOAD:
3234 case G_STORE:
Matt Arsenault7f09fd62019-02-05 00:26:12 +00003235 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Tim Northover33b07d62016-07-22 20:03:43 +00003236 default:
3237 return UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +00003238 }
3239}
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003240
3241LegalizerHelper::LegalizeResult
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003242LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
3243 const LLT HalfTy, const LLT AmtTy) {
3244
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003245 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3246 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Jay Foad63f73542020-01-16 12:37:00 +00003247 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003248
3249 if (Amt.isNullValue()) {
Jay Foad63f73542020-01-16 12:37:00 +00003250 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003251 MI.eraseFromParent();
3252 return Legalized;
3253 }
3254
3255 LLT NVT = HalfTy;
3256 unsigned NVTBits = HalfTy.getSizeInBits();
3257 unsigned VTBits = 2 * NVTBits;
3258
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003259 SrcOp Lo(Register(0)), Hi(Register(0));
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003260 if (MI.getOpcode() == TargetOpcode::G_SHL) {
3261 if (Amt.ugt(VTBits)) {
3262 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3263 } else if (Amt.ugt(NVTBits)) {
3264 Lo = MIRBuilder.buildConstant(NVT, 0);
3265 Hi = MIRBuilder.buildShl(NVT, InL,
3266 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3267 } else if (Amt == NVTBits) {
3268 Lo = MIRBuilder.buildConstant(NVT, 0);
3269 Hi = InL;
3270 } else {
3271 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
Matt Arsenaulte98cab12019-02-07 20:44:08 +00003272 auto OrLHS =
3273 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
3274 auto OrRHS = MIRBuilder.buildLShr(
3275 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3276 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003277 }
3278 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3279 if (Amt.ugt(VTBits)) {
3280 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
3281 } else if (Amt.ugt(NVTBits)) {
3282 Lo = MIRBuilder.buildLShr(NVT, InH,
3283 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3284 Hi = MIRBuilder.buildConstant(NVT, 0);
3285 } else if (Amt == NVTBits) {
3286 Lo = InH;
3287 Hi = MIRBuilder.buildConstant(NVT, 0);
3288 } else {
3289 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3290
3291 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3292 auto OrRHS = MIRBuilder.buildShl(
3293 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3294
3295 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3296 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
3297 }
3298 } else {
3299 if (Amt.ugt(VTBits)) {
3300 Hi = Lo = MIRBuilder.buildAShr(
3301 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3302 } else if (Amt.ugt(NVTBits)) {
3303 Lo = MIRBuilder.buildAShr(NVT, InH,
3304 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
3305 Hi = MIRBuilder.buildAShr(NVT, InH,
3306 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3307 } else if (Amt == NVTBits) {
3308 Lo = InH;
3309 Hi = MIRBuilder.buildAShr(NVT, InH,
3310 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
3311 } else {
3312 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
3313
3314 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
3315 auto OrRHS = MIRBuilder.buildShl(
3316 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
3317
3318 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
3319 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
3320 }
3321 }
3322
Jay Foad63f73542020-01-16 12:37:00 +00003323 MIRBuilder.buildMerge(MI.getOperand(0), {Lo.getReg(), Hi.getReg()});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003324 MI.eraseFromParent();
3325
3326 return Legalized;
3327}
3328
3329// TODO: Optimize if constant shift amount.
3330LegalizerHelper::LegalizeResult
3331LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
3332 LLT RequestedTy) {
3333 if (TypeIdx == 1) {
3334 Observer.changingInstr(MI);
3335 narrowScalarSrc(MI, RequestedTy, 2);
3336 Observer.changedInstr(MI);
3337 return Legalized;
3338 }
3339
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003340 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003341 LLT DstTy = MRI.getType(DstReg);
3342 if (DstTy.isVector())
3343 return UnableToLegalize;
3344
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003345 Register Amt = MI.getOperand(2).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003346 LLT ShiftAmtTy = MRI.getType(Amt);
3347 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
3348 if (DstEltSize % 2 != 0)
3349 return UnableToLegalize;
3350
3351 // Ignore the input type. We can only go to exactly half the size of the
3352 // input. If that isn't small enough, the resulting pieces will be further
3353 // legalized.
3354 const unsigned NewBitSize = DstEltSize / 2;
3355 const LLT HalfTy = LLT::scalar(NewBitSize);
3356 const LLT CondTy = LLT::scalar(1);
3357
3358 if (const MachineInstr *KShiftAmt =
3359 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
3360 return narrowScalarShiftByConstant(
3361 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
3362 }
3363
3364 // TODO: Expand with known bits.
3365
3366 // Handle the fully general expansion by an unknown amount.
3367 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
3368
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003369 Register InL = MRI.createGenericVirtualRegister(HalfTy);
3370 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Jay Foad63f73542020-01-16 12:37:00 +00003371 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003372
3373 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
3374 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
3375
3376 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
3377 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
3378 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
3379
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003380 Register ResultRegs[2];
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003381 switch (MI.getOpcode()) {
3382 case TargetOpcode::G_SHL: {
3383 // Short: ShAmt < NewBitSize
Petar Avramovicd568ed42019-08-27 14:22:32 +00003384 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003385
Petar Avramovicd568ed42019-08-27 14:22:32 +00003386 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
3387 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
3388 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003389
3390 // Long: ShAmt >= NewBitSize
3391 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
3392 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
3393
3394 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
3395 auto Hi = MIRBuilder.buildSelect(
3396 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
3397
3398 ResultRegs[0] = Lo.getReg(0);
3399 ResultRegs[1] = Hi.getReg(0);
3400 break;
3401 }
Petar Avramovica3932382019-08-27 14:33:05 +00003402 case TargetOpcode::G_LSHR:
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003403 case TargetOpcode::G_ASHR: {
3404 // Short: ShAmt < NewBitSize
Petar Avramovica3932382019-08-27 14:33:05 +00003405 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003406
Petar Avramovicd568ed42019-08-27 14:22:32 +00003407 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
3408 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
3409 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003410
3411 // Long: ShAmt >= NewBitSize
Petar Avramovica3932382019-08-27 14:33:05 +00003412 MachineInstrBuilder HiL;
3413 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
3414 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
3415 } else {
3416 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
3417 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
3418 }
3419 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
3420 {InH, AmtExcess}); // Lo from Hi part.
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00003421
3422 auto Lo = MIRBuilder.buildSelect(
3423 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
3424
3425 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
3426
3427 ResultRegs[0] = Lo.getReg(0);
3428 ResultRegs[1] = Hi.getReg(0);
3429 break;
3430 }
3431 default:
3432 llvm_unreachable("not a shift");
3433 }
3434
3435 MIRBuilder.buildMerge(DstReg, ResultRegs);
3436 MI.eraseFromParent();
3437 return Legalized;
3438}
3439
3440LegalizerHelper::LegalizeResult
Matt Arsenault72bcf152019-02-28 00:01:05 +00003441LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3442 LLT MoreTy) {
3443 assert(TypeIdx == 0 && "Expecting only Idx 0");
3444
3445 Observer.changingInstr(MI);
3446 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3447 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3448 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3449 moreElementsVectorSrc(MI, MoreTy, I);
3450 }
3451
3452 MachineBasicBlock &MBB = *MI.getParent();
Amara Emerson9d647212019-09-16 23:46:03 +00003453 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
Matt Arsenault72bcf152019-02-28 00:01:05 +00003454 moreElementsVectorDst(MI, MoreTy, 0);
3455 Observer.changedInstr(MI);
3456 return Legalized;
3457}
3458
3459LegalizerHelper::LegalizeResult
Matt Arsenault18ec3822019-02-11 22:00:39 +00003460LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
3461 LLT MoreTy) {
3462 MIRBuilder.setInstr(MI);
3463 unsigned Opc = MI.getOpcode();
3464 switch (Opc) {
Matt Arsenault7bedceb2019-08-01 01:44:22 +00003465 case TargetOpcode::G_IMPLICIT_DEF:
3466 case TargetOpcode::G_LOAD: {
3467 if (TypeIdx != 0)
3468 return UnableToLegalize;
Matt Arsenault18ec3822019-02-11 22:00:39 +00003469 Observer.changingInstr(MI);
3470 moreElementsVectorDst(MI, MoreTy, 0);
3471 Observer.changedInstr(MI);
3472 return Legalized;
3473 }
Matt Arsenault7bedceb2019-08-01 01:44:22 +00003474 case TargetOpcode::G_STORE:
3475 if (TypeIdx != 0)
3476 return UnableToLegalize;
3477 Observer.changingInstr(MI);
3478 moreElementsVectorSrc(MI, MoreTy, 0);
3479 Observer.changedInstr(MI);
3480 return Legalized;
Matt Arsenault26b7e852019-02-19 16:30:19 +00003481 case TargetOpcode::G_AND:
3482 case TargetOpcode::G_OR:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00003483 case TargetOpcode::G_XOR:
3484 case TargetOpcode::G_SMIN:
3485 case TargetOpcode::G_SMAX:
3486 case TargetOpcode::G_UMIN:
Matt Arsenault9fd31fd2019-07-27 17:47:08 -04003487 case TargetOpcode::G_UMAX:
3488 case TargetOpcode::G_FMINNUM:
3489 case TargetOpcode::G_FMAXNUM:
3490 case TargetOpcode::G_FMINNUM_IEEE:
3491 case TargetOpcode::G_FMAXNUM_IEEE:
3492 case TargetOpcode::G_FMINIMUM:
3493 case TargetOpcode::G_FMAXIMUM: {
Matt Arsenault26b7e852019-02-19 16:30:19 +00003494 Observer.changingInstr(MI);
3495 moreElementsVectorSrc(MI, MoreTy, 1);
3496 moreElementsVectorSrc(MI, MoreTy, 2);
3497 moreElementsVectorDst(MI, MoreTy, 0);
3498 Observer.changedInstr(MI);
3499 return Legalized;
3500 }
Matt Arsenault4d884272019-02-19 16:44:22 +00003501 case TargetOpcode::G_EXTRACT:
3502 if (TypeIdx != 1)
3503 return UnableToLegalize;
3504 Observer.changingInstr(MI);
3505 moreElementsVectorSrc(MI, MoreTy, 1);
3506 Observer.changedInstr(MI);
3507 return Legalized;
Matt Arsenaultc4d07552019-02-20 16:11:22 +00003508 case TargetOpcode::G_INSERT:
3509 if (TypeIdx != 0)
3510 return UnableToLegalize;
3511 Observer.changingInstr(MI);
3512 moreElementsVectorSrc(MI, MoreTy, 1);
3513 moreElementsVectorDst(MI, MoreTy, 0);
3514 Observer.changedInstr(MI);
3515 return Legalized;
Matt Arsenaultb4c95b32019-02-19 17:03:09 +00003516 case TargetOpcode::G_SELECT:
3517 if (TypeIdx != 0)
3518 return UnableToLegalize;
3519 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
3520 return UnableToLegalize;
3521
3522 Observer.changingInstr(MI);
3523 moreElementsVectorSrc(MI, MoreTy, 2);
3524 moreElementsVectorSrc(MI, MoreTy, 3);
3525 moreElementsVectorDst(MI, MoreTy, 0);
3526 Observer.changedInstr(MI);
3527 return Legalized;
Matt Arsenault954a0122019-08-21 16:59:10 +00003528 case TargetOpcode::G_UNMERGE_VALUES: {
3529 if (TypeIdx != 1)
3530 return UnableToLegalize;
3531
3532 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3533 int NumDst = MI.getNumOperands() - 1;
3534 moreElementsVectorSrc(MI, MoreTy, NumDst);
3535
3536 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3537 for (int I = 0; I != NumDst; ++I)
3538 MIB.addDef(MI.getOperand(I).getReg());
3539
3540 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
3541 for (int I = NumDst; I != NewNumDst; ++I)
3542 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
3543
3544 MIB.addUse(MI.getOperand(NumDst).getReg());
3545 MI.eraseFromParent();
3546 return Legalized;
3547 }
Matt Arsenault72bcf152019-02-28 00:01:05 +00003548 case TargetOpcode::G_PHI:
3549 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +00003550 default:
3551 return UnableToLegalize;
3552 }
3553}
3554
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003555void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
3556 ArrayRef<Register> Src1Regs,
3557 ArrayRef<Register> Src2Regs,
Petar Avramovic0b17e592019-03-11 10:00:17 +00003558 LLT NarrowTy) {
3559 MachineIRBuilder &B = MIRBuilder;
3560 unsigned SrcParts = Src1Regs.size();
3561 unsigned DstParts = DstRegs.size();
3562
3563 unsigned DstIdx = 0; // Low bits of the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003564 Register FactorSum =
Petar Avramovic0b17e592019-03-11 10:00:17 +00003565 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
3566 DstRegs[DstIdx] = FactorSum;
3567
3568 unsigned CarrySumPrevDstIdx;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003569 SmallVector<Register, 4> Factors;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003570
3571 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
3572 // Collect low parts of muls for DstIdx.
3573 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
3574 i <= std::min(DstIdx, SrcParts - 1); ++i) {
3575 MachineInstrBuilder Mul =
3576 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
3577 Factors.push_back(Mul.getReg(0));
3578 }
3579 // Collect high parts of muls from previous DstIdx.
3580 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
3581 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
3582 MachineInstrBuilder Umulh =
3583 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
3584 Factors.push_back(Umulh.getReg(0));
3585 }
Greg Bedwellb1c4b4d2019-10-28 14:28:00 +00003586 // Add CarrySum from additions calculated for previous DstIdx.
Petar Avramovic0b17e592019-03-11 10:00:17 +00003587 if (DstIdx != 1) {
3588 Factors.push_back(CarrySumPrevDstIdx);
3589 }
3590
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003591 Register CarrySum;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003592 // Add all factors and accumulate all carries into CarrySum.
3593 if (DstIdx != DstParts - 1) {
3594 MachineInstrBuilder Uaddo =
3595 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
3596 FactorSum = Uaddo.getReg(0);
3597 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
3598 for (unsigned i = 2; i < Factors.size(); ++i) {
3599 MachineInstrBuilder Uaddo =
3600 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
3601 FactorSum = Uaddo.getReg(0);
3602 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
3603 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
3604 }
3605 } else {
3606 // Since value for the next index is not calculated, neither is CarrySum.
3607 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
3608 for (unsigned i = 2; i < Factors.size(); ++i)
3609 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
3610 }
3611
3612 CarrySumPrevDstIdx = CarrySum;
3613 DstRegs[DstIdx] = FactorSum;
3614 Factors.clear();
3615 }
3616}
3617
Matt Arsenault18ec3822019-02-11 22:00:39 +00003618LegalizerHelper::LegalizeResult
Petar Avramovic0b17e592019-03-11 10:00:17 +00003619LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003620 Register DstReg = MI.getOperand(0).getReg();
3621 Register Src1 = MI.getOperand(1).getReg();
3622 Register Src2 = MI.getOperand(2).getReg();
Petar Avramovic0b17e592019-03-11 10:00:17 +00003623
Matt Arsenault211e89d2019-01-27 00:52:51 +00003624 LLT Ty = MRI.getType(DstReg);
3625 if (Ty.isVector())
3626 return UnableToLegalize;
3627
Petar Avramovic0b17e592019-03-11 10:00:17 +00003628 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
3629 unsigned DstSize = Ty.getSizeInBits();
3630 unsigned NarrowSize = NarrowTy.getSizeInBits();
3631 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
Matt Arsenault211e89d2019-01-27 00:52:51 +00003632 return UnableToLegalize;
3633
Petar Avramovic0b17e592019-03-11 10:00:17 +00003634 unsigned NumDstParts = DstSize / NarrowSize;
3635 unsigned NumSrcParts = SrcSize / NarrowSize;
Petar Avramovic5229f472019-03-11 10:08:44 +00003636 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
3637 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003638
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003639 SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
Petar Avramovic0b17e592019-03-11 10:00:17 +00003640 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
3641 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
Petar Avramovic5229f472019-03-11 10:08:44 +00003642 DstTmpRegs.resize(DstTmpParts);
3643 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003644
Petar Avramovic5229f472019-03-11 10:08:44 +00003645 // Take only high half of registers if this is high mul.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003646 ArrayRef<Register> DstRegs(
Petar Avramovic5229f472019-03-11 10:08:44 +00003647 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
Petar Avramovic0b17e592019-03-11 10:00:17 +00003648 MIRBuilder.buildMerge(DstReg, DstRegs);
Matt Arsenault211e89d2019-01-27 00:52:51 +00003649 MI.eraseFromParent();
3650 return Legalized;
3651}
3652
Matt Arsenault1cf713662019-02-12 14:54:52 +00003653LegalizerHelper::LegalizeResult
3654LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
3655 LLT NarrowTy) {
3656 if (TypeIdx != 1)
3657 return UnableToLegalize;
3658
3659 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3660
3661 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
3662 // FIXME: add support for when SizeOp1 isn't an exact multiple of
3663 // NarrowSize.
3664 if (SizeOp1 % NarrowSize != 0)
3665 return UnableToLegalize;
3666 int NumParts = SizeOp1 / NarrowSize;
3667
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003668 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003669 SmallVector<uint64_t, 2> Indexes;
3670 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3671
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003672 Register OpReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003673 uint64_t OpStart = MI.getOperand(2).getImm();
3674 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3675 for (int i = 0; i < NumParts; ++i) {
3676 unsigned SrcStart = i * NarrowSize;
3677
3678 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
3679 // No part of the extract uses this subregister, ignore it.
3680 continue;
3681 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3682 // The entire subregister is extracted, forward the value.
3683 DstRegs.push_back(SrcRegs[i]);
3684 continue;
3685 }
3686
3687 // OpSegStart is where this destination segment would start in OpReg if it
3688 // extended infinitely in both directions.
3689 int64_t ExtractOffset;
3690 uint64_t SegSize;
3691 if (OpStart < SrcStart) {
3692 ExtractOffset = 0;
3693 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
3694 } else {
3695 ExtractOffset = OpStart - SrcStart;
3696 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
3697 }
3698
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003699 Register SegReg = SrcRegs[i];
Matt Arsenault1cf713662019-02-12 14:54:52 +00003700 if (ExtractOffset != 0 || SegSize != NarrowSize) {
3701 // A genuine extract is needed.
3702 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3703 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
3704 }
3705
3706 DstRegs.push_back(SegReg);
3707 }
3708
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003709 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003710 if(MRI.getType(DstReg).isVector())
3711 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3712 else
3713 MIRBuilder.buildMerge(DstReg, DstRegs);
3714 MI.eraseFromParent();
3715 return Legalized;
3716}
3717
3718LegalizerHelper::LegalizeResult
3719LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
3720 LLT NarrowTy) {
3721 // FIXME: Don't know how to handle secondary types yet.
3722 if (TypeIdx != 0)
3723 return UnableToLegalize;
3724
3725 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
3726 uint64_t NarrowSize = NarrowTy.getSizeInBits();
3727
3728 // FIXME: add support for when SizeOp0 isn't an exact multiple of
3729 // NarrowSize.
3730 if (SizeOp0 % NarrowSize != 0)
3731 return UnableToLegalize;
3732
3733 int NumParts = SizeOp0 / NarrowSize;
3734
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003735 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003736 SmallVector<uint64_t, 2> Indexes;
3737 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
3738
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003739 Register OpReg = MI.getOperand(2).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003740 uint64_t OpStart = MI.getOperand(3).getImm();
3741 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
3742 for (int i = 0; i < NumParts; ++i) {
3743 unsigned DstStart = i * NarrowSize;
3744
3745 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
3746 // No part of the insert affects this subregister, forward the original.
3747 DstRegs.push_back(SrcRegs[i]);
3748 continue;
3749 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
3750 // The entire subregister is defined by this insert, forward the new
3751 // value.
3752 DstRegs.push_back(OpReg);
3753 continue;
3754 }
3755
3756 // OpSegStart is where this destination segment would start in OpReg if it
3757 // extended infinitely in both directions.
3758 int64_t ExtractOffset, InsertOffset;
3759 uint64_t SegSize;
3760 if (OpStart < DstStart) {
3761 InsertOffset = 0;
3762 ExtractOffset = DstStart - OpStart;
3763 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
3764 } else {
3765 InsertOffset = OpStart - DstStart;
3766 ExtractOffset = 0;
3767 SegSize =
3768 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
3769 }
3770
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003771 Register SegReg = OpReg;
Matt Arsenault1cf713662019-02-12 14:54:52 +00003772 if (ExtractOffset != 0 || SegSize != OpSize) {
3773 // A genuine extract is needed.
3774 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
3775 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
3776 }
3777
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003778 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +00003779 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
3780 DstRegs.push_back(DstReg);
3781 }
3782
3783 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003784 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00003785 if(MRI.getType(DstReg).isVector())
3786 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3787 else
3788 MIRBuilder.buildMerge(DstReg, DstRegs);
3789 MI.eraseFromParent();
3790 return Legalized;
3791}
3792
Matt Arsenault211e89d2019-01-27 00:52:51 +00003793LegalizerHelper::LegalizeResult
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003794LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
3795 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003796 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003797 LLT DstTy = MRI.getType(DstReg);
3798
3799 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
3800
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003801 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3802 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
3803 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003804 LLT LeftoverTy;
3805 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
3806 Src0Regs, Src0LeftoverRegs))
3807 return UnableToLegalize;
3808
3809 LLT Unused;
3810 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
3811 Src1Regs, Src1LeftoverRegs))
3812 llvm_unreachable("inconsistent extractParts result");
3813
3814 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3815 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
3816 {Src0Regs[I], Src1Regs[I]});
Jay Foadb482e1b2020-01-23 11:51:35 +00003817 DstRegs.push_back(Inst.getReg(0));
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003818 }
3819
3820 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3821 auto Inst = MIRBuilder.buildInstr(
3822 MI.getOpcode(),
3823 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
Jay Foadb482e1b2020-01-23 11:51:35 +00003824 DstLeftoverRegs.push_back(Inst.getReg(0));
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00003825 }
3826
3827 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3828 LeftoverTy, DstLeftoverRegs);
3829
3830 MI.eraseFromParent();
3831 return Legalized;
3832}
3833
3834LegalizerHelper::LegalizeResult
Matt Arsenaultbe31a7b2020-01-10 11:02:18 -05003835LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
3836 LLT NarrowTy) {
3837 if (TypeIdx != 0)
3838 return UnableToLegalize;
3839
3840 Register DstReg = MI.getOperand(0).getReg();
3841 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenaultbe31a7b2020-01-10 11:02:18 -05003842
Matt Arsenaulta66d2812020-01-10 10:41:29 -05003843 LLT DstTy = MRI.getType(DstReg);
3844 if (DstTy.isVector())
Matt Arsenaultbe31a7b2020-01-10 11:02:18 -05003845 return UnableToLegalize;
3846
Matt Arsenaulta66d2812020-01-10 10:41:29 -05003847 SmallVector<Register, 8> Parts;
3848 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
3849 buildLCMMerge(DstReg, NarrowTy, GCDTy, Parts, MI.getOpcode());
Matt Arsenaultbe31a7b2020-01-10 11:02:18 -05003850 MI.eraseFromParent();
3851 return Legalized;
3852}
3853
3854LegalizerHelper::LegalizeResult
Matt Arsenault81511e52019-02-05 00:13:44 +00003855LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
3856 LLT NarrowTy) {
3857 if (TypeIdx != 0)
3858 return UnableToLegalize;
3859
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003860 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00003861 LLT CondTy = MRI.getType(CondReg);
3862 if (CondTy.isVector()) // TODO: Handle vselect
3863 return UnableToLegalize;
3864
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003865 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00003866 LLT DstTy = MRI.getType(DstReg);
3867
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00003868 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
3869 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
3870 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
Matt Arsenault81511e52019-02-05 00:13:44 +00003871 LLT LeftoverTy;
3872 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
3873 Src1Regs, Src1LeftoverRegs))
3874 return UnableToLegalize;
3875
3876 LLT Unused;
3877 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
3878 Src2Regs, Src2LeftoverRegs))
3879 llvm_unreachable("inconsistent extractParts result");
3880
3881 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
3882 auto Select = MIRBuilder.buildSelect(NarrowTy,
3883 CondReg, Src1Regs[I], Src2Regs[I]);
Jay Foadb482e1b2020-01-23 11:51:35 +00003884 DstRegs.push_back(Select.getReg(0));
Matt Arsenault81511e52019-02-05 00:13:44 +00003885 }
3886
3887 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
3888 auto Select = MIRBuilder.buildSelect(
3889 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
Jay Foadb482e1b2020-01-23 11:51:35 +00003890 DstLeftoverRegs.push_back(Select.getReg(0));
Matt Arsenault81511e52019-02-05 00:13:44 +00003891 }
3892
3893 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
3894 LeftoverTy, DstLeftoverRegs);
3895
3896 MI.eraseFromParent();
3897 return Legalized;
3898}
3899
3900LegalizerHelper::LegalizeResult
Petar Avramovic2b66d322020-01-27 09:43:38 +01003901LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
3902 LLT NarrowTy) {
3903 if (TypeIdx != 1)
3904 return UnableToLegalize;
3905
3906 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3907 unsigned NarrowSize = NarrowTy.getSizeInBits();
3908
3909 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
3910 MachineIRBuilder &B = MIRBuilder;
3911 auto UnmergeSrc = B.buildUnmerge(NarrowTy, MI.getOperand(1));
3912 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
3913 auto C_0 = B.buildConstant(NarrowTy, 0);
3914 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3915 UnmergeSrc.getReg(1), C_0);
3916 auto LoCTLZ = B.buildCTLZ(NarrowTy, UnmergeSrc.getReg(0));
3917 auto C_NarrowSize = B.buildConstant(NarrowTy, NarrowSize);
3918 auto HiIsZeroCTLZ = B.buildAdd(NarrowTy, LoCTLZ, C_NarrowSize);
3919 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(NarrowTy, UnmergeSrc.getReg(1));
3920 auto LoOut = B.buildSelect(NarrowTy, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
3921
3922 B.buildMerge(MI.getOperand(0), {LoOut.getReg(0), C_0.getReg(0)});
3923
3924 MI.eraseFromParent();
3925 return Legalized;
3926 }
3927
3928 return UnableToLegalize;
3929}
3930
3931LegalizerHelper::LegalizeResult
Petar Avramovic8bc7ba52020-01-27 09:51:06 +01003932LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
3933 LLT NarrowTy) {
3934 if (TypeIdx != 1)
3935 return UnableToLegalize;
3936
3937 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3938 unsigned NarrowSize = NarrowTy.getSizeInBits();
3939
3940 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
3941 MachineIRBuilder &B = MIRBuilder;
3942 auto UnmergeSrc = B.buildUnmerge(NarrowTy, MI.getOperand(1));
3943 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
3944 auto C_0 = B.buildConstant(NarrowTy, 0);
3945 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3946 UnmergeSrc.getReg(0), C_0);
3947 auto HiCTTZ = B.buildCTTZ(NarrowTy, UnmergeSrc.getReg(1));
3948 auto C_NarrowSize = B.buildConstant(NarrowTy, NarrowSize);
3949 auto LoIsZeroCTTZ = B.buildAdd(NarrowTy, HiCTTZ, C_NarrowSize);
3950 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(NarrowTy, UnmergeSrc.getReg(0));
3951 auto LoOut = B.buildSelect(NarrowTy, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
3952
3953 B.buildMerge(MI.getOperand(0), {LoOut.getReg(0), C_0.getReg(0)});
3954
3955 MI.eraseFromParent();
3956 return Legalized;
3957 }
3958
3959 return UnableToLegalize;
3960}
3961
3962LegalizerHelper::LegalizeResult
Petar Avramoviccbf03aee2020-01-27 09:59:50 +01003963LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
3964 LLT NarrowTy) {
3965 if (TypeIdx != 1)
3966 return UnableToLegalize;
3967
3968 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3969 unsigned NarrowSize = NarrowTy.getSizeInBits();
3970
3971 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
3972 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
3973
3974 auto LoCTPOP = MIRBuilder.buildCTPOP(NarrowTy, UnmergeSrc.getReg(0));
3975 auto HiCTPOP = MIRBuilder.buildCTPOP(NarrowTy, UnmergeSrc.getReg(1));
3976 auto Out = MIRBuilder.buildAdd(NarrowTy, HiCTPOP, LoCTPOP);
3977 MIRBuilder.buildZExt(MI.getOperand(0), Out);
3978
3979 MI.eraseFromParent();
3980 return Legalized;
3981 }
3982
3983 return UnableToLegalize;
3984}
3985
3986LegalizerHelper::LegalizeResult
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003987LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3988 unsigned Opc = MI.getOpcode();
3989 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
Diana Picus0528e2c2018-11-26 11:07:02 +00003990 auto isSupported = [this](const LegalityQuery &Q) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003991 auto QAction = LI.getAction(Q).Action;
Diana Picus0528e2c2018-11-26 11:07:02 +00003992 return QAction == Legal || QAction == Libcall || QAction == Custom;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003993 };
3994 switch (Opc) {
3995 default:
3996 return UnableToLegalize;
3997 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
3998 // This trivially expands to CTLZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00003999 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004000 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00004001 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004002 return Legalized;
4003 }
4004 case TargetOpcode::G_CTLZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004005 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004006 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00004007 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
Diana Picus0528e2c2018-11-26 11:07:02 +00004008 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
Jay Foad28bb43b2020-01-16 12:09:48 +00004009 auto MIBCtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(Ty, SrcReg);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004010 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
4011 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
4012 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4013 SrcReg, MIBZero);
Jay Foad63f73542020-01-16 12:37:00 +00004014 MIRBuilder.buildSelect(MI.getOperand(0), MIBICmp, MIBLen, MIBCtlzZU);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004015 MI.eraseFromParent();
4016 return Legalized;
4017 }
4018 // for now, we do this:
4019 // NewLen = NextPowerOf2(Len);
4020 // x = x | (x >> 1);
4021 // x = x | (x >> 2);
4022 // ...
4023 // x = x | (x >>16);
4024 // x = x | (x >>32); // for 64-bit input
4025 // Upto NewLen/2
4026 // return Len - popcount(x);
4027 //
4028 // Ref: "Hacker's Delight" by Henry Warren
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004029 Register Op = SrcReg;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004030 unsigned NewLen = PowerOf2Ceil(Len);
4031 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
4032 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
Jay Foad28bb43b2020-01-16 12:09:48 +00004033 auto MIBOp =
4034 MIRBuilder.buildOr(Ty, Op, MIRBuilder.buildLShr(Ty, Op, MIBShiftAmt));
Jay Foadb482e1b2020-01-23 11:51:35 +00004035 Op = MIBOp.getReg(0);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004036 }
Jay Foad28bb43b2020-01-16 12:09:48 +00004037 auto MIBPop = MIRBuilder.buildCTPOP(Ty, Op);
Jay Foad63f73542020-01-16 12:37:00 +00004038 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(Ty, Len),
4039 MIBPop);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004040 MI.eraseFromParent();
4041 return Legalized;
4042 }
4043 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
4044 // This trivially expands to CTTZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00004045 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004046 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00004047 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004048 return Legalized;
4049 }
4050 case TargetOpcode::G_CTTZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004051 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004052 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00004053 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004054 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
4055 // zero.
Jay Foad28bb43b2020-01-16 12:09:48 +00004056 auto MIBCttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(Ty, SrcReg);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004057 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
4058 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
4059 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4060 SrcReg, MIBZero);
Jay Foad63f73542020-01-16 12:37:00 +00004061 MIRBuilder.buildSelect(MI.getOperand(0), MIBICmp, MIBLen, MIBCttzZU);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004062 MI.eraseFromParent();
4063 return Legalized;
4064 }
4065 // for now, we use: { return popcount(~x & (x - 1)); }
4066 // unless the target has ctlz but not ctpop, in which case we use:
4067 // { return 32 - nlz(~x & (x-1)); }
4068 // Ref: "Hacker's Delight" by Henry Warren
4069 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
Jay Foad28bb43b2020-01-16 12:09:48 +00004070 auto MIBNot = MIRBuilder.buildXor(Ty, SrcReg, MIBCstNeg1);
4071 auto MIBTmp = MIRBuilder.buildAnd(
4072 Ty, MIBNot, MIRBuilder.buildAdd(Ty, SrcReg, MIBCstNeg1));
Matt Arsenaultd5684f72019-01-31 02:09:57 +00004073 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
4074 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004075 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
Jay Foad63f73542020-01-16 12:37:00 +00004076 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen,
Jay Foad28bb43b2020-01-16 12:09:48 +00004077 MIRBuilder.buildCTLZ(Ty, MIBTmp));
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004078 MI.eraseFromParent();
4079 return Legalized;
4080 }
4081 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
Jay Foadb482e1b2020-01-23 11:51:35 +00004082 MI.getOperand(1).setReg(MIBTmp.getReg(0));
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004083 return Legalized;
4084 }
Petar Avramoviccbf03aee2020-01-27 09:59:50 +01004085 case TargetOpcode::G_CTPOP: {
4086 unsigned Size = Ty.getSizeInBits();
4087 MachineIRBuilder &B = MIRBuilder;
4088
4089 // Count set bits in blocks of 2 bits. Default approach would be
4090 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
4091 // We use following formula instead:
4092 // B2Count = val - { (val >> 1) & 0x55555555 }
4093 // since it gives same result in blocks of 2 with one instruction less.
4094 auto C_1 = B.buildConstant(Ty, 1);
4095 auto B2Set1LoTo1Hi = B.buildLShr(Ty, MI.getOperand(1).getReg(), C_1);
4096 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55));
4097 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0);
4098 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
4099 auto B2Count = B.buildSub(Ty, MI.getOperand(1).getReg(), B2Count1Hi);
4100
4101 // In order to get count in blocks of 4 add values from adjacent block of 2.
4102 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
4103 auto C_2 = B.buildConstant(Ty, 2);
4104 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2);
4105 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33));
4106 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0);
4107 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
4108 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
4109 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
4110
4111 // For count in blocks of 8 bits we don't have to mask high 4 bits before
4112 // addition since count value sits in range {0,...,8} and 4 bits are enough
4113 // to hold such binary values. After addition high 4 bits still hold count
4114 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
4115 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
4116 auto C_4 = B.buildConstant(Ty, 4);
4117 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4);
4118 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count);
4119 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F));
4120 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0);
4121 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
4122
4123 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm");
4124 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
4125 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
4126 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01)));
4127 auto ResTmp = B.buildMul(Ty, B8Count, MulMask);
4128
4129 // Shift count result from 8 high bits to low bits.
4130 auto C_SizeM8 = B.buildConstant(Ty, Size - 8);
4131 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
4132
4133 MI.eraseFromParent();
4134 return Legalized;
4135 }
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00004136 }
4137}
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004138
4139// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
4140// representation.
4141LegalizerHelper::LegalizeResult
4142LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004143 Register Dst = MI.getOperand(0).getReg();
4144 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004145 const LLT S64 = LLT::scalar(64);
4146 const LLT S32 = LLT::scalar(32);
4147 const LLT S1 = LLT::scalar(1);
4148
4149 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
4150
4151 // unsigned cul2f(ulong u) {
4152 // uint lz = clz(u);
4153 // uint e = (u != 0) ? 127U + 63U - lz : 0;
4154 // u = (u << lz) & 0x7fffffffffffffffUL;
4155 // ulong t = u & 0xffffffffffUL;
4156 // uint v = (e << 23) | (uint)(u >> 40);
4157 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
4158 // return as_float(v + r);
4159 // }
4160
4161 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
4162 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
4163
4164 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
4165
4166 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
4167 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
4168
4169 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
4170 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
4171
4172 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
4173 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
4174
4175 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
4176
4177 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
4178 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
4179
4180 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
4181 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
4182 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
4183
4184 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
4185 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
4186 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
4187 auto One = MIRBuilder.buildConstant(S32, 1);
4188
4189 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
4190 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
4191 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
4192 MIRBuilder.buildAdd(Dst, V, R);
4193
4194 return Legalized;
4195}
4196
4197LegalizerHelper::LegalizeResult
4198LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004199 Register Dst = MI.getOperand(0).getReg();
4200 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004201 LLT DstTy = MRI.getType(Dst);
4202 LLT SrcTy = MRI.getType(Src);
4203
Matt Arsenaultbc276c62019-11-15 11:59:12 +05304204 if (SrcTy == LLT::scalar(1)) {
4205 auto True = MIRBuilder.buildFConstant(DstTy, 1.0);
4206 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
4207 MIRBuilder.buildSelect(Dst, Src, True, False);
4208 MI.eraseFromParent();
4209 return Legalized;
4210 }
4211
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004212 if (SrcTy != LLT::scalar(64))
4213 return UnableToLegalize;
4214
4215 if (DstTy == LLT::scalar(32)) {
4216 // TODO: SelectionDAG has several alternative expansions to port which may
4217 // be more reasonble depending on the available instructions. If a target
4218 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
4219 // intermediate type, this is probably worse.
4220 return lowerU64ToF32BitOps(MI);
4221 }
4222
4223 return UnableToLegalize;
4224}
4225
4226LegalizerHelper::LegalizeResult
4227LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004228 Register Dst = MI.getOperand(0).getReg();
4229 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004230 LLT DstTy = MRI.getType(Dst);
4231 LLT SrcTy = MRI.getType(Src);
4232
4233 const LLT S64 = LLT::scalar(64);
4234 const LLT S32 = LLT::scalar(32);
4235 const LLT S1 = LLT::scalar(1);
4236
Matt Arsenaultbc276c62019-11-15 11:59:12 +05304237 if (SrcTy == S1) {
4238 auto True = MIRBuilder.buildFConstant(DstTy, -1.0);
4239 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
4240 MIRBuilder.buildSelect(Dst, Src, True, False);
4241 MI.eraseFromParent();
4242 return Legalized;
4243 }
4244
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004245 if (SrcTy != S64)
4246 return UnableToLegalize;
4247
4248 if (DstTy == S32) {
4249 // signed cl2f(long l) {
4250 // long s = l >> 63;
4251 // float r = cul2f((l + s) ^ s);
4252 // return s ? -r : r;
4253 // }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00004254 Register L = Src;
Matt Arsenault02b5ca82019-05-17 23:05:13 +00004255 auto SignBit = MIRBuilder.buildConstant(S64, 63);
4256 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
4257
4258 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
4259 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
4260 auto R = MIRBuilder.buildUITOFP(S32, Xor);
4261
4262 auto RNeg = MIRBuilder.buildFNeg(S32, R);
4263 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
4264 MIRBuilder.buildConstant(S64, 0));
4265 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
4266 return Legalized;
4267 }
4268
4269 return UnableToLegalize;
4270}
Matt Arsenault6f74f552019-07-01 17:18:03 +00004271
Petar Avramovic6412b562019-08-30 05:44:02 +00004272LegalizerHelper::LegalizeResult
4273LegalizerHelper::lowerFPTOUI(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
4274 Register Dst = MI.getOperand(0).getReg();
4275 Register Src = MI.getOperand(1).getReg();
4276 LLT DstTy = MRI.getType(Dst);
4277 LLT SrcTy = MRI.getType(Src);
4278 const LLT S64 = LLT::scalar(64);
4279 const LLT S32 = LLT::scalar(32);
4280
4281 if (SrcTy != S64 && SrcTy != S32)
4282 return UnableToLegalize;
4283 if (DstTy != S32 && DstTy != S64)
4284 return UnableToLegalize;
4285
4286 // FPTOSI gives same result as FPTOUI for positive signed integers.
4287 // FPTOUI needs to deal with fp values that convert to unsigned integers
4288 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
4289
4290 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
4291 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
4292 : APFloat::IEEEdouble(),
4293 APInt::getNullValue(SrcTy.getSizeInBits()));
4294 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
4295
4296 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
4297
4298 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
4299 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
4300 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
4301 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
4302 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
4303 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
4304 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
4305
Matt Arsenault1060b9e2020-01-04 17:06:47 -05004306 const LLT S1 = LLT::scalar(1);
4307
Petar Avramovic6412b562019-08-30 05:44:02 +00004308 MachineInstrBuilder FCMP =
Matt Arsenault1060b9e2020-01-04 17:06:47 -05004309 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold);
Petar Avramovic6412b562019-08-30 05:44:02 +00004310 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
4311
4312 MI.eraseFromParent();
4313 return Legalized;
4314}
4315
Matt Arsenaultea956682020-01-04 17:09:48 -05004316LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
4317 Register Dst = MI.getOperand(0).getReg();
4318 Register Src = MI.getOperand(1).getReg();
4319 LLT DstTy = MRI.getType(Dst);
4320 LLT SrcTy = MRI.getType(Src);
4321 const LLT S64 = LLT::scalar(64);
4322 const LLT S32 = LLT::scalar(32);
4323
4324 // FIXME: Only f32 to i64 conversions are supported.
4325 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
4326 return UnableToLegalize;
4327
4328 // Expand f32 -> i64 conversion
4329 // This algorithm comes from compiler-rt's implementation of fixsfdi:
4330 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c
4331
4332 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
4333
4334 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000);
4335 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23);
4336
4337 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask);
4338 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit);
4339
4340 auto SignMask = MIRBuilder.buildConstant(SrcTy,
4341 APInt::getSignMask(SrcEltBits));
4342 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask);
4343 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1);
4344 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit);
4345 Sign = MIRBuilder.buildSExt(DstTy, Sign);
4346
4347 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF);
4348 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask);
4349 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000);
4350
4351 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K);
4352 R = MIRBuilder.buildZExt(DstTy, R);
4353
4354 auto Bias = MIRBuilder.buildConstant(SrcTy, 127);
4355 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias);
4356 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit);
4357 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent);
4358
4359 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent);
4360 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub);
4361
4362 const LLT S1 = LLT::scalar(1);
4363 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT,
4364 S1, Exponent, ExponentLoBit);
4365
4366 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl);
4367
4368 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign);
4369 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign);
4370
4371 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0);
4372
4373 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT,
4374 S1, Exponent, ZeroSrcTy);
4375
4376 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0);
4377 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret);
4378
4379 MI.eraseFromParent();
4380 return Legalized;
4381}
4382
Matt Arsenault6f74f552019-07-01 17:18:03 +00004383static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
4384 switch (Opc) {
4385 case TargetOpcode::G_SMIN:
4386 return CmpInst::ICMP_SLT;
4387 case TargetOpcode::G_SMAX:
4388 return CmpInst::ICMP_SGT;
4389 case TargetOpcode::G_UMIN:
4390 return CmpInst::ICMP_ULT;
4391 case TargetOpcode::G_UMAX:
4392 return CmpInst::ICMP_UGT;
4393 default:
4394 llvm_unreachable("not in integer min/max");
4395 }
4396}
4397
4398LegalizerHelper::LegalizeResult
4399LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
4400 Register Dst = MI.getOperand(0).getReg();
4401 Register Src0 = MI.getOperand(1).getReg();
4402 Register Src1 = MI.getOperand(2).getReg();
4403
4404 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
4405 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
4406
4407 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
4408 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
4409
4410 MI.eraseFromParent();
4411 return Legalized;
4412}
Matt Arsenaultb1843e12019-07-09 23:34:29 +00004413
4414LegalizerHelper::LegalizeResult
4415LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
4416 Register Dst = MI.getOperand(0).getReg();
4417 Register Src0 = MI.getOperand(1).getReg();
4418 Register Src1 = MI.getOperand(2).getReg();
4419
4420 const LLT Src0Ty = MRI.getType(Src0);
4421 const LLT Src1Ty = MRI.getType(Src1);
4422
4423 const int Src0Size = Src0Ty.getScalarSizeInBits();
4424 const int Src1Size = Src1Ty.getScalarSizeInBits();
4425
4426 auto SignBitMask = MIRBuilder.buildConstant(
4427 Src0Ty, APInt::getSignMask(Src0Size));
4428
4429 auto NotSignBitMask = MIRBuilder.buildConstant(
4430 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
4431
4432 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
4433 MachineInstr *Or;
4434
4435 if (Src0Ty == Src1Ty) {
4436 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
4437 Or = MIRBuilder.buildOr(Dst, And0, And1);
4438 } else if (Src0Size > Src1Size) {
4439 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
4440 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
4441 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
4442 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
4443 Or = MIRBuilder.buildOr(Dst, And0, And1);
4444 } else {
4445 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
4446 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
4447 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
4448 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
4449 Or = MIRBuilder.buildOr(Dst, And0, And1);
4450 }
4451
4452 // Be careful about setting nsz/nnan/ninf on every instruction, since the
4453 // constants are a nan and -0.0, but the final result should preserve
4454 // everything.
4455 if (unsigned Flags = MI.getFlags())
4456 Or->setFlags(Flags);
4457
4458 MI.eraseFromParent();
4459 return Legalized;
4460}
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00004461
4462LegalizerHelper::LegalizeResult
4463LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
4464 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
4465 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
4466
4467 Register Dst = MI.getOperand(0).getReg();
4468 Register Src0 = MI.getOperand(1).getReg();
4469 Register Src1 = MI.getOperand(2).getReg();
4470 LLT Ty = MRI.getType(Dst);
4471
4472 if (!MI.getFlag(MachineInstr::FmNoNans)) {
4473 // Insert canonicalizes if it's possible we need to quiet to get correct
4474 // sNaN behavior.
4475
4476 // Note this must be done here, and not as an optimization combine in the
4477 // absence of a dedicate quiet-snan instruction as we're using an
4478 // omni-purpose G_FCANONICALIZE.
4479 if (!isKnownNeverSNaN(Src0, MRI))
4480 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
4481
4482 if (!isKnownNeverSNaN(Src1, MRI))
4483 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
4484 }
4485
4486 // If there are no nans, it's safe to simply replace this with the non-IEEE
4487 // version.
4488 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
4489 MI.eraseFromParent();
4490 return Legalized;
4491}
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00004492
Matt Arsenault4d339182019-09-13 00:44:35 +00004493LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
4494 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
4495 Register DstReg = MI.getOperand(0).getReg();
4496 LLT Ty = MRI.getType(DstReg);
4497 unsigned Flags = MI.getFlags();
4498
4499 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
4500 Flags);
4501 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
4502 MI.eraseFromParent();
4503 return Legalized;
4504}
4505
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00004506LegalizerHelper::LegalizeResult
Matt Arsenaultf3de8ab2019-12-24 14:49:31 -05004507LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
4508 Register DstReg = MI.getOperand(0).getReg();
4509 Register SrcReg = MI.getOperand(1).getReg();
4510 unsigned Flags = MI.getFlags();
4511 LLT Ty = MRI.getType(DstReg);
4512 const LLT CondTy = Ty.changeElementSize(1);
4513
4514 // result = trunc(src);
4515 // if (src < 0.0 && src != result)
4516 // result += -1.0.
4517
4518 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
4519 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags);
4520
4521 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy,
4522 SrcReg, Zero, Flags);
4523 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy,
4524 SrcReg, Trunc, Flags);
4525 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc);
4526 auto AddVal = MIRBuilder.buildSITOFP(Ty, And);
4527
4528 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal);
4529 MI.eraseFromParent();
4530 return Legalized;
4531}
4532
4533LegalizerHelper::LegalizeResult
Matt Arsenaultd9d30a42019-08-01 19:10:05 +00004534LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
4535 const unsigned NumDst = MI.getNumOperands() - 1;
4536 const Register SrcReg = MI.getOperand(NumDst).getReg();
4537 LLT SrcTy = MRI.getType(SrcReg);
4538
4539 Register Dst0Reg = MI.getOperand(0).getReg();
4540 LLT DstTy = MRI.getType(Dst0Reg);
4541
4542
4543 // Expand scalarizing unmerge as bitcast to integer and shift.
4544 if (!DstTy.isVector() && SrcTy.isVector() &&
4545 SrcTy.getElementType() == DstTy) {
4546 LLT IntTy = LLT::scalar(SrcTy.getSizeInBits());
4547 Register Cast = MIRBuilder.buildBitcast(IntTy, SrcReg).getReg(0);
4548
4549 MIRBuilder.buildTrunc(Dst0Reg, Cast);
4550
4551 const unsigned DstSize = DstTy.getSizeInBits();
4552 unsigned Offset = DstSize;
4553 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
4554 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
4555 auto Shift = MIRBuilder.buildLShr(IntTy, Cast, ShiftAmt);
4556 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
4557 }
4558
4559 MI.eraseFromParent();
4560 return Legalized;
4561 }
4562
4563 return UnableToLegalize;
4564}
Matt Arsenault690645b2019-08-13 16:09:07 +00004565
4566LegalizerHelper::LegalizeResult
4567LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
4568 Register DstReg = MI.getOperand(0).getReg();
4569 Register Src0Reg = MI.getOperand(1).getReg();
4570 Register Src1Reg = MI.getOperand(2).getReg();
Aditya Nandakumar615eee62019-08-13 21:49:11 +00004571 LLT Src0Ty = MRI.getType(Src0Reg);
Matt Arsenault690645b2019-08-13 16:09:07 +00004572 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault690645b2019-08-13 16:09:07 +00004573 LLT IdxTy = LLT::scalar(32);
4574
Eli Friedmane68e4cb2020-01-13 15:32:45 -08004575 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
Matt Arsenault690645b2019-08-13 16:09:07 +00004576
Amara Emersonc8092302019-08-16 18:06:53 +00004577 if (DstTy.isScalar()) {
4578 if (Src0Ty.isVector())
4579 return UnableToLegalize;
4580
4581 // This is just a SELECT.
4582 assert(Mask.size() == 1 && "Expected a single mask element");
4583 Register Val;
4584 if (Mask[0] < 0 || Mask[0] > 1)
4585 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
4586 else
4587 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
4588 MIRBuilder.buildCopy(DstReg, Val);
4589 MI.eraseFromParent();
4590 return Legalized;
4591 }
4592
Matt Arsenault690645b2019-08-13 16:09:07 +00004593 Register Undef;
4594 SmallVector<Register, 32> BuildVec;
Amara Emersonc8092302019-08-16 18:06:53 +00004595 LLT EltTy = DstTy.getElementType();
Matt Arsenault690645b2019-08-13 16:09:07 +00004596
4597 for (int Idx : Mask) {
4598 if (Idx < 0) {
4599 if (!Undef.isValid())
4600 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
4601 BuildVec.push_back(Undef);
4602 continue;
4603 }
4604
Aditya Nandakumar615eee62019-08-13 21:49:11 +00004605 if (Src0Ty.isScalar()) {
4606 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
4607 } else {
Aditya Nandakumarc65ac862019-08-14 01:23:33 +00004608 int NumElts = Src0Ty.getNumElements();
Aditya Nandakumar615eee62019-08-13 21:49:11 +00004609 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
4610 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
4611 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
4612 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
4613 BuildVec.push_back(Extract.getReg(0));
4614 }
Matt Arsenault690645b2019-08-13 16:09:07 +00004615 }
4616
4617 MIRBuilder.buildBuildVector(DstReg, BuildVec);
4618 MI.eraseFromParent();
4619 return Legalized;
4620}
Amara Emersone20b91c2019-08-27 19:54:27 +00004621
4622LegalizerHelper::LegalizeResult
4623LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
4624 Register Dst = MI.getOperand(0).getReg();
4625 Register AllocSize = MI.getOperand(1).getReg();
4626 unsigned Align = MI.getOperand(2).getImm();
4627
4628 const auto &MF = *MI.getMF();
4629 const auto &TLI = *MF.getSubtarget().getTargetLowering();
4630
4631 LLT PtrTy = MRI.getType(Dst);
4632 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
4633
4634 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
4635 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
4636 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
4637
4638 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
4639 // have to generate an extra instruction to negate the alloc and then use
Daniel Sanderse74c5b92019-11-01 13:18:00 -07004640 // G_PTR_ADD to add the negative offset.
Amara Emersone20b91c2019-08-27 19:54:27 +00004641 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
4642 if (Align) {
4643 APInt AlignMask(IntPtrTy.getSizeInBits(), Align, true);
4644 AlignMask.negate();
4645 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
4646 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
4647 }
4648
4649 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
4650 MIRBuilder.buildCopy(SPReg, SPTmp);
4651 MIRBuilder.buildCopy(Dst, SPTmp);
4652
4653 MI.eraseFromParent();
4654 return Legalized;
4655}
Matt Arsenaulta5b9c752019-10-06 01:37:35 +00004656
4657LegalizerHelper::LegalizeResult
4658LegalizerHelper::lowerExtract(MachineInstr &MI) {
4659 Register Dst = MI.getOperand(0).getReg();
4660 Register Src = MI.getOperand(1).getReg();
4661 unsigned Offset = MI.getOperand(2).getImm();
4662
4663 LLT DstTy = MRI.getType(Dst);
4664 LLT SrcTy = MRI.getType(Src);
4665
4666 if (DstTy.isScalar() &&
4667 (SrcTy.isScalar() ||
4668 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
4669 LLT SrcIntTy = SrcTy;
4670 if (!SrcTy.isScalar()) {
4671 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
4672 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
4673 }
4674
4675 if (Offset == 0)
4676 MIRBuilder.buildTrunc(Dst, Src);
4677 else {
4678 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
4679 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
4680 MIRBuilder.buildTrunc(Dst, Shr);
4681 }
4682
4683 MI.eraseFromParent();
4684 return Legalized;
4685 }
4686
4687 return UnableToLegalize;
4688}
Matt Arsenault4bcdcad2019-10-07 19:13:27 +00004689
4690LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
4691 Register Dst = MI.getOperand(0).getReg();
4692 Register Src = MI.getOperand(1).getReg();
4693 Register InsertSrc = MI.getOperand(2).getReg();
4694 uint64_t Offset = MI.getOperand(3).getImm();
4695
4696 LLT DstTy = MRI.getType(Src);
4697 LLT InsertTy = MRI.getType(InsertSrc);
4698
4699 if (InsertTy.isScalar() &&
4700 (DstTy.isScalar() ||
4701 (DstTy.isVector() && DstTy.getElementType() == InsertTy))) {
4702 LLT IntDstTy = DstTy;
4703 if (!DstTy.isScalar()) {
4704 IntDstTy = LLT::scalar(DstTy.getSizeInBits());
4705 Src = MIRBuilder.buildBitcast(IntDstTy, Src).getReg(0);
4706 }
4707
4708 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
4709 if (Offset != 0) {
4710 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
4711 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
4712 }
4713
Matt Arsenaultb63629a2020-01-21 18:38:19 -05004714 APInt MaskVal = APInt::getBitsSetWithWrap(DstTy.getSizeInBits(),
4715 Offset + InsertTy.getSizeInBits(),
4716 Offset);
Matt Arsenault4bcdcad2019-10-07 19:13:27 +00004717
4718 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
4719 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
4720 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
4721
4722 MIRBuilder.buildBitcast(Dst, Or);
4723 MI.eraseFromParent();
4724 return Legalized;
4725 }
4726
4727 return UnableToLegalize;
4728}
Matt Arsenault34ed76e2019-10-16 20:46:32 +00004729
4730LegalizerHelper::LegalizeResult
4731LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
4732 Register Dst0 = MI.getOperand(0).getReg();
4733 Register Dst1 = MI.getOperand(1).getReg();
4734 Register LHS = MI.getOperand(2).getReg();
4735 Register RHS = MI.getOperand(3).getReg();
4736 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
4737
4738 LLT Ty = MRI.getType(Dst0);
4739 LLT BoolTy = MRI.getType(Dst1);
4740
4741 if (IsAdd)
4742 MIRBuilder.buildAdd(Dst0, LHS, RHS);
4743 else
4744 MIRBuilder.buildSub(Dst0, LHS, RHS);
4745
4746 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
4747
4748 auto Zero = MIRBuilder.buildConstant(Ty, 0);
4749
4750 // For an addition, the result should be less than one of the operands (LHS)
4751 // if and only if the other operand (RHS) is negative, otherwise there will
4752 // be overflow.
4753 // For a subtraction, the result should be less than one of the operands
4754 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
4755 // otherwise there will be overflow.
4756 auto ResultLowerThanLHS =
4757 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS);
4758 auto ConditionRHS = MIRBuilder.buildICmp(
4759 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero);
4760
4761 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS);
4762 MI.eraseFromParent();
4763 return Legalized;
4764}
Petar Avramovic94a24e72019-12-30 11:13:22 +01004765
4766LegalizerHelper::LegalizeResult
4767LegalizerHelper::lowerBswap(MachineInstr &MI) {
4768 Register Dst = MI.getOperand(0).getReg();
4769 Register Src = MI.getOperand(1).getReg();
4770 const LLT Ty = MRI.getType(Src);
4771 unsigned SizeInBytes = Ty.getSizeInBytes();
4772 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
4773
4774 // Swap most and least significant byte, set remaining bytes in Res to zero.
4775 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt);
4776 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt);
4777 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
4778 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft);
4779
4780 // Set i-th high/low byte in Res to i-th low/high byte from Src.
4781 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
4782 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
4783 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
4784 auto Mask = MIRBuilder.buildConstant(Ty, APMask);
4785 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i);
4786 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
4787 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask);
4788 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt);
4789 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft);
4790 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
4791 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
4792 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask);
4793 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight);
4794 }
4795 Res.getInstr()->getOperand(0).setReg(Dst);
4796
4797 MI.eraseFromParent();
4798 return Legalized;
4799}
Petar Avramovic98f72a52019-12-30 18:06:29 +01004800
4801//{ (Src & Mask) >> N } | { (Src << N) & Mask }
4802static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
4803 MachineInstrBuilder Src, APInt Mask) {
4804 const LLT Ty = Dst.getLLTTy(*B.getMRI());
4805 MachineInstrBuilder C_N = B.buildConstant(Ty, N);
4806 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
4807 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
4808 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0);
4809 return B.buildOr(Dst, LHS, RHS);
4810}
4811
4812LegalizerHelper::LegalizeResult
4813LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
4814 Register Dst = MI.getOperand(0).getReg();
4815 Register Src = MI.getOperand(1).getReg();
4816 const LLT Ty = MRI.getType(Src);
4817 unsigned Size = Ty.getSizeInBits();
4818
4819 MachineInstrBuilder BSWAP =
4820 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src});
4821
4822 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
4823 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
4824 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
4825 MachineInstrBuilder Swap4 =
4826 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0)));
4827
4828 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
4829 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
4830 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
4831 MachineInstrBuilder Swap2 =
4832 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC)));
4833
4834 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7
4835 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
4836 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
4837 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA)));
4838
4839 MI.eraseFromParent();
4840 return Legalized;
4841}
Matt Arsenault0ea3c722019-12-27 19:26:51 -05004842
4843LegalizerHelper::LegalizeResult
Matt Arsenaultc5c1bb32020-01-12 13:29:44 -05004844LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
Matt Arsenault0ea3c722019-12-27 19:26:51 -05004845 MachineFunction &MF = MIRBuilder.getMF();
4846 const TargetSubtargetInfo &STI = MF.getSubtarget();
4847 const TargetLowering *TLI = STI.getTargetLowering();
Matt Arsenaultc5c1bb32020-01-12 13:29:44 -05004848
4849 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
4850 int NameOpIdx = IsRead ? 1 : 0;
4851 int ValRegIndex = IsRead ? 0 : 1;
4852
4853 Register ValReg = MI.getOperand(ValRegIndex).getReg();
4854 const LLT Ty = MRI.getType(ValReg);
4855 const MDString *RegStr = cast<MDString>(
4856 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
4857
4858 Register PhysReg = TLI->getRegisterByName(RegStr->getString().data(), Ty, MF);
4859 if (!PhysReg.isValid())
Matt Arsenault0ea3c722019-12-27 19:26:51 -05004860 return UnableToLegalize;
4861
Matt Arsenaultc5c1bb32020-01-12 13:29:44 -05004862 if (IsRead)
4863 MIRBuilder.buildCopy(ValReg, PhysReg);
4864 else
4865 MIRBuilder.buildCopy(PhysReg, ValReg);
4866
Matt Arsenault0ea3c722019-12-27 19:26:51 -05004867 MI.eraseFromParent();
4868 return Legalized;
4869}