blob: d5a4e891ce6903656d9e142babc9a69539fa5272 [file] [log] [blame]
Tim Northover69fa84a2016-10-14 22:18:18 +00001//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
Tim Northover33b07d62016-07-22 20:03:43 +00002//
Chandler Carruth2946cd72019-01-19 08:50:56 +00003// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Tim Northover33b07d62016-07-22 20:03:43 +00006//
7//===----------------------------------------------------------------------===//
8//
Tim Northover69fa84a2016-10-14 22:18:18 +00009/// \file This file implements the LegalizerHelper class to legalize
Tim Northover33b07d62016-07-22 20:03:43 +000010/// individual instructions and the LegalizeMachineIR wrapper pass for the
11/// primary legalization.
12//
13//===----------------------------------------------------------------------===//
14
Tim Northover69fa84a2016-10-14 22:18:18 +000015#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
Tim Northoveredb3c8c2016-08-29 19:07:16 +000016#include "llvm/CodeGen/GlobalISel/CallLowering.h"
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000017#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
Tim Northover69fa84a2016-10-14 22:18:18 +000018#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000019#include "llvm/CodeGen/MachineRegisterInfo.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000020#include "llvm/CodeGen/TargetInstrInfo.h"
David Blaikieb3bde2e2017-11-17 01:07:10 +000021#include "llvm/CodeGen/TargetLowering.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
Tim Northover33b07d62016-07-22 20:03:43 +000023#include "llvm/Support/Debug.h"
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000024#include "llvm/Support/MathExtras.h"
Tim Northover33b07d62016-07-22 20:03:43 +000025#include "llvm/Support/raw_ostream.h"
Tim Northover33b07d62016-07-22 20:03:43 +000026
Daniel Sanders5377fb32017-04-20 15:46:12 +000027#define DEBUG_TYPE "legalizer"
Tim Northover33b07d62016-07-22 20:03:43 +000028
29using namespace llvm;
Daniel Sanders9ade5592018-01-29 17:37:29 +000030using namespace LegalizeActions;
Tim Northover33b07d62016-07-22 20:03:43 +000031
Matt Arsenaultc83b8232019-02-07 17:38:00 +000032/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
33///
34/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
35/// with any leftover piece as type \p LeftoverTy
36///
Matt Arsenaultd3093c22019-02-28 00:16:32 +000037/// Returns -1 in the first element of the pair if the breakdown is not
38/// satisfiable.
39static std::pair<int, int>
40getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
Matt Arsenaultc83b8232019-02-07 17:38:00 +000041 assert(!LeftoverTy.isValid() && "this is an out argument");
42
43 unsigned Size = OrigTy.getSizeInBits();
44 unsigned NarrowSize = NarrowTy.getSizeInBits();
45 unsigned NumParts = Size / NarrowSize;
46 unsigned LeftoverSize = Size - NumParts * NarrowSize;
47 assert(Size > NarrowSize);
48
49 if (LeftoverSize == 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000050 return {NumParts, 0};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000051
52 if (NarrowTy.isVector()) {
53 unsigned EltSize = OrigTy.getScalarSizeInBits();
54 if (LeftoverSize % EltSize != 0)
Matt Arsenaultd3093c22019-02-28 00:16:32 +000055 return {-1, -1};
Matt Arsenaultc83b8232019-02-07 17:38:00 +000056 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
57 } else {
58 LeftoverTy = LLT::scalar(LeftoverSize);
59 }
60
Matt Arsenaultd3093c22019-02-28 00:16:32 +000061 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
62 return std::make_pair(NumParts, NumLeftover);
Matt Arsenaultc83b8232019-02-07 17:38:00 +000063}
64
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000065LegalizerHelper::LegalizerHelper(MachineFunction &MF,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +000066 GISelChangeObserver &Observer,
67 MachineIRBuilder &Builder)
68 : MIRBuilder(Builder), MRI(MF.getRegInfo()),
69 LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
Tim Northover33b07d62016-07-22 20:03:43 +000070 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000071 MIRBuilder.setChangeObserver(Observer);
Tim Northover33b07d62016-07-22 20:03:43 +000072}
73
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000074LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
Aditya Nandakumar500e3ea2019-01-16 00:40:37 +000075 GISelChangeObserver &Observer,
76 MachineIRBuilder &B)
77 : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000078 MIRBuilder.setMF(MF);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +000079 MIRBuilder.setChangeObserver(Observer);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +000080}
Tim Northover69fa84a2016-10-14 22:18:18 +000081LegalizerHelper::LegalizeResult
Volkan Keles685fbda2017-03-10 18:34:57 +000082LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
Nicola Zaghend34e60c2018-05-14 12:53:11 +000083 LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
Daniel Sanders5377fb32017-04-20 15:46:12 +000084
Aditya Nandakumar1023a2e2019-07-01 17:53:50 +000085 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
86 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
87 return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized
88 : UnableToLegalize;
Daniel Sanders262ed0e2018-01-24 17:17:46 +000089 auto Step = LI.getAction(MI, MRI);
90 switch (Step.Action) {
Daniel Sanders9ade5592018-01-29 17:37:29 +000091 case Legal:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000092 LLVM_DEBUG(dbgs() << ".. Already legal\n");
Tim Northover33b07d62016-07-22 20:03:43 +000093 return AlreadyLegal;
Daniel Sanders9ade5592018-01-29 17:37:29 +000094 case Libcall:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000095 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
Tim Northoveredb3c8c2016-08-29 19:07:16 +000096 return libcall(MI);
Daniel Sanders9ade5592018-01-29 17:37:29 +000097 case NarrowScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +000098 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +000099 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000100 case WidenScalar:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000101 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000102 return widenScalar(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000103 case Lower:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000104 LLVM_DEBUG(dbgs() << ".. Lower\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000105 return lower(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000106 case FewerElements:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000107 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
Daniel Sanders262ed0e2018-01-24 17:17:46 +0000108 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
Matt Arsenault18ec3822019-02-11 22:00:39 +0000109 case MoreElements:
110 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
111 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
Daniel Sanders9ade5592018-01-29 17:37:29 +0000112 case Custom:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000113 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +0000114 return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
115 : UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +0000116 default:
Nicola Zaghend34e60c2018-05-14 12:53:11 +0000117 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
Tim Northover33b07d62016-07-22 20:03:43 +0000118 return UnableToLegalize;
119 }
120}
121
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000122void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
123 SmallVectorImpl<Register> &VRegs) {
Tim Northoverbf017292017-03-03 22:46:09 +0000124 for (int i = 0; i < NumParts; ++i)
Tim Northover0f140c72016-09-09 11:46:34 +0000125 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
Tim Northoverbf017292017-03-03 22:46:09 +0000126 MIRBuilder.buildUnmerge(VRegs, Reg);
Tim Northover33b07d62016-07-22 20:03:43 +0000127}
128
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000129bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000130 LLT MainTy, LLT &LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000131 SmallVectorImpl<Register> &VRegs,
132 SmallVectorImpl<Register> &LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000133 assert(!LeftoverTy.isValid() && "this is an out argument");
134
135 unsigned RegSize = RegTy.getSizeInBits();
136 unsigned MainSize = MainTy.getSizeInBits();
137 unsigned NumParts = RegSize / MainSize;
138 unsigned LeftoverSize = RegSize - NumParts * MainSize;
139
140 // Use an unmerge when possible.
141 if (LeftoverSize == 0) {
142 for (unsigned I = 0; I < NumParts; ++I)
143 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
144 MIRBuilder.buildUnmerge(VRegs, Reg);
145 return true;
146 }
147
148 if (MainTy.isVector()) {
149 unsigned EltSize = MainTy.getScalarSizeInBits();
150 if (LeftoverSize % EltSize != 0)
151 return false;
152 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
153 } else {
154 LeftoverTy = LLT::scalar(LeftoverSize);
155 }
156
157 // For irregular sizes, extract the individual parts.
158 for (unsigned I = 0; I != NumParts; ++I) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000159 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000160 VRegs.push_back(NewReg);
161 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
162 }
163
164 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
165 Offset += LeftoverSize) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000166 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000167 LeftoverRegs.push_back(NewReg);
168 MIRBuilder.buildExtract(NewReg, Reg, Offset);
169 }
170
171 return true;
172}
173
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000174void LegalizerHelper::insertParts(Register DstReg,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000175 LLT ResultTy, LLT PartTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000176 ArrayRef<Register> PartRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000177 LLT LeftoverTy,
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000178 ArrayRef<Register> LeftoverRegs) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000179 if (!LeftoverTy.isValid()) {
180 assert(LeftoverRegs.empty());
181
Matt Arsenault81511e52019-02-05 00:13:44 +0000182 if (!ResultTy.isVector()) {
183 MIRBuilder.buildMerge(DstReg, PartRegs);
184 return;
185 }
186
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000187 if (PartTy.isVector())
188 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
189 else
190 MIRBuilder.buildBuildVector(DstReg, PartRegs);
191 return;
192 }
193
194 unsigned PartSize = PartTy.getSizeInBits();
195 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
196
Matt Arsenault3018d182019-06-28 01:47:44 +0000197 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000198 MIRBuilder.buildUndef(CurResultReg);
199
200 unsigned Offset = 0;
Matt Arsenault3018d182019-06-28 01:47:44 +0000201 for (Register PartReg : PartRegs) {
202 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000203 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
204 CurResultReg = NewResultReg;
205 Offset += PartSize;
206 }
207
208 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
209 // Use the original output register for the final insert to avoid a copy.
Matt Arsenault3018d182019-06-28 01:47:44 +0000210 Register NewResultReg = (I + 1 == E) ?
Matt Arsenaultc7bce732019-01-31 02:46:05 +0000211 DstReg : MRI.createGenericVirtualRegister(ResultTy);
212
213 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
214 CurResultReg = NewResultReg;
215 Offset += LeftoverPartSize;
216 }
217}
218
Tim Northovere0418412017-02-08 23:23:39 +0000219static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
220 switch (Opcode) {
Diana Picuse97822e2017-04-24 07:22:31 +0000221 case TargetOpcode::G_SDIV:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000222 assert((Size == 32 || Size == 64) && "Unsupported size");
223 return Size == 64 ? RTLIB::SDIV_I64 : RTLIB::SDIV_I32;
Diana Picuse97822e2017-04-24 07:22:31 +0000224 case TargetOpcode::G_UDIV:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000225 assert((Size == 32 || Size == 64) && "Unsupported size");
226 return Size == 64 ? RTLIB::UDIV_I64 : RTLIB::UDIV_I32;
Diana Picus02e11012017-06-15 10:53:31 +0000227 case TargetOpcode::G_SREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000228 assert((Size == 32 || Size == 64) && "Unsupported size");
229 return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
Diana Picus02e11012017-06-15 10:53:31 +0000230 case TargetOpcode::G_UREM:
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000231 assert((Size == 32 || Size == 64) && "Unsupported size");
232 return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
Diana Picus0528e2c2018-11-26 11:07:02 +0000233 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
234 assert(Size == 32 && "Unsupported size");
235 return RTLIB::CTLZ_I32;
Diana Picus1314a282017-04-11 10:52:34 +0000236 case TargetOpcode::G_FADD:
237 assert((Size == 32 || Size == 64) && "Unsupported size");
238 return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
Javed Absar5cde1cc2017-10-30 13:51:56 +0000239 case TargetOpcode::G_FSUB:
240 assert((Size == 32 || Size == 64) && "Unsupported size");
241 return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
Diana Picus9faa09b2017-11-23 12:44:20 +0000242 case TargetOpcode::G_FMUL:
243 assert((Size == 32 || Size == 64) && "Unsupported size");
244 return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
Diana Picusc01f7f12017-11-23 13:26:07 +0000245 case TargetOpcode::G_FDIV:
246 assert((Size == 32 || Size == 64) && "Unsupported size");
247 return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
Jessica Paquette84bedac2019-01-30 23:46:15 +0000248 case TargetOpcode::G_FEXP:
249 assert((Size == 32 || Size == 64) && "Unsupported size");
250 return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
Jessica Paquettee7941212019-04-03 16:58:32 +0000251 case TargetOpcode::G_FEXP2:
252 assert((Size == 32 || Size == 64) && "Unsupported size");
253 return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000254 case TargetOpcode::G_FREM:
255 return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
256 case TargetOpcode::G_FPOW:
257 return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
Diana Picuse74243d2018-01-12 11:30:45 +0000258 case TargetOpcode::G_FMA:
259 assert((Size == 32 || Size == 64) && "Unsupported size");
260 return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
Jessica Paquette7db82d72019-01-28 18:34:18 +0000261 case TargetOpcode::G_FSIN:
262 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
263 return Size == 128 ? RTLIB::SIN_F128
264 : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
265 case TargetOpcode::G_FCOS:
266 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
267 return Size == 128 ? RTLIB::COS_F128
268 : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
Jessica Paquettec49428a2019-01-28 19:53:14 +0000269 case TargetOpcode::G_FLOG10:
270 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
271 return Size == 128 ? RTLIB::LOG10_F128
272 : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000273 case TargetOpcode::G_FLOG:
274 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
275 return Size == 128 ? RTLIB::LOG_F128
276 : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
Jessica Paquette0154bd12019-01-30 21:16:04 +0000277 case TargetOpcode::G_FLOG2:
278 assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
279 return Size == 128 ? RTLIB::LOG2_F128
280 : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000281 case TargetOpcode::G_FCEIL:
282 assert((Size == 32 || Size == 64) && "Unsupported size");
283 return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32;
284 case TargetOpcode::G_FFLOOR:
285 assert((Size == 32 || Size == 64) && "Unsupported size");
286 return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32;
Tim Northovere0418412017-02-08 23:23:39 +0000287 }
288 llvm_unreachable("Unknown libcall function");
289}
290
Diana Picusfc1675e2017-07-05 12:57:24 +0000291LegalizerHelper::LegalizeResult
292llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
293 const CallLowering::ArgInfo &Result,
294 ArrayRef<CallLowering::ArgInfo> Args) {
Diana Picuse97822e2017-04-24 07:22:31 +0000295 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
296 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
Diana Picuse97822e2017-04-24 07:22:31 +0000297 const char *Name = TLI.getLibcallName(Libcall);
Diana Picusd0104ea2017-07-06 09:09:33 +0000298
Diana Picuse97822e2017-04-24 07:22:31 +0000299 MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
Diana Picus02e11012017-06-15 10:53:31 +0000300 if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
301 MachineOperand::CreateES(Name), Result, Args))
302 return LegalizerHelper::UnableToLegalize;
Diana Picusd0104ea2017-07-06 09:09:33 +0000303
Diana Picuse97822e2017-04-24 07:22:31 +0000304 return LegalizerHelper::Legalized;
305}
306
Diana Picus65ed3642018-01-17 13:34:10 +0000307// Useful for libcalls where all operands have the same type.
Diana Picus02e11012017-06-15 10:53:31 +0000308static LegalizerHelper::LegalizeResult
309simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
310 Type *OpType) {
311 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
Diana Picuse74243d2018-01-12 11:30:45 +0000312
313 SmallVector<CallLowering::ArgInfo, 3> Args;
314 for (unsigned i = 1; i < MI.getNumOperands(); i++)
315 Args.push_back({MI.getOperand(i).getReg(), OpType});
Diana Picusfc1675e2017-07-05 12:57:24 +0000316 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
Diana Picuse74243d2018-01-12 11:30:45 +0000317 Args);
Diana Picus02e11012017-06-15 10:53:31 +0000318}
319
Diana Picus65ed3642018-01-17 13:34:10 +0000320static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
321 Type *FromType) {
322 auto ToMVT = MVT::getVT(ToType);
323 auto FromMVT = MVT::getVT(FromType);
324
325 switch (Opcode) {
326 case TargetOpcode::G_FPEXT:
327 return RTLIB::getFPEXT(FromMVT, ToMVT);
328 case TargetOpcode::G_FPTRUNC:
329 return RTLIB::getFPROUND(FromMVT, ToMVT);
Diana Picus4ed0ee72018-01-30 07:54:52 +0000330 case TargetOpcode::G_FPTOSI:
331 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
332 case TargetOpcode::G_FPTOUI:
333 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
Diana Picus517531e2018-01-30 09:15:17 +0000334 case TargetOpcode::G_SITOFP:
335 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
336 case TargetOpcode::G_UITOFP:
337 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
Diana Picus65ed3642018-01-17 13:34:10 +0000338 }
339 llvm_unreachable("Unsupported libcall function");
340}
341
342static LegalizerHelper::LegalizeResult
343conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
344 Type *FromType) {
345 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
346 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
347 {{MI.getOperand(1).getReg(), FromType}});
348}
349
Tim Northover69fa84a2016-10-14 22:18:18 +0000350LegalizerHelper::LegalizeResult
351LegalizerHelper::libcall(MachineInstr &MI) {
Diana Picus02e11012017-06-15 10:53:31 +0000352 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
353 unsigned Size = LLTy.getSizeInBits();
Matthias Braunf1caa282017-12-15 22:22:58 +0000354 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000355
Diana Picusfc1675e2017-07-05 12:57:24 +0000356 MIRBuilder.setInstr(MI);
357
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000358 switch (MI.getOpcode()) {
359 default:
360 return UnableToLegalize;
Diana Picuse97822e2017-04-24 07:22:31 +0000361 case TargetOpcode::G_SDIV:
Diana Picus02e11012017-06-15 10:53:31 +0000362 case TargetOpcode::G_UDIV:
363 case TargetOpcode::G_SREM:
Diana Picus0528e2c2018-11-26 11:07:02 +0000364 case TargetOpcode::G_UREM:
365 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
Petar Avramovic0a5e4eb2018-12-18 15:59:51 +0000366 Type *HLTy = IntegerType::get(Ctx, Size);
Diana Picusfc1675e2017-07-05 12:57:24 +0000367 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
368 if (Status != Legalized)
369 return Status;
370 break;
Diana Picuse97822e2017-04-24 07:22:31 +0000371 }
Diana Picus1314a282017-04-11 10:52:34 +0000372 case TargetOpcode::G_FADD:
Javed Absar5cde1cc2017-10-30 13:51:56 +0000373 case TargetOpcode::G_FSUB:
Diana Picus9faa09b2017-11-23 12:44:20 +0000374 case TargetOpcode::G_FMUL:
Diana Picusc01f7f12017-11-23 13:26:07 +0000375 case TargetOpcode::G_FDIV:
Diana Picuse74243d2018-01-12 11:30:45 +0000376 case TargetOpcode::G_FMA:
Tim Northovere0418412017-02-08 23:23:39 +0000377 case TargetOpcode::G_FPOW:
Jessica Paquette7db82d72019-01-28 18:34:18 +0000378 case TargetOpcode::G_FREM:
379 case TargetOpcode::G_FCOS:
Jessica Paquettec49428a2019-01-28 19:53:14 +0000380 case TargetOpcode::G_FSIN:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +0000381 case TargetOpcode::G_FLOG10:
Jessica Paquette0154bd12019-01-30 21:16:04 +0000382 case TargetOpcode::G_FLOG:
Jessica Paquette84bedac2019-01-30 23:46:15 +0000383 case TargetOpcode::G_FLOG2:
Jessica Paquettee7941212019-04-03 16:58:32 +0000384 case TargetOpcode::G_FEXP:
Petar Avramovicfaaa2b5d2019-06-06 09:02:24 +0000385 case TargetOpcode::G_FEXP2:
386 case TargetOpcode::G_FCEIL:
387 case TargetOpcode::G_FFLOOR: {
Jessica Paquette7db82d72019-01-28 18:34:18 +0000388 if (Size > 64) {
389 LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
390 return UnableToLegalize;
391 }
Diana Picus02e11012017-06-15 10:53:31 +0000392 Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
Diana Picusfc1675e2017-07-05 12:57:24 +0000393 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
394 if (Status != Legalized)
395 return Status;
396 break;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000397 }
Diana Picus65ed3642018-01-17 13:34:10 +0000398 case TargetOpcode::G_FPEXT: {
399 // FIXME: Support other floating point types (half, fp128 etc)
400 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
401 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
402 if (ToSize != 64 || FromSize != 32)
403 return UnableToLegalize;
404 LegalizeResult Status = conversionLibcall(
405 MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
406 if (Status != Legalized)
407 return Status;
408 break;
409 }
410 case TargetOpcode::G_FPTRUNC: {
411 // FIXME: Support other floating point types (half, fp128 etc)
412 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
413 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
414 if (ToSize != 32 || FromSize != 64)
415 return UnableToLegalize;
416 LegalizeResult Status = conversionLibcall(
417 MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
418 if (Status != Legalized)
419 return Status;
420 break;
421 }
Diana Picus4ed0ee72018-01-30 07:54:52 +0000422 case TargetOpcode::G_FPTOSI:
423 case TargetOpcode::G_FPTOUI: {
424 // FIXME: Support other types
425 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
426 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000427 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
Diana Picus4ed0ee72018-01-30 07:54:52 +0000428 return UnableToLegalize;
429 LegalizeResult Status = conversionLibcall(
Petar Avramovic4b4dae12019-06-20 08:52:53 +0000430 MI, MIRBuilder,
431 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
Diana Picus4ed0ee72018-01-30 07:54:52 +0000432 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
433 if (Status != Legalized)
434 return Status;
435 break;
436 }
Diana Picus517531e2018-01-30 09:15:17 +0000437 case TargetOpcode::G_SITOFP:
438 case TargetOpcode::G_UITOFP: {
439 // FIXME: Support other types
440 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
441 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
Petar Avramovic153bd242019-06-20 09:05:02 +0000442 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
Diana Picus517531e2018-01-30 09:15:17 +0000443 return UnableToLegalize;
444 LegalizeResult Status = conversionLibcall(
445 MI, MIRBuilder,
446 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
Petar Avramovic153bd242019-06-20 09:05:02 +0000447 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
Diana Picus517531e2018-01-30 09:15:17 +0000448 if (Status != Legalized)
449 return Status;
450 break;
451 }
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000452 }
Diana Picusfc1675e2017-07-05 12:57:24 +0000453
454 MI.eraseFromParent();
455 return Legalized;
Tim Northoveredb3c8c2016-08-29 19:07:16 +0000456}
457
Tim Northover69fa84a2016-10-14 22:18:18 +0000458LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
459 unsigned TypeIdx,
460 LLT NarrowTy) {
Justin Bognerfde01042017-01-18 17:29:54 +0000461 MIRBuilder.setInstr(MI);
462
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000463 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
464 uint64_t NarrowSize = NarrowTy.getSizeInBits();
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000465
Tim Northover9656f142016-08-04 20:54:13 +0000466 switch (MI.getOpcode()) {
467 default:
468 return UnableToLegalize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000469 case TargetOpcode::G_IMPLICIT_DEF: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000470 // FIXME: add support for when SizeOp0 isn't an exact multiple of
471 // NarrowSize.
472 if (SizeOp0 % NarrowSize != 0)
473 return UnableToLegalize;
474 int NumParts = SizeOp0 / NarrowSize;
Tim Northoverff5e7e12017-06-30 20:27:36 +0000475
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000476 SmallVector<Register, 2> DstRegs;
Volkan Keles02bb1742018-02-14 19:58:36 +0000477 for (int i = 0; i < NumParts; ++i)
478 DstRegs.push_back(
479 MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
Amara Emerson5ec146042018-12-10 18:44:58 +0000480
Matt Arsenault3018d182019-06-28 01:47:44 +0000481 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000482 if(MRI.getType(DstReg).isVector())
483 MIRBuilder.buildBuildVector(DstReg, DstRegs);
484 else
485 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northoverff5e7e12017-06-30 20:27:36 +0000486 MI.eraseFromParent();
487 return Legalized;
488 }
Matt Arsenault71872722019-04-10 17:27:53 +0000489 case TargetOpcode::G_CONSTANT: {
490 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
491 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
492 unsigned TotalSize = Ty.getSizeInBits();
493 unsigned NarrowSize = NarrowTy.getSizeInBits();
494 int NumParts = TotalSize / NarrowSize;
495
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000496 SmallVector<Register, 4> PartRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000497 for (int I = 0; I != NumParts; ++I) {
498 unsigned Offset = I * NarrowSize;
499 auto K = MIRBuilder.buildConstant(NarrowTy,
500 Val.lshr(Offset).trunc(NarrowSize));
501 PartRegs.push_back(K.getReg(0));
502 }
503
504 LLT LeftoverTy;
505 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000506 SmallVector<Register, 1> LeftoverRegs;
Matt Arsenault71872722019-04-10 17:27:53 +0000507 if (LeftoverBits != 0) {
508 LeftoverTy = LLT::scalar(LeftoverBits);
509 auto K = MIRBuilder.buildConstant(
510 LeftoverTy,
511 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
512 LeftoverRegs.push_back(K.getReg(0));
513 }
514
515 insertParts(MI.getOperand(0).getReg(),
516 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
517
518 MI.eraseFromParent();
519 return Legalized;
520 }
Tim Northover9656f142016-08-04 20:54:13 +0000521 case TargetOpcode::G_ADD: {
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000522 // FIXME: add support for when SizeOp0 isn't an exact multiple of
523 // NarrowSize.
524 if (SizeOp0 % NarrowSize != 0)
525 return UnableToLegalize;
Tim Northover9656f142016-08-04 20:54:13 +0000526 // Expand in terms of carry-setting/consuming G_ADDE instructions.
Kristof Beylsaf9814a2017-11-07 10:34:34 +0000527 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
Tim Northover9656f142016-08-04 20:54:13 +0000528
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000529 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Tim Northover9656f142016-08-04 20:54:13 +0000530 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
531 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
532
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000533 Register CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
Tim Northover0f140c72016-09-09 11:46:34 +0000534 MIRBuilder.buildConstant(CarryIn, 0);
Tim Northover9656f142016-08-04 20:54:13 +0000535
536 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000537 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
538 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Tim Northover9656f142016-08-04 20:54:13 +0000539
Tim Northover0f140c72016-09-09 11:46:34 +0000540 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
Tim Northover91c81732016-08-19 17:17:06 +0000541 Src2Regs[i], CarryIn);
Tim Northover9656f142016-08-04 20:54:13 +0000542
543 DstRegs.push_back(DstReg);
544 CarryIn = CarryOut;
545 }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000546 Register DstReg = MI.getOperand(0).getReg();
Amara Emerson5ec146042018-12-10 18:44:58 +0000547 if(MRI.getType(DstReg).isVector())
548 MIRBuilder.buildBuildVector(DstReg, DstRegs);
549 else
550 MIRBuilder.buildMerge(DstReg, DstRegs);
Tim Northover9656f142016-08-04 20:54:13 +0000551 MI.eraseFromParent();
552 return Legalized;
553 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000554 case TargetOpcode::G_SUB: {
555 // FIXME: add support for when SizeOp0 isn't an exact multiple of
556 // NarrowSize.
557 if (SizeOp0 % NarrowSize != 0)
558 return UnableToLegalize;
559
560 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
561
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000562 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000563 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
564 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
565
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000566 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
567 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000568 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
569 {Src1Regs[0], Src2Regs[0]});
570 DstRegs.push_back(DstReg);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000571 Register BorrowIn = BorrowOut;
Petar Avramovic7cecadb2019-01-28 12:10:17 +0000572 for (int i = 1; i < NumParts; ++i) {
573 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
574 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
575
576 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
577 {Src1Regs[i], Src2Regs[i], BorrowIn});
578
579 DstRegs.push_back(DstReg);
580 BorrowIn = BorrowOut;
581 }
582 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
583 MI.eraseFromParent();
584 return Legalized;
585 }
Matt Arsenault211e89d2019-01-27 00:52:51 +0000586 case TargetOpcode::G_MUL:
Petar Avramovic5229f472019-03-11 10:08:44 +0000587 case TargetOpcode::G_UMULH:
Petar Avramovic0b17e592019-03-11 10:00:17 +0000588 return narrowScalarMul(MI, NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +0000589 case TargetOpcode::G_EXTRACT:
590 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
591 case TargetOpcode::G_INSERT:
592 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000593 case TargetOpcode::G_LOAD: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000594 const auto &MMO = **MI.memoperands_begin();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000595 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000596 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000597 if (DstTy.isVector())
Matt Arsenault045bc9a2019-01-30 02:35:38 +0000598 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000599
600 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000601 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000602 auto &MMO = **MI.memoperands_begin();
603 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
604 MIRBuilder.buildAnyExt(DstReg, TmpReg);
605 MI.eraseFromParent();
606 return Legalized;
607 }
608
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000609 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Justin Bognerd09c3ce2017-01-19 01:05:48 +0000610 }
Matt Arsenault6614f852019-01-22 19:02:10 +0000611 case TargetOpcode::G_ZEXTLOAD:
612 case TargetOpcode::G_SEXTLOAD: {
613 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000614 Register DstReg = MI.getOperand(0).getReg();
615 Register PtrReg = MI.getOperand(1).getReg();
Matt Arsenault6614f852019-01-22 19:02:10 +0000616
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000617 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault6614f852019-01-22 19:02:10 +0000618 auto &MMO = **MI.memoperands_begin();
Amara Emersond51adf02019-04-17 22:21:05 +0000619 if (MMO.getSizeInBits() == NarrowSize) {
Matt Arsenault6614f852019-01-22 19:02:10 +0000620 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
621 } else {
622 unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
623 : TargetOpcode::G_SEXTLOAD;
624 MIRBuilder.buildInstr(ExtLoad)
625 .addDef(TmpReg)
626 .addUse(PtrReg)
627 .addMemOperand(&MMO);
628 }
629
630 if (ZExt)
631 MIRBuilder.buildZExt(DstReg, TmpReg);
632 else
633 MIRBuilder.buildSExt(DstReg, TmpReg);
634
635 MI.eraseFromParent();
636 return Legalized;
637 }
Justin Bognerfde01042017-01-18 17:29:54 +0000638 case TargetOpcode::G_STORE: {
Daniel Sanders27fe8a52018-04-27 19:48:53 +0000639 const auto &MMO = **MI.memoperands_begin();
Matt Arsenault18619af2019-01-29 18:13:02 +0000640
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000641 Register SrcReg = MI.getOperand(0).getReg();
Matt Arsenault18619af2019-01-29 18:13:02 +0000642 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000643 if (SrcTy.isVector())
644 return UnableToLegalize;
645
646 int NumParts = SizeOp0 / NarrowSize;
647 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
648 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
649 if (SrcTy.isVector() && LeftoverBits != 0)
650 return UnableToLegalize;
Matt Arsenault18619af2019-01-29 18:13:02 +0000651
652 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000653 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault18619af2019-01-29 18:13:02 +0000654 auto &MMO = **MI.memoperands_begin();
655 MIRBuilder.buildTrunc(TmpReg, SrcReg);
656 MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
657 MI.eraseFromParent();
658 return Legalized;
659 }
660
Matt Arsenault7f09fd62019-02-05 00:26:12 +0000661 return reduceLoadStoreWidth(MI, 0, NarrowTy);
Justin Bognerfde01042017-01-18 17:29:54 +0000662 }
Matt Arsenault81511e52019-02-05 00:13:44 +0000663 case TargetOpcode::G_SELECT:
664 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
Petar Avramovic150fd432018-12-18 11:36:14 +0000665 case TargetOpcode::G_AND:
666 case TargetOpcode::G_OR:
667 case TargetOpcode::G_XOR: {
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000668 // Legalize bitwise operation:
669 // A = BinOp<Ty> B, C
670 // into:
671 // B1, ..., BN = G_UNMERGE_VALUES B
672 // C1, ..., CN = G_UNMERGE_VALUES C
673 // A1 = BinOp<Ty/N> B1, C2
674 // ...
675 // AN = BinOp<Ty/N> BN, CN
676 // A = G_MERGE_VALUES A1, ..., AN
Matt Arsenault9e0eeba2019-04-10 17:07:56 +0000677 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
Quentin Colombetc2f3cea2017-10-03 04:53:56 +0000678 }
Matt Arsenault30989e42019-01-22 21:42:11 +0000679 case TargetOpcode::G_SHL:
680 case TargetOpcode::G_LSHR:
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +0000681 case TargetOpcode::G_ASHR:
682 return narrowScalarShift(MI, TypeIdx, NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000683 case TargetOpcode::G_CTLZ:
684 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
685 case TargetOpcode::G_CTTZ:
686 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
687 case TargetOpcode::G_CTPOP:
688 if (TypeIdx != 0)
689 return UnableToLegalize; // TODO
690
691 Observer.changingInstr(MI);
692 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
693 Observer.changedInstr(MI);
694 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +0000695 case TargetOpcode::G_INTTOPTR:
696 if (TypeIdx != 1)
697 return UnableToLegalize;
698
699 Observer.changingInstr(MI);
700 narrowScalarSrc(MI, NarrowTy, 1);
701 Observer.changedInstr(MI);
702 return Legalized;
703 case TargetOpcode::G_PTRTOINT:
704 if (TypeIdx != 0)
705 return UnableToLegalize;
706
707 Observer.changingInstr(MI);
708 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
709 Observer.changedInstr(MI);
710 return Legalized;
Petar Avramovicbe20e362019-07-09 14:36:17 +0000711 case TargetOpcode::G_PHI: {
712 unsigned NumParts = SizeOp0 / NarrowSize;
713 SmallVector<Register, 2> DstRegs;
714 SmallVector<SmallVector<Register, 2>, 2> SrcRegs;
715 DstRegs.resize(NumParts);
716 SrcRegs.resize(MI.getNumOperands() / 2);
717 Observer.changingInstr(MI);
718 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
719 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
720 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
721 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
722 SrcRegs[i / 2]);
723 }
724 MachineBasicBlock &MBB = *MI.getParent();
725 MIRBuilder.setInsertPt(MBB, MI);
726 for (unsigned i = 0; i < NumParts; ++i) {
727 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
728 MachineInstrBuilder MIB =
729 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
730 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
731 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
732 }
733 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
734 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
735 Observer.changedInstr(MI);
736 MI.eraseFromParent();
737 return Legalized;
738 }
Tim Northover9656f142016-08-04 20:54:13 +0000739 }
Tim Northover33b07d62016-07-22 20:03:43 +0000740}
741
Roman Tereshind5fa9fd2018-05-09 17:28:18 +0000742void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
743 unsigned OpIdx, unsigned ExtOpcode) {
744 MachineOperand &MO = MI.getOperand(OpIdx);
Aditya Nandakumarcef44a22018-12-11 00:48:50 +0000745 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
Roman Tereshind5fa9fd2018-05-09 17:28:18 +0000746 MO.setReg(ExtB->getOperand(0).getReg());
747}
748
Matt Arsenault30989e42019-01-22 21:42:11 +0000749void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
750 unsigned OpIdx) {
751 MachineOperand &MO = MI.getOperand(OpIdx);
752 auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
753 {MO.getReg()});
754 MO.setReg(ExtB->getOperand(0).getReg());
755}
756
Roman Tereshind5fa9fd2018-05-09 17:28:18 +0000757void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
758 unsigned OpIdx, unsigned TruncOpcode) {
759 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000760 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +0000761 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
Aditya Nandakumarcef44a22018-12-11 00:48:50 +0000762 MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
Roman Tereshind5fa9fd2018-05-09 17:28:18 +0000763 MO.setReg(DstExt);
764}
765
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000766void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
767 unsigned OpIdx, unsigned ExtOpcode) {
768 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000769 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultd5684f72019-01-31 02:09:57 +0000770 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
771 MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
772 MO.setReg(DstTrunc);
773}
774
Matt Arsenault18ec3822019-02-11 22:00:39 +0000775void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
776 unsigned OpIdx) {
777 MachineOperand &MO = MI.getOperand(OpIdx);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000778 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +0000779 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
780 MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
781 MO.setReg(DstExt);
782}
783
Matt Arsenault26b7e852019-02-19 16:30:19 +0000784void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
785 unsigned OpIdx) {
786 MachineOperand &MO = MI.getOperand(OpIdx);
787
788 LLT OldTy = MRI.getType(MO.getReg());
789 unsigned OldElts = OldTy.getNumElements();
790 unsigned NewElts = MoreTy.getNumElements();
791
792 unsigned NumParts = NewElts / OldElts;
793
794 // Use concat_vectors if the result is a multiple of the number of elements.
795 if (NumParts * OldElts == NewElts) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000796 SmallVector<Register, 8> Parts;
Matt Arsenault26b7e852019-02-19 16:30:19 +0000797 Parts.push_back(MO.getReg());
798
Matt Arsenault3018d182019-06-28 01:47:44 +0000799 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +0000800 for (unsigned I = 1; I != NumParts; ++I)
801 Parts.push_back(ImpDef);
802
803 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
804 MO.setReg(Concat.getReg(0));
805 return;
806 }
807
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +0000808 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
809 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
Matt Arsenault26b7e852019-02-19 16:30:19 +0000810 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
811 MO.setReg(MoreReg);
812}
813
Tim Northover69fa84a2016-10-14 22:18:18 +0000814LegalizerHelper::LegalizeResult
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000815LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
816 LLT WideTy) {
817 if (TypeIdx != 1)
818 return UnableToLegalize;
819
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000820 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000821 LLT DstTy = MRI.getType(DstReg);
Matt Arsenault43cbca52019-07-03 23:08:06 +0000822 if (DstTy.isVector())
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000823 return UnableToLegalize;
824
Matt Arsenaultc9f14f22019-07-01 19:36:10 +0000825 Register Src1 = MI.getOperand(1).getReg();
826 LLT SrcTy = MRI.getType(Src1);
827 int NumMerge = DstTy.getSizeInBits() / WideTy.getSizeInBits();
828
829 // Try to turn this into a merge of merges if we can use the requested type as
830 // the source.
831
832 // TODO: Pad with undef if DstTy > WideTy
833 if (NumMerge > 1 && WideTy.getSizeInBits() % SrcTy.getSizeInBits() == 0) {
834 int PartsPerMerge = WideTy.getSizeInBits() / SrcTy.getSizeInBits();
835 SmallVector<Register, 4> Parts;
836 SmallVector<Register, 4> SubMerges;
837
838 for (int I = 0; I != NumMerge; ++I) {
839 for (int J = 0; J != PartsPerMerge; ++J)
840 Parts.push_back(MI.getOperand(I * PartsPerMerge + J + 1).getReg());
841
842 auto SubMerge = MIRBuilder.buildMerge(WideTy, Parts);
843 SubMerges.push_back(SubMerge.getReg(0));
844 Parts.clear();
845 }
846
847 MIRBuilder.buildMerge(DstReg, SubMerges);
848 MI.eraseFromParent();
849 return Legalized;
850 }
851
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000852 unsigned NumOps = MI.getNumOperands();
853 unsigned NumSrc = MI.getNumOperands() - 1;
854 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
855
Matt Arsenaultc9f14f22019-07-01 19:36:10 +0000856 Register ResultReg = MIRBuilder.buildZExt(DstTy, Src1).getReg(0);
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000857
858 for (unsigned I = 2; I != NumOps; ++I) {
859 const unsigned Offset = (I - 1) * PartSize;
860
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000861 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000862 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
863
864 auto ZextInput = MIRBuilder.buildZExt(DstTy, SrcReg);
865
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000866 Register NextResult = I + 1 == NumOps ? DstReg :
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000867 MRI.createGenericVirtualRegister(DstTy);
868
869 auto ShiftAmt = MIRBuilder.buildConstant(DstTy, Offset);
870 auto Shl = MIRBuilder.buildShl(DstTy, ZextInput, ShiftAmt);
871 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
872 ResultReg = NextResult;
873 }
874
875 MI.eraseFromParent();
876 return Legalized;
877}
878
879LegalizerHelper::LegalizeResult
880LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
881 LLT WideTy) {
882 if (TypeIdx != 0)
883 return UnableToLegalize;
884
885 unsigned NumDst = MI.getNumOperands() - 1;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000886 Register SrcReg = MI.getOperand(NumDst).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000887 LLT SrcTy = MRI.getType(SrcReg);
888 if (!SrcTy.isScalar())
889 return UnableToLegalize;
890
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000891 Register Dst0Reg = MI.getOperand(0).getReg();
Matt Arsenault888aa5d2019-02-03 00:07:33 +0000892 LLT DstTy = MRI.getType(Dst0Reg);
893 if (!DstTy.isScalar())
894 return UnableToLegalize;
895
896 unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
897 LLT NewSrcTy = LLT::scalar(NewSrcSize);
898 unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
899
900 auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
901
902 for (unsigned I = 1; I != NumDst; ++I) {
903 auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
904 auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
905 WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
906 }
907
908 Observer.changingInstr(MI);
909
910 MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
911 for (unsigned I = 0; I != NumDst; ++I)
912 widenScalarDst(MI, WideTy, I);
913
914 Observer.changedInstr(MI);
915
916 return Legalized;
917}
918
919LegalizerHelper::LegalizeResult
Matt Arsenault1cf713662019-02-12 14:54:52 +0000920LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
921 LLT WideTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +0000922 Register DstReg = MI.getOperand(0).getReg();
923 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +0000924 LLT SrcTy = MRI.getType(SrcReg);
Matt Arsenaultfbe92a52019-02-18 22:39:27 +0000925
926 LLT DstTy = MRI.getType(DstReg);
927 unsigned Offset = MI.getOperand(2).getImm();
928
929 if (TypeIdx == 0) {
930 if (SrcTy.isVector() || DstTy.isVector())
931 return UnableToLegalize;
932
933 SrcOp Src(SrcReg);
934 if (SrcTy.isPointer()) {
935 // Extracts from pointers can be handled only if they are really just
936 // simple integers.
937 const DataLayout &DL = MIRBuilder.getDataLayout();
938 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
939 return UnableToLegalize;
940
941 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
942 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
943 SrcTy = SrcAsIntTy;
944 }
945
946 if (DstTy.isPointer())
947 return UnableToLegalize;
948
949 if (Offset == 0) {
950 // Avoid a shift in the degenerate case.
951 MIRBuilder.buildTrunc(DstReg,
952 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
953 MI.eraseFromParent();
954 return Legalized;
955 }
956
957 // Do a shift in the source type.
958 LLT ShiftTy = SrcTy;
959 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
960 Src = MIRBuilder.buildAnyExt(WideTy, Src);
961 ShiftTy = WideTy;
962 } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
963 return UnableToLegalize;
964
965 auto LShr = MIRBuilder.buildLShr(
966 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
967 MIRBuilder.buildTrunc(DstReg, LShr);
968 MI.eraseFromParent();
969 return Legalized;
970 }
971
Matt Arsenault8f624ab2019-04-22 15:10:42 +0000972 if (SrcTy.isScalar()) {
973 Observer.changingInstr(MI);
974 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
975 Observer.changedInstr(MI);
976 return Legalized;
977 }
978
Matt Arsenault1cf713662019-02-12 14:54:52 +0000979 if (!SrcTy.isVector())
980 return UnableToLegalize;
981
Matt Arsenault1cf713662019-02-12 14:54:52 +0000982 if (DstTy != SrcTy.getElementType())
983 return UnableToLegalize;
984
Matt Arsenault1cf713662019-02-12 14:54:52 +0000985 if (Offset % SrcTy.getScalarSizeInBits() != 0)
986 return UnableToLegalize;
987
988 Observer.changingInstr(MI);
989 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
990
991 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
992 Offset);
993 widenScalarDst(MI, WideTy.getScalarType(), 0);
994 Observer.changedInstr(MI);
995 return Legalized;
996}
997
998LegalizerHelper::LegalizeResult
999LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1000 LLT WideTy) {
1001 if (TypeIdx != 0)
1002 return UnableToLegalize;
1003 Observer.changingInstr(MI);
1004 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1005 widenScalarDst(MI, WideTy);
1006 Observer.changedInstr(MI);
1007 return Legalized;
1008}
1009
1010LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00001011LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
Tim Northover3c73e362016-08-23 18:20:09 +00001012 MIRBuilder.setInstr(MI);
1013
Tim Northover32335812016-08-04 18:35:11 +00001014 switch (MI.getOpcode()) {
1015 default:
1016 return UnableToLegalize;
Matt Arsenault1cf713662019-02-12 14:54:52 +00001017 case TargetOpcode::G_EXTRACT:
1018 return widenScalarExtract(MI, TypeIdx, WideTy);
1019 case TargetOpcode::G_INSERT:
1020 return widenScalarInsert(MI, TypeIdx, WideTy);
Matt Arsenault888aa5d2019-02-03 00:07:33 +00001021 case TargetOpcode::G_MERGE_VALUES:
1022 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1023 case TargetOpcode::G_UNMERGE_VALUES:
1024 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001025 case TargetOpcode::G_UADDO:
1026 case TargetOpcode::G_USUBO: {
1027 if (TypeIdx == 1)
1028 return UnableToLegalize; // TODO
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001029 auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1030 {MI.getOperand(2).getReg()});
1031 auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
1032 {MI.getOperand(3).getReg()});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001033 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1034 ? TargetOpcode::G_ADD
1035 : TargetOpcode::G_SUB;
1036 // Do the arithmetic in the larger type.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001037 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001038 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1039 APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
1040 auto AndOp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001041 TargetOpcode::G_AND, {WideTy},
1042 {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
Aditya Nandakumar6d47a412018-08-29 03:17:08 +00001043 // There is no overflow if the AndOp is the same as NewOp.
1044 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
1045 AndOp);
1046 // Now trunc the NewOp to the original result.
1047 MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1048 MI.eraseFromParent();
1049 return Legalized;
1050 }
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001051 case TargetOpcode::G_CTTZ:
1052 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1053 case TargetOpcode::G_CTLZ:
1054 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1055 case TargetOpcode::G_CTPOP: {
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001056 if (TypeIdx == 0) {
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001057 Observer.changingInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001058 widenScalarDst(MI, WideTy, 0);
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001059 Observer.changedInstr(MI);
Matt Arsenaultd5684f72019-01-31 02:09:57 +00001060 return Legalized;
1061 }
1062
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001063 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001064
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001065 // First ZEXT the input.
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001066 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1067 LLT CurTy = MRI.getType(SrcReg);
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001068 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1069 // The count is the same in the larger type except if the original
1070 // value was zero. This can be handled by setting the bit just off
1071 // the top of the original type.
1072 auto TopBit =
1073 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001074 MIBSrc = MIRBuilder.buildOr(
1075 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001076 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001077
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001078 // Perform the operation at the larger size.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001079 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001080 // This is already the correct result for CTPOP and CTTZs
1081 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1082 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1083 // The correct result is NewOp - (Difference in widety and current ty).
1084 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00001085 MIBNewOp = MIRBuilder.buildInstr(
1086 TargetOpcode::G_SUB, {WideTy},
1087 {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001088 }
Matt Arsenault3d6a49b2019-02-04 22:26:33 +00001089
1090 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1091 MI.eraseFromParent();
Aditya Nandakumarc1061832018-08-22 17:59:18 +00001092 return Legalized;
1093 }
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001094 case TargetOpcode::G_BSWAP: {
1095 Observer.changingInstr(MI);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001096 Register DstReg = MI.getOperand(0).getReg();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001097
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001098 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1099 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1100 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00001101 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1102
1103 MI.getOperand(0).setReg(DstExt);
1104
1105 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1106
1107 LLT Ty = MRI.getType(DstReg);
1108 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1109 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1110 MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1111 .addDef(ShrReg)
1112 .addUse(DstExt)
1113 .addUse(ShiftAmtReg);
1114
1115 MIRBuilder.buildTrunc(DstReg, ShrReg);
1116 Observer.changedInstr(MI);
1117 return Legalized;
1118 }
Tim Northover61c16142016-08-04 21:39:49 +00001119 case TargetOpcode::G_ADD:
1120 case TargetOpcode::G_AND:
1121 case TargetOpcode::G_MUL:
1122 case TargetOpcode::G_OR:
1123 case TargetOpcode::G_XOR:
Justin Bognerddb80ae2017-01-19 07:51:17 +00001124 case TargetOpcode::G_SUB:
Matt Arsenault1cf713662019-02-12 14:54:52 +00001125 // Perform operation at larger width (any extension is fines here, high bits
Tim Northover32335812016-08-04 18:35:11 +00001126 // don't affect the result) and then truncate the result back to the
1127 // original type.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001128 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001129 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1130 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1131 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001132 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001133 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001134
Roman Tereshin6d266382018-05-09 21:43:30 +00001135 case TargetOpcode::G_SHL:
Matt Arsenault012ecbb2019-05-16 04:08:46 +00001136 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001137
1138 if (TypeIdx == 0) {
1139 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1140 widenScalarDst(MI, WideTy);
1141 } else {
1142 assert(TypeIdx == 1);
1143 // The "number of bits to shift" operand must preserve its value as an
1144 // unsigned integer:
1145 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1146 }
1147
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001148 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001149 return Legalized;
1150
Tim Northover7a753d92016-08-26 17:46:06 +00001151 case TargetOpcode::G_SDIV:
Roman Tereshin27bba442018-05-09 01:43:12 +00001152 case TargetOpcode::G_SREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001153 case TargetOpcode::G_SMIN:
1154 case TargetOpcode::G_SMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001155 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001156 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1157 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1158 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001159 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001160 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001161
Roman Tereshin6d266382018-05-09 21:43:30 +00001162 case TargetOpcode::G_ASHR:
Matt Arsenault30989e42019-01-22 21:42:11 +00001163 case TargetOpcode::G_LSHR:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001164 Observer.changingInstr(MI);
Matt Arsenault30989e42019-01-22 21:42:11 +00001165
1166 if (TypeIdx == 0) {
1167 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1168 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1169
1170 widenScalarSrc(MI, WideTy, 1, CvtOp);
1171 widenScalarDst(MI, WideTy);
1172 } else {
1173 assert(TypeIdx == 1);
1174 // The "number of bits to shift" operand must preserve its value as an
1175 // unsigned integer:
1176 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1177 }
1178
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001179 Observer.changedInstr(MI);
Roman Tereshin6d266382018-05-09 21:43:30 +00001180 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001181 case TargetOpcode::G_UDIV:
1182 case TargetOpcode::G_UREM:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00001183 case TargetOpcode::G_UMIN:
1184 case TargetOpcode::G_UMAX:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001185 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001186 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1187 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1188 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001189 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001190 return Legalized;
1191
1192 case TargetOpcode::G_SELECT:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001193 Observer.changingInstr(MI);
Petar Avramovic09dff332018-12-25 14:42:30 +00001194 if (TypeIdx == 0) {
1195 // Perform operation at larger width (any extension is fine here, high
1196 // bits don't affect the result) and then truncate the result back to the
1197 // original type.
1198 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1199 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1200 widenScalarDst(MI, WideTy);
1201 } else {
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001202 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
Petar Avramovic09dff332018-12-25 14:42:30 +00001203 // Explicit extension is required here since high bits affect the result.
Matt Arsenault6d8e1b42019-01-30 02:57:43 +00001204 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
Petar Avramovic09dff332018-12-25 14:42:30 +00001205 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001206 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001207 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001208
Ahmed Bougachab6137062017-01-23 21:10:14 +00001209 case TargetOpcode::G_FPTOSI:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001210 case TargetOpcode::G_FPTOUI:
Ahmed Bougachab6137062017-01-23 21:10:14 +00001211 if (TypeIdx != 0)
1212 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001213 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001214 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001215 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001216 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001217
Ahmed Bougachad2948232017-01-20 01:37:24 +00001218 case TargetOpcode::G_SITOFP:
Ahmed Bougachad2948232017-01-20 01:37:24 +00001219 if (TypeIdx != 1)
1220 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001221 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001222 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001223 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001224 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001225
1226 case TargetOpcode::G_UITOFP:
1227 if (TypeIdx != 1)
1228 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001229 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001230 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001231 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001232 return Legalized;
1233
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001234 case TargetOpcode::G_LOAD:
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001235 case TargetOpcode::G_SEXTLOAD:
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001236 case TargetOpcode::G_ZEXTLOAD:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001237 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001238 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001239 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001240 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001241
Tim Northover3c73e362016-08-23 18:20:09 +00001242 case TargetOpcode::G_STORE: {
Matt Arsenault92c50012019-01-30 02:04:31 +00001243 if (TypeIdx != 0)
1244 return UnableToLegalize;
1245
1246 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1247 if (!isPowerOf2_32(Ty.getSizeInBits()))
Tim Northover548feee2017-03-21 22:22:05 +00001248 return UnableToLegalize;
1249
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001250 Observer.changingInstr(MI);
Matt Arsenault92c50012019-01-30 02:04:31 +00001251
1252 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1253 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1254 widenScalarSrc(MI, WideTy, 0, ExtType);
1255
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001256 Observer.changedInstr(MI);
Tim Northover3c73e362016-08-23 18:20:09 +00001257 return Legalized;
1258 }
Tim Northoverea904f92016-08-19 22:40:00 +00001259 case TargetOpcode::G_CONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001260 MachineOperand &SrcMO = MI.getOperand(1);
1261 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
1262 const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001263 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001264 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1265
1266 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001267 Observer.changedInstr(MI);
Tim Northoverea904f92016-08-19 22:40:00 +00001268 return Legalized;
1269 }
Tim Northovera11be042016-08-19 22:40:08 +00001270 case TargetOpcode::G_FCONSTANT: {
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001271 MachineOperand &SrcMO = MI.getOperand(1);
Amara Emerson77a5c962018-01-27 07:07:20 +00001272 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001273 APFloat Val = SrcMO.getFPImm()->getValueAPF();
Amara Emerson77a5c962018-01-27 07:07:20 +00001274 bool LosesInfo;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001275 switch (WideTy.getSizeInBits()) {
1276 case 32:
Matt Arsenault996c6662019-02-12 14:54:54 +00001277 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
1278 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001279 break;
1280 case 64:
Matt Arsenault996c6662019-02-12 14:54:54 +00001281 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
1282 &LosesInfo);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001283 break;
1284 default:
Matt Arsenault996c6662019-02-12 14:54:54 +00001285 return UnableToLegalize;
Tim Northover6cd4b232016-08-23 21:01:26 +00001286 }
Matt Arsenault996c6662019-02-12 14:54:54 +00001287
1288 assert(!LosesInfo && "extend should always be lossless");
1289
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001290 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001291 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1292
1293 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001294 Observer.changedInstr(MI);
Roman Tereshin25cbfe62018-05-08 22:53:09 +00001295 return Legalized;
Roman Tereshin27bba442018-05-09 01:43:12 +00001296 }
Matt Arsenaultbefee402019-01-09 07:34:14 +00001297 case TargetOpcode::G_IMPLICIT_DEF: {
1298 Observer.changingInstr(MI);
1299 widenScalarDst(MI, WideTy);
1300 Observer.changedInstr(MI);
1301 return Legalized;
1302 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001303 case TargetOpcode::G_BRCOND:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001304 Observer.changingInstr(MI);
Petar Avramovic5d9b8ee2019-02-14 11:39:53 +00001305 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001306 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001307 return Legalized;
1308
1309 case TargetOpcode::G_FCMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001310 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001311 if (TypeIdx == 0)
1312 widenScalarDst(MI, WideTy);
1313 else {
1314 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1315 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
Roman Tereshin27bba442018-05-09 01:43:12 +00001316 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001317 Observer.changedInstr(MI);
Roman Tereshin27bba442018-05-09 01:43:12 +00001318 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001319
1320 case TargetOpcode::G_ICMP:
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001321 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001322 if (TypeIdx == 0)
1323 widenScalarDst(MI, WideTy);
1324 else {
1325 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1326 MI.getOperand(1).getPredicate()))
1327 ? TargetOpcode::G_SEXT
1328 : TargetOpcode::G_ZEXT;
1329 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1330 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1331 }
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001332 Observer.changedInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001333 return Legalized;
1334
1335 case TargetOpcode::G_GEP:
Tim Northover22d82cf2016-09-15 11:02:19 +00001336 assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001337 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001338 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001339 Observer.changedInstr(MI);
Tim Northover22d82cf2016-09-15 11:02:19 +00001340 return Legalized;
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001341
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001342 case TargetOpcode::G_PHI: {
1343 assert(TypeIdx == 0 && "Expecting only Idx 0");
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001344
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001345 Observer.changingInstr(MI);
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001346 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1347 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1348 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1349 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001350 }
Roman Tereshind5fa9fd2018-05-09 17:28:18 +00001351
1352 MachineBasicBlock &MBB = *MI.getParent();
1353 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1354 widenScalarDst(MI, WideTy);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001355 Observer.changedInstr(MI);
Aditya Nandakumar892979e2017-08-25 04:57:27 +00001356 return Legalized;
1357 }
Matt Arsenault63786292019-01-22 20:38:15 +00001358 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1359 if (TypeIdx == 0) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001360 Register VecReg = MI.getOperand(1).getReg();
Matt Arsenault63786292019-01-22 20:38:15 +00001361 LLT VecTy = MRI.getType(VecReg);
1362 Observer.changingInstr(MI);
1363
1364 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1365 WideTy.getSizeInBits()),
1366 1, TargetOpcode::G_SEXT);
1367
1368 widenScalarDst(MI, WideTy, 0);
1369 Observer.changedInstr(MI);
1370 return Legalized;
1371 }
1372
Amara Emersoncbd86d82018-10-25 14:04:54 +00001373 if (TypeIdx != 2)
1374 return UnableToLegalize;
Daniel Sandersd001e0e2018-12-12 23:48:13 +00001375 Observer.changingInstr(MI);
Amara Emersoncbd86d82018-10-25 14:04:54 +00001376 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00001377 Observer.changedInstr(MI);
Amara Emersoncbd86d82018-10-25 14:04:54 +00001378 return Legalized;
Matt Arsenault63786292019-01-22 20:38:15 +00001379 }
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001380 case TargetOpcode::G_FADD:
1381 case TargetOpcode::G_FMUL:
1382 case TargetOpcode::G_FSUB:
1383 case TargetOpcode::G_FMA:
1384 case TargetOpcode::G_FNEG:
1385 case TargetOpcode::G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00001386 case TargetOpcode::G_FCANONICALIZE:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00001387 case TargetOpcode::G_FMINNUM:
1388 case TargetOpcode::G_FMAXNUM:
1389 case TargetOpcode::G_FMINNUM_IEEE:
1390 case TargetOpcode::G_FMAXNUM_IEEE:
1391 case TargetOpcode::G_FMINIMUM:
1392 case TargetOpcode::G_FMAXIMUM:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001393 case TargetOpcode::G_FDIV:
1394 case TargetOpcode::G_FREM:
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001395 case TargetOpcode::G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00001396 case TargetOpcode::G_FFLOOR:
Jessica Paquette7db82d72019-01-28 18:34:18 +00001397 case TargetOpcode::G_FCOS:
1398 case TargetOpcode::G_FSIN:
Jessica Paquettec49428a2019-01-28 19:53:14 +00001399 case TargetOpcode::G_FLOG10:
Jessica Paquette2d73ecd2019-01-28 21:27:23 +00001400 case TargetOpcode::G_FLOG:
Jessica Paquette0154bd12019-01-30 21:16:04 +00001401 case TargetOpcode::G_FLOG2:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00001402 case TargetOpcode::G_FRINT:
Jessica Paquetteba557672019-04-25 16:44:40 +00001403 case TargetOpcode::G_FNEARBYINT:
Jessica Paquette22457f82019-01-30 21:03:52 +00001404 case TargetOpcode::G_FSQRT:
Jessica Paquette84bedac2019-01-30 23:46:15 +00001405 case TargetOpcode::G_FEXP:
Jessica Paquettee7941212019-04-03 16:58:32 +00001406 case TargetOpcode::G_FEXP2:
Jessica Paquettedfd87f62019-04-19 16:28:08 +00001407 case TargetOpcode::G_FPOW:
Jessica Paquette56342642019-04-23 18:20:44 +00001408 case TargetOpcode::G_INTRINSIC_TRUNC:
Jessica Paquette3cc6d1f2019-04-23 21:11:57 +00001409 case TargetOpcode::G_INTRINSIC_ROUND:
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001410 assert(TypeIdx == 0);
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001411 Observer.changingInstr(MI);
Matt Arsenault745fd9f2019-01-20 19:10:31 +00001412
1413 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1414 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1415
Jessica Paquette453ab1d2018-12-21 17:05:26 +00001416 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1417 Observer.changedInstr(MI);
1418 return Legalized;
Matt Arsenaultcbaada62019-02-02 23:29:55 +00001419 case TargetOpcode::G_INTTOPTR:
1420 if (TypeIdx != 1)
1421 return UnableToLegalize;
1422
1423 Observer.changingInstr(MI);
1424 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1425 Observer.changedInstr(MI);
1426 return Legalized;
1427 case TargetOpcode::G_PTRTOINT:
1428 if (TypeIdx != 0)
1429 return UnableToLegalize;
1430
1431 Observer.changingInstr(MI);
1432 widenScalarDst(MI, WideTy, 0);
1433 Observer.changedInstr(MI);
1434 return Legalized;
Matt Arsenaultbd791b52019-07-08 13:48:06 +00001435 case TargetOpcode::G_BUILD_VECTOR: {
1436 Observer.changingInstr(MI);
1437
1438 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
1439 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
1440 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
1441
1442 // Avoid changing the result vector type if the source element type was
1443 // requested.
1444 if (TypeIdx == 1) {
1445 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
1446 MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
1447 } else {
1448 widenScalarDst(MI, WideTy, 0);
1449 }
1450
1451 Observer.changedInstr(MI);
1452 return Legalized;
1453 }
Tim Northover32335812016-08-04 18:35:11 +00001454 }
Tim Northover33b07d62016-07-22 20:03:43 +00001455}
1456
Tim Northover69fa84a2016-10-14 22:18:18 +00001457LegalizerHelper::LegalizeResult
1458LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Tim Northovercecee562016-08-26 17:46:13 +00001459 using namespace TargetOpcode;
Tim Northovercecee562016-08-26 17:46:13 +00001460 MIRBuilder.setInstr(MI);
1461
1462 switch(MI.getOpcode()) {
1463 default:
1464 return UnableToLegalize;
1465 case TargetOpcode::G_SREM:
1466 case TargetOpcode::G_UREM: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001467 Register QuotReg = MRI.createGenericVirtualRegister(Ty);
Tim Northover0f140c72016-09-09 11:46:34 +00001468 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
Tim Northovercecee562016-08-26 17:46:13 +00001469 .addDef(QuotReg)
1470 .addUse(MI.getOperand(1).getReg())
1471 .addUse(MI.getOperand(2).getReg());
1472
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001473 Register ProdReg = MRI.createGenericVirtualRegister(Ty);
Tim Northover0f140c72016-09-09 11:46:34 +00001474 MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1475 MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
1476 ProdReg);
Tim Northovercecee562016-08-26 17:46:13 +00001477 MI.eraseFromParent();
1478 return Legalized;
1479 }
Tim Northover0a9b2792017-02-08 21:22:15 +00001480 case TargetOpcode::G_SMULO:
1481 case TargetOpcode::G_UMULO: {
1482 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1483 // result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001484 Register Res = MI.getOperand(0).getReg();
1485 Register Overflow = MI.getOperand(1).getReg();
1486 Register LHS = MI.getOperand(2).getReg();
1487 Register RHS = MI.getOperand(3).getReg();
Tim Northover0a9b2792017-02-08 21:22:15 +00001488
1489 MIRBuilder.buildMul(Res, LHS, RHS);
1490
1491 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1492 ? TargetOpcode::G_SMULH
1493 : TargetOpcode::G_UMULH;
1494
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001495 Register HiPart = MRI.createGenericVirtualRegister(Ty);
Tim Northover0a9b2792017-02-08 21:22:15 +00001496 MIRBuilder.buildInstr(Opcode)
1497 .addDef(HiPart)
1498 .addUse(LHS)
1499 .addUse(RHS);
1500
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001501 Register Zero = MRI.createGenericVirtualRegister(Ty);
Tim Northover0a9b2792017-02-08 21:22:15 +00001502 MIRBuilder.buildConstant(Zero, 0);
Amara Emerson9de62132018-01-03 04:56:56 +00001503
1504 // For *signed* multiply, overflow is detected by checking:
1505 // (hi != (lo >> bitwidth-1))
1506 if (Opcode == TargetOpcode::G_SMULH) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001507 Register Shifted = MRI.createGenericVirtualRegister(Ty);
1508 Register ShiftAmt = MRI.createGenericVirtualRegister(Ty);
Amara Emerson9de62132018-01-03 04:56:56 +00001509 MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1510 MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1511 .addDef(Shifted)
1512 .addUse(Res)
1513 .addUse(ShiftAmt);
1514 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1515 } else {
1516 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1517 }
Tim Northover0a9b2792017-02-08 21:22:15 +00001518 MI.eraseFromParent();
1519 return Legalized;
1520 }
Volkan Keles5698b2a2017-03-08 18:09:14 +00001521 case TargetOpcode::G_FNEG: {
1522 // TODO: Handle vector types once we are able to
1523 // represent them.
1524 if (Ty.isVector())
1525 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001526 Register Res = MI.getOperand(0).getReg();
Volkan Keles5698b2a2017-03-08 18:09:14 +00001527 Type *ZeroTy;
Matthias Braunf1caa282017-12-15 22:22:58 +00001528 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
Volkan Keles5698b2a2017-03-08 18:09:14 +00001529 switch (Ty.getSizeInBits()) {
1530 case 16:
1531 ZeroTy = Type::getHalfTy(Ctx);
1532 break;
1533 case 32:
1534 ZeroTy = Type::getFloatTy(Ctx);
1535 break;
1536 case 64:
1537 ZeroTy = Type::getDoubleTy(Ctx);
1538 break;
Amara Emersonb6ddbef2017-12-19 17:21:35 +00001539 case 128:
1540 ZeroTy = Type::getFP128Ty(Ctx);
1541 break;
Volkan Keles5698b2a2017-03-08 18:09:14 +00001542 default:
1543 llvm_unreachable("unexpected floating-point type");
1544 }
1545 ConstantFP &ZeroForNegation =
1546 *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
Volkan Keles02bb1742018-02-14 19:58:36 +00001547 auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001548 Register SubByReg = MI.getOperand(1).getReg();
1549 Register ZeroReg = Zero->getOperand(0).getReg();
Michael Bergd573aa02019-04-18 18:48:57 +00001550 MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg},
Matt Arsenault5a321b82019-06-17 23:48:43 +00001551 MI.getFlags());
Volkan Keles5698b2a2017-03-08 18:09:14 +00001552 MI.eraseFromParent();
1553 return Legalized;
1554 }
Volkan Keles225921a2017-03-10 21:25:09 +00001555 case TargetOpcode::G_FSUB: {
1556 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1557 // First, check if G_FNEG is marked as Lower. If so, we may
1558 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
Daniel Sanders9ade5592018-01-29 17:37:29 +00001559 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
Volkan Keles225921a2017-03-10 21:25:09 +00001560 return UnableToLegalize;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001561 Register Res = MI.getOperand(0).getReg();
1562 Register LHS = MI.getOperand(1).getReg();
1563 Register RHS = MI.getOperand(2).getReg();
1564 Register Neg = MRI.createGenericVirtualRegister(Ty);
Volkan Keles225921a2017-03-10 21:25:09 +00001565 MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
Michael Bergd573aa02019-04-18 18:48:57 +00001566 MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags());
Volkan Keles225921a2017-03-10 21:25:09 +00001567 MI.eraseFromParent();
1568 return Legalized;
1569 }
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00001570 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001571 Register OldValRes = MI.getOperand(0).getReg();
1572 Register SuccessRes = MI.getOperand(1).getReg();
1573 Register Addr = MI.getOperand(2).getReg();
1574 Register CmpVal = MI.getOperand(3).getReg();
1575 Register NewVal = MI.getOperand(4).getReg();
Daniel Sandersaef1dfc2017-11-30 20:11:42 +00001576 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1577 **MI.memoperands_begin());
1578 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1579 MI.eraseFromParent();
1580 return Legalized;
1581 }
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001582 case TargetOpcode::G_LOAD:
1583 case TargetOpcode::G_SEXTLOAD:
1584 case TargetOpcode::G_ZEXTLOAD: {
1585 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001586 Register DstReg = MI.getOperand(0).getReg();
1587 Register PtrReg = MI.getOperand(1).getReg();
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001588 LLT DstTy = MRI.getType(DstReg);
1589 auto &MMO = **MI.memoperands_begin();
1590
Amara Emerson42866522019-04-20 21:25:00 +00001591 if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
1592 // In the case of G_LOAD, this was a non-extending load already and we're
1593 // about to lower to the same instruction.
1594 if (MI.getOpcode() == TargetOpcode::G_LOAD)
Daniel Sanders2de9d4a2018-04-30 17:20:01 +00001595 return UnableToLegalize;
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001596 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
1597 MI.eraseFromParent();
1598 return Legalized;
1599 }
1600
1601 if (DstTy.isScalar()) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001602 Register TmpReg =
Amara Emersond51adf02019-04-17 22:21:05 +00001603 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
Daniel Sanders5eb9f582018-04-28 18:14:50 +00001604 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1605 switch (MI.getOpcode()) {
1606 default:
1607 llvm_unreachable("Unexpected opcode");
1608 case TargetOpcode::G_LOAD:
1609 MIRBuilder.buildAnyExt(DstReg, TmpReg);
1610 break;
1611 case TargetOpcode::G_SEXTLOAD:
1612 MIRBuilder.buildSExt(DstReg, TmpReg);
1613 break;
1614 case TargetOpcode::G_ZEXTLOAD:
1615 MIRBuilder.buildZExt(DstReg, TmpReg);
1616 break;
1617 }
1618 MI.eraseFromParent();
1619 return Legalized;
1620 }
1621
1622 return UnableToLegalize;
1623 }
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00001624 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1625 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1626 case TargetOpcode::G_CTLZ:
1627 case TargetOpcode::G_CTTZ:
1628 case TargetOpcode::G_CTPOP:
1629 return lowerBitCount(MI, TypeIdx, Ty);
Petar Avramovicbd395692019-02-26 17:22:42 +00001630 case G_UADDO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001631 Register Res = MI.getOperand(0).getReg();
1632 Register CarryOut = MI.getOperand(1).getReg();
1633 Register LHS = MI.getOperand(2).getReg();
1634 Register RHS = MI.getOperand(3).getReg();
Petar Avramovicbd395692019-02-26 17:22:42 +00001635
1636 MIRBuilder.buildAdd(Res, LHS, RHS);
1637 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
1638
1639 MI.eraseFromParent();
1640 return Legalized;
1641 }
Petar Avramovicb8276f22018-12-17 12:31:07 +00001642 case G_UADDE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001643 Register Res = MI.getOperand(0).getReg();
1644 Register CarryOut = MI.getOperand(1).getReg();
1645 Register LHS = MI.getOperand(2).getReg();
1646 Register RHS = MI.getOperand(3).getReg();
1647 Register CarryIn = MI.getOperand(4).getReg();
Petar Avramovicb8276f22018-12-17 12:31:07 +00001648
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001649 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
1650 Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
Petar Avramovicb8276f22018-12-17 12:31:07 +00001651
1652 MIRBuilder.buildAdd(TmpRes, LHS, RHS);
1653 MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
1654 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
1655 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
1656
1657 MI.eraseFromParent();
1658 return Legalized;
1659 }
Petar Avramovic7cecadb2019-01-28 12:10:17 +00001660 case G_USUBO: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001661 Register Res = MI.getOperand(0).getReg();
1662 Register BorrowOut = MI.getOperand(1).getReg();
1663 Register LHS = MI.getOperand(2).getReg();
1664 Register RHS = MI.getOperand(3).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00001665
1666 MIRBuilder.buildSub(Res, LHS, RHS);
1667 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
1668
1669 MI.eraseFromParent();
1670 return Legalized;
1671 }
1672 case G_USUBE: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001673 Register Res = MI.getOperand(0).getReg();
1674 Register BorrowOut = MI.getOperand(1).getReg();
1675 Register LHS = MI.getOperand(2).getReg();
1676 Register RHS = MI.getOperand(3).getReg();
1677 Register BorrowIn = MI.getOperand(4).getReg();
Petar Avramovic7cecadb2019-01-28 12:10:17 +00001678
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001679 Register TmpRes = MRI.createGenericVirtualRegister(Ty);
1680 Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
1681 Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1682 Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
Petar Avramovic7cecadb2019-01-28 12:10:17 +00001683
1684 MIRBuilder.buildSub(TmpRes, LHS, RHS);
1685 MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
1686 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
1687 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
1688 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
1689 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
1690
1691 MI.eraseFromParent();
1692 return Legalized;
1693 }
Matt Arsenault02b5ca82019-05-17 23:05:13 +00001694 case G_UITOFP:
1695 return lowerUITOFP(MI, TypeIdx, Ty);
1696 case G_SITOFP:
1697 return lowerSITOFP(MI, TypeIdx, Ty);
Matt Arsenault6f74f552019-07-01 17:18:03 +00001698 case G_SMIN:
1699 case G_SMAX:
1700 case G_UMIN:
1701 case G_UMAX:
1702 return lowerMinMax(MI, TypeIdx, Ty);
Matt Arsenaultb1843e12019-07-09 23:34:29 +00001703 case G_FCOPYSIGN:
1704 return lowerFCopySign(MI, TypeIdx, Ty);
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00001705 case G_FMINNUM:
1706 case G_FMAXNUM:
1707 return lowerFMinNumMaxNum(MI);
Tim Northovercecee562016-08-26 17:46:13 +00001708 }
1709}
1710
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001711LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
1712 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001713 SmallVector<Register, 2> DstRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001714
1715 unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001716 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001717 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1718 int NumParts = Size / NarrowSize;
1719 // FIXME: Don't know how to handle the situation where the small vectors
1720 // aren't all the same size yet.
1721 if (Size % NarrowSize != 0)
1722 return UnableToLegalize;
1723
1724 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001725 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001726 MIRBuilder.buildUndef(TmpReg);
1727 DstRegs.push_back(TmpReg);
1728 }
1729
1730 if (NarrowTy.isVector())
1731 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1732 else
1733 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1734
1735 MI.eraseFromParent();
1736 return Legalized;
1737}
1738
1739LegalizerHelper::LegalizeResult
1740LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
1741 LLT NarrowTy) {
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001742 const unsigned Opc = MI.getOpcode();
1743 const unsigned NumOps = MI.getNumOperands() - 1;
1744 const unsigned NarrowSize = NarrowTy.getSizeInBits();
Matt Arsenault3018d182019-06-28 01:47:44 +00001745 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001746 const unsigned Flags = MI.getFlags();
1747 const LLT DstTy = MRI.getType(DstReg);
1748 const unsigned Size = DstTy.getSizeInBits();
1749 const int NumParts = Size / NarrowSize;
1750 const LLT EltTy = DstTy.getElementType();
1751 const unsigned EltSize = EltTy.getSizeInBits();
1752 const unsigned BitsForNumParts = NarrowSize * NumParts;
1753
1754 // Check if we have any leftovers. If we do, then only handle the case where
1755 // the leftover is one element.
1756 if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001757 return UnableToLegalize;
1758
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001759 if (BitsForNumParts != Size) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001760 Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001761 MIRBuilder.buildUndef(AccumDstReg);
1762
1763 // Handle the pieces which evenly divide into the requested type with
1764 // extract/op/insert sequence.
1765 for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
1766 SmallVector<SrcOp, 4> SrcOps;
1767 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001768 Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001769 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
1770 SrcOps.push_back(PartOpReg);
1771 }
1772
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001773 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001774 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1775
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001776 Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001777 MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
1778 AccumDstReg = PartInsertReg;
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001779 }
1780
1781 // Handle the remaining element sized leftover piece.
1782 SmallVector<SrcOp, 4> SrcOps;
1783 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001784 Register PartOpReg = MRI.createGenericVirtualRegister(EltTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001785 MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
1786 BitsForNumParts);
1787 SrcOps.push_back(PartOpReg);
1788 }
1789
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001790 Register PartDstReg = MRI.createGenericVirtualRegister(EltTy);
Matt Arsenaultccefbbd2019-01-30 02:22:13 +00001791 MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1792 MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
1793 MI.eraseFromParent();
1794
1795 return Legalized;
1796 }
1797
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001798 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001799
1800 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
1801
1802 if (NumOps >= 2)
1803 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
1804
1805 if (NumOps >= 3)
1806 extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
1807
1808 for (int i = 0; i < NumParts; ++i) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001809 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001810
1811 if (NumOps == 1)
1812 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
1813 else if (NumOps == 2) {
1814 MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
1815 } else if (NumOps == 3) {
1816 MIRBuilder.buildInstr(Opc, {DstReg},
1817 {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
1818 }
1819
1820 DstRegs.push_back(DstReg);
1821 }
1822
1823 if (NarrowTy.isVector())
1824 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1825 else
1826 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1827
1828 MI.eraseFromParent();
1829 return Legalized;
1830}
1831
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001832// Handle splitting vector operations which need to have the same number of
1833// elements in each type index, but each type index may have a different element
1834// type.
1835//
1836// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
1837// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1838// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1839//
1840// Also handles some irregular breakdown cases, e.g.
1841// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
1842// <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1843// s64 = G_SHL s64, s32
1844LegalizerHelper::LegalizeResult
1845LegalizerHelper::fewerElementsVectorMultiEltType(
1846 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
1847 if (TypeIdx != 0)
1848 return UnableToLegalize;
1849
1850 const LLT NarrowTy0 = NarrowTyArg;
1851 const unsigned NewNumElts =
1852 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
1853
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001854 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001855 LLT DstTy = MRI.getType(DstReg);
1856 LLT LeftoverTy0;
1857
Matt Arsenaultd3093c22019-02-28 00:16:32 +00001858 int NumParts, NumLeftover;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001859 // All of the operands need to have the same number of elements, so if we can
1860 // determine a type breakdown for the result type, we can for all of the
1861 // source types.
Matt Arsenaultd3093c22019-02-28 00:16:32 +00001862 std::tie(NumParts, NumLeftover)
1863 = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001864 if (NumParts < 0)
1865 return UnableToLegalize;
1866
1867 SmallVector<MachineInstrBuilder, 4> NewInsts;
1868
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001869 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
1870 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001871
1872 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1873 LLT LeftoverTy;
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001874 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001875 LLT SrcTyI = MRI.getType(SrcReg);
1876 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
1877 LLT LeftoverTyI;
1878
1879 // Split this operand into the requested typed registers, and any leftover
1880 // required to reproduce the original type.
1881 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
1882 LeftoverRegs))
1883 return UnableToLegalize;
1884
1885 if (I == 1) {
1886 // For the first operand, create an instruction for each part and setup
1887 // the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001888 for (Register PartReg : PartRegs) {
1889 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001890 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
1891 .addDef(PartDstReg)
1892 .addUse(PartReg));
1893 DstRegs.push_back(PartDstReg);
1894 }
1895
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001896 for (Register LeftoverReg : LeftoverRegs) {
1897 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00001898 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
1899 .addDef(PartDstReg)
1900 .addUse(LeftoverReg));
1901 LeftoverDstRegs.push_back(PartDstReg);
1902 }
1903 } else {
1904 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
1905
1906 // Add the newly created operand splits to the existing instructions. The
1907 // odd-sized pieces are ordered after the requested NarrowTyArg sized
1908 // pieces.
1909 unsigned InstCount = 0;
1910 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
1911 NewInsts[InstCount++].addUse(PartRegs[J]);
1912 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
1913 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
1914 }
1915
1916 PartRegs.clear();
1917 LeftoverRegs.clear();
1918 }
1919
1920 // Insert the newly built operations and rebuild the result register.
1921 for (auto &MIB : NewInsts)
1922 MIRBuilder.insertInstr(MIB);
1923
1924 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
1925
1926 MI.eraseFromParent();
1927 return Legalized;
1928}
1929
Tim Northover69fa84a2016-10-14 22:18:18 +00001930LegalizerHelper::LegalizeResult
Matt Arsenaultca676342019-01-25 02:36:32 +00001931LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
1932 LLT NarrowTy) {
1933 if (TypeIdx != 0)
1934 return UnableToLegalize;
1935
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001936 Register DstReg = MI.getOperand(0).getReg();
1937 Register SrcReg = MI.getOperand(1).getReg();
Matt Arsenaultca676342019-01-25 02:36:32 +00001938 LLT DstTy = MRI.getType(DstReg);
1939 LLT SrcTy = MRI.getType(SrcReg);
1940
1941 LLT NarrowTy0 = NarrowTy;
1942 LLT NarrowTy1;
1943 unsigned NumParts;
1944
Matt Arsenaultcbaada62019-02-02 23:29:55 +00001945 if (NarrowTy.isVector()) {
Matt Arsenaultca676342019-01-25 02:36:32 +00001946 // Uneven breakdown not handled.
1947 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
1948 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
1949 return UnableToLegalize;
1950
1951 NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
Matt Arsenaultcbaada62019-02-02 23:29:55 +00001952 } else {
1953 NumParts = DstTy.getNumElements();
1954 NarrowTy1 = SrcTy.getElementType();
Matt Arsenaultca676342019-01-25 02:36:32 +00001955 }
1956
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00001957 SmallVector<Register, 4> SrcRegs, DstRegs;
Matt Arsenaultca676342019-01-25 02:36:32 +00001958 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
1959
1960 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001961 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultca676342019-01-25 02:36:32 +00001962 MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
1963 .addDef(DstReg)
1964 .addUse(SrcRegs[I]);
1965
1966 NewInst->setFlags(MI.getFlags());
1967 DstRegs.push_back(DstReg);
1968 }
1969
1970 if (NarrowTy.isVector())
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00001971 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00001972 else
1973 MIRBuilder.buildBuildVector(DstReg, DstRegs);
1974
1975 MI.eraseFromParent();
1976 return Legalized;
1977}
1978
1979LegalizerHelper::LegalizeResult
1980LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
1981 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00001982 Register DstReg = MI.getOperand(0).getReg();
1983 Register Src0Reg = MI.getOperand(2).getReg();
Matt Arsenault1b1e6852019-01-25 02:59:34 +00001984 LLT DstTy = MRI.getType(DstReg);
1985 LLT SrcTy = MRI.getType(Src0Reg);
1986
1987 unsigned NumParts;
1988 LLT NarrowTy0, NarrowTy1;
1989
1990 if (TypeIdx == 0) {
1991 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
1992 unsigned OldElts = DstTy.getNumElements();
1993
1994 NarrowTy0 = NarrowTy;
1995 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
1996 NarrowTy1 = NarrowTy.isVector() ?
1997 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
1998 SrcTy.getElementType();
1999
2000 } else {
2001 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
2002 unsigned OldElts = SrcTy.getNumElements();
2003
2004 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
2005 NarrowTy.getNumElements();
2006 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
2007 DstTy.getScalarSizeInBits());
2008 NarrowTy1 = NarrowTy;
2009 }
2010
2011 // FIXME: Don't know how to handle the situation where the small vectors
2012 // aren't all the same size yet.
2013 if (NarrowTy1.isVector() &&
2014 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
2015 return UnableToLegalize;
2016
2017 CmpInst::Predicate Pred
2018 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
2019
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002020 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002021 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
2022 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
2023
2024 for (unsigned I = 0; I < NumParts; ++I) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002025 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002026 DstRegs.push_back(DstReg);
2027
2028 if (MI.getOpcode() == TargetOpcode::G_ICMP)
2029 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2030 else {
2031 MachineInstr *NewCmp
2032 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
2033 NewCmp->setFlags(MI.getFlags());
2034 }
2035 }
2036
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002037 if (NarrowTy1.isVector())
Matt Arsenaultca676342019-01-25 02:36:32 +00002038 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2039 else
2040 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2041
2042 MI.eraseFromParent();
2043 return Legalized;
2044}
2045
2046LegalizerHelper::LegalizeResult
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002047LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
2048 LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002049 Register DstReg = MI.getOperand(0).getReg();
2050 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002051
2052 unsigned NumParts = 0;
2053 LLT NarrowTy0, NarrowTy1;
2054
2055 LLT DstTy = MRI.getType(DstReg);
2056 LLT CondTy = MRI.getType(CondReg);
2057 unsigned Size = DstTy.getSizeInBits();
2058
2059 assert(TypeIdx == 0 || CondTy.isVector());
2060
2061 if (TypeIdx == 0) {
2062 NarrowTy0 = NarrowTy;
2063 NarrowTy1 = CondTy;
2064
2065 unsigned NarrowSize = NarrowTy0.getSizeInBits();
2066 // FIXME: Don't know how to handle the situation where the small vectors
2067 // aren't all the same size yet.
2068 if (Size % NarrowSize != 0)
2069 return UnableToLegalize;
2070
2071 NumParts = Size / NarrowSize;
2072
2073 // Need to break down the condition type
2074 if (CondTy.isVector()) {
2075 if (CondTy.getNumElements() == NumParts)
2076 NarrowTy1 = CondTy.getElementType();
2077 else
2078 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
2079 CondTy.getScalarSizeInBits());
2080 }
2081 } else {
2082 NumParts = CondTy.getNumElements();
2083 if (NarrowTy.isVector()) {
2084 // TODO: Handle uneven breakdown.
2085 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
2086 return UnableToLegalize;
2087
2088 return UnableToLegalize;
2089 } else {
2090 NarrowTy0 = DstTy.getElementType();
2091 NarrowTy1 = NarrowTy;
2092 }
2093 }
2094
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002095 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002096 if (CondTy.isVector())
2097 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2098
2099 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2100 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2101
2102 for (unsigned i = 0; i < NumParts; ++i) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002103 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002104 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2105 Src1Regs[i], Src2Regs[i]);
2106 DstRegs.push_back(DstReg);
2107 }
2108
2109 if (NarrowTy0.isVector())
2110 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2111 else
2112 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2113
2114 MI.eraseFromParent();
2115 return Legalized;
2116}
2117
2118LegalizerHelper::LegalizeResult
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002119LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2120 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002121 const Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002122 LLT PhiTy = MRI.getType(DstReg);
2123 LLT LeftoverTy;
2124
2125 // All of the operands need to have the same number of elements, so if we can
2126 // determine a type breakdown for the result type, we can for all of the
2127 // source types.
2128 int NumParts, NumLeftover;
2129 std::tie(NumParts, NumLeftover)
2130 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2131 if (NumParts < 0)
2132 return UnableToLegalize;
2133
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002134 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002135 SmallVector<MachineInstrBuilder, 4> NewInsts;
2136
2137 const int TotalNumParts = NumParts + NumLeftover;
2138
2139 // Insert the new phis in the result block first.
2140 for (int I = 0; I != TotalNumParts; ++I) {
2141 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002142 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002143 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2144 .addDef(PartDstReg));
2145 if (I < NumParts)
2146 DstRegs.push_back(PartDstReg);
2147 else
2148 LeftoverDstRegs.push_back(PartDstReg);
2149 }
2150
2151 MachineBasicBlock *MBB = MI.getParent();
2152 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2153 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2154
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002155 SmallVector<Register, 4> PartRegs, LeftoverRegs;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002156
2157 // Insert code to extract the incoming values in each predecessor block.
2158 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2159 PartRegs.clear();
2160 LeftoverRegs.clear();
2161
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002162 Register SrcReg = MI.getOperand(I).getReg();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002163 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2164 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2165
2166 LLT Unused;
2167 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2168 LeftoverRegs))
2169 return UnableToLegalize;
2170
2171 // Add the newly created operand splits to the existing instructions. The
2172 // odd-sized pieces are ordered after the requested NarrowTyArg sized
2173 // pieces.
2174 for (int J = 0; J != TotalNumParts; ++J) {
2175 MachineInstrBuilder MIB = NewInsts[J];
2176 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2177 MIB.addMBB(&OpMBB);
2178 }
2179 }
2180
2181 MI.eraseFromParent();
2182 return Legalized;
2183}
2184
2185LegalizerHelper::LegalizeResult
Matt Arsenault7f09fd62019-02-05 00:26:12 +00002186LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2187 LLT NarrowTy) {
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002188 // FIXME: Don't know how to handle secondary types yet.
2189 if (TypeIdx != 0)
2190 return UnableToLegalize;
2191
Matt Arsenaultcfca2a72019-01-27 22:36:24 +00002192 MachineMemOperand *MMO = *MI.memoperands_begin();
2193
2194 // This implementation doesn't work for atomics. Give up instead of doing
2195 // something invalid.
2196 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2197 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
2198 return UnableToLegalize;
2199
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002200 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002201 Register ValReg = MI.getOperand(0).getReg();
2202 Register AddrReg = MI.getOperand(1).getReg();
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002203 LLT ValTy = MRI.getType(ValReg);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002204
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002205 int NumParts = -1;
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002206 int NumLeftover = -1;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002207 LLT LeftoverTy;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002208 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002209 if (IsLoad) {
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002210 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002211 } else {
2212 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002213 NarrowLeftoverRegs)) {
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002214 NumParts = NarrowRegs.size();
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002215 NumLeftover = NarrowLeftoverRegs.size();
2216 }
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002217 }
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002218
2219 if (NumParts == -1)
2220 return UnableToLegalize;
2221
2222 const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2223
2224 unsigned TotalSize = ValTy.getSizeInBits();
2225
2226 // Split the load/store into PartTy sized pieces starting at Offset. If this
2227 // is a load, return the new registers in ValRegs. For a store, each elements
2228 // of ValRegs should be PartTy. Returns the next offset that needs to be
2229 // handled.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002230 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002231 unsigned Offset) -> unsigned {
2232 MachineFunction &MF = MIRBuilder.getMF();
2233 unsigned PartSize = PartTy.getSizeInBits();
2234 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2235 Offset += PartSize, ++Idx) {
2236 unsigned ByteSize = PartSize / 8;
2237 unsigned ByteOffset = Offset / 8;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002238 Register NewAddrReg;
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002239
2240 MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2241
2242 MachineMemOperand *NewMMO =
2243 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2244
2245 if (IsLoad) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002246 Register Dst = MRI.createGenericVirtualRegister(PartTy);
Matt Arsenaultc7bce732019-01-31 02:46:05 +00002247 ValRegs.push_back(Dst);
2248 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2249 } else {
2250 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2251 }
2252 }
2253
2254 return Offset;
2255 };
2256
2257 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2258
2259 // Handle the rest of the register if this isn't an even type breakdown.
2260 if (LeftoverTy.isValid())
2261 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2262
2263 if (IsLoad) {
2264 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2265 LeftoverTy, NarrowLeftoverRegs);
2266 }
2267
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002268 MI.eraseFromParent();
2269 return Legalized;
2270}
2271
2272LegalizerHelper::LegalizeResult
Tim Northover69fa84a2016-10-14 22:18:18 +00002273LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
2274 LLT NarrowTy) {
Matt Arsenault1b1e6852019-01-25 02:59:34 +00002275 using namespace TargetOpcode;
Volkan Keles574d7372018-12-14 22:11:20 +00002276
2277 MIRBuilder.setInstr(MI);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002278 switch (MI.getOpcode()) {
2279 case G_IMPLICIT_DEF:
2280 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2281 case G_AND:
2282 case G_OR:
2283 case G_XOR:
2284 case G_ADD:
2285 case G_SUB:
2286 case G_MUL:
2287 case G_SMULH:
2288 case G_UMULH:
2289 case G_FADD:
2290 case G_FMUL:
2291 case G_FSUB:
2292 case G_FNEG:
2293 case G_FABS:
Matt Arsenault9dba67f2019-02-11 17:05:20 +00002294 case G_FCANONICALIZE:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002295 case G_FDIV:
2296 case G_FREM:
2297 case G_FMA:
2298 case G_FPOW:
2299 case G_FEXP:
2300 case G_FEXP2:
2301 case G_FLOG:
2302 case G_FLOG2:
2303 case G_FLOG10:
Jessica Paquetteba557672019-04-25 16:44:40 +00002304 case G_FNEARBYINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002305 case G_FCEIL:
Jessica Paquetteebdb0212019-02-11 17:22:58 +00002306 case G_FFLOOR:
Jessica Paquetted5c69e02019-04-19 23:41:52 +00002307 case G_FRINT:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002308 case G_INTRINSIC_ROUND:
2309 case G_INTRINSIC_TRUNC:
Jessica Paquette7db82d72019-01-28 18:34:18 +00002310 case G_FCOS:
2311 case G_FSIN:
Jessica Paquette22457f82019-01-30 21:03:52 +00002312 case G_FSQRT:
Matt Arsenaultd1bfc8d2019-01-31 02:34:03 +00002313 case G_BSWAP:
Amara Emersonae878da2019-04-10 23:06:08 +00002314 case G_SDIV:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00002315 case G_SMIN:
2316 case G_SMAX:
2317 case G_UMIN:
2318 case G_UMAX:
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00002319 case G_FMINNUM:
2320 case G_FMAXNUM:
2321 case G_FMINNUM_IEEE:
2322 case G_FMAXNUM_IEEE:
2323 case G_FMINIMUM:
2324 case G_FMAXIMUM:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002325 return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002326 case G_SHL:
2327 case G_LSHR:
2328 case G_ASHR:
Matt Arsenault75e30c42019-02-20 16:42:52 +00002329 case G_CTLZ:
2330 case G_CTLZ_ZERO_UNDEF:
2331 case G_CTTZ:
2332 case G_CTTZ_ZERO_UNDEF:
2333 case G_CTPOP:
Matt Arsenault1448f562019-05-17 12:19:52 +00002334 case G_FCOPYSIGN:
Matt Arsenaultc83b8232019-02-07 17:38:00 +00002335 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002336 case G_ZEXT:
2337 case G_SEXT:
2338 case G_ANYEXT:
2339 case G_FPEXT:
2340 case G_FPTRUNC:
2341 case G_SITOFP:
2342 case G_UITOFP:
2343 case G_FPTOSI:
2344 case G_FPTOUI:
Matt Arsenaultcbaada62019-02-02 23:29:55 +00002345 case G_INTTOPTR:
2346 case G_PTRTOINT:
Matt Arsenaulta8b43392019-02-08 02:40:47 +00002347 case G_ADDRSPACE_CAST:
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002348 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2349 case G_ICMP:
2350 case G_FCMP:
2351 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
Matt Arsenaultdc6c7852019-01-30 04:19:31 +00002352 case G_SELECT:
2353 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
Matt Arsenaultd3093c22019-02-28 00:16:32 +00002354 case G_PHI:
2355 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
Matt Arsenault816c9b3e2019-01-27 21:53:09 +00002356 case G_LOAD:
2357 case G_STORE:
Matt Arsenault7f09fd62019-02-05 00:26:12 +00002358 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
Tim Northover33b07d62016-07-22 20:03:43 +00002359 default:
2360 return UnableToLegalize;
Tim Northover33b07d62016-07-22 20:03:43 +00002361 }
2362}
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002363
2364LegalizerHelper::LegalizeResult
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002365LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
2366 const LLT HalfTy, const LLT AmtTy) {
2367
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002368 Register InL = MRI.createGenericVirtualRegister(HalfTy);
2369 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002370 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2371
2372 if (Amt.isNullValue()) {
2373 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
2374 MI.eraseFromParent();
2375 return Legalized;
2376 }
2377
2378 LLT NVT = HalfTy;
2379 unsigned NVTBits = HalfTy.getSizeInBits();
2380 unsigned VTBits = 2 * NVTBits;
2381
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002382 SrcOp Lo(Register(0)), Hi(Register(0));
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002383 if (MI.getOpcode() == TargetOpcode::G_SHL) {
2384 if (Amt.ugt(VTBits)) {
2385 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2386 } else if (Amt.ugt(NVTBits)) {
2387 Lo = MIRBuilder.buildConstant(NVT, 0);
2388 Hi = MIRBuilder.buildShl(NVT, InL,
2389 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2390 } else if (Amt == NVTBits) {
2391 Lo = MIRBuilder.buildConstant(NVT, 0);
2392 Hi = InL;
2393 } else {
2394 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
Matt Arsenaulte98cab12019-02-07 20:44:08 +00002395 auto OrLHS =
2396 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
2397 auto OrRHS = MIRBuilder.buildLShr(
2398 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2399 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002400 }
2401 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2402 if (Amt.ugt(VTBits)) {
2403 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2404 } else if (Amt.ugt(NVTBits)) {
2405 Lo = MIRBuilder.buildLShr(NVT, InH,
2406 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2407 Hi = MIRBuilder.buildConstant(NVT, 0);
2408 } else if (Amt == NVTBits) {
2409 Lo = InH;
2410 Hi = MIRBuilder.buildConstant(NVT, 0);
2411 } else {
2412 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2413
2414 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2415 auto OrRHS = MIRBuilder.buildShl(
2416 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2417
2418 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2419 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
2420 }
2421 } else {
2422 if (Amt.ugt(VTBits)) {
2423 Hi = Lo = MIRBuilder.buildAShr(
2424 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2425 } else if (Amt.ugt(NVTBits)) {
2426 Lo = MIRBuilder.buildAShr(NVT, InH,
2427 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2428 Hi = MIRBuilder.buildAShr(NVT, InH,
2429 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2430 } else if (Amt == NVTBits) {
2431 Lo = InH;
2432 Hi = MIRBuilder.buildAShr(NVT, InH,
2433 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2434 } else {
2435 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2436
2437 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2438 auto OrRHS = MIRBuilder.buildShl(
2439 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2440
2441 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2442 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
2443 }
2444 }
2445
2446 MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
2447 MI.eraseFromParent();
2448
2449 return Legalized;
2450}
2451
2452// TODO: Optimize if constant shift amount.
2453LegalizerHelper::LegalizeResult
2454LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
2455 LLT RequestedTy) {
2456 if (TypeIdx == 1) {
2457 Observer.changingInstr(MI);
2458 narrowScalarSrc(MI, RequestedTy, 2);
2459 Observer.changedInstr(MI);
2460 return Legalized;
2461 }
2462
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002463 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002464 LLT DstTy = MRI.getType(DstReg);
2465 if (DstTy.isVector())
2466 return UnableToLegalize;
2467
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002468 Register Amt = MI.getOperand(2).getReg();
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002469 LLT ShiftAmtTy = MRI.getType(Amt);
2470 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
2471 if (DstEltSize % 2 != 0)
2472 return UnableToLegalize;
2473
2474 // Ignore the input type. We can only go to exactly half the size of the
2475 // input. If that isn't small enough, the resulting pieces will be further
2476 // legalized.
2477 const unsigned NewBitSize = DstEltSize / 2;
2478 const LLT HalfTy = LLT::scalar(NewBitSize);
2479 const LLT CondTy = LLT::scalar(1);
2480
2481 if (const MachineInstr *KShiftAmt =
2482 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
2483 return narrowScalarShiftByConstant(
2484 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
2485 }
2486
2487 // TODO: Expand with known bits.
2488
2489 // Handle the fully general expansion by an unknown amount.
2490 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
2491
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002492 Register InL = MRI.createGenericVirtualRegister(HalfTy);
2493 Register InH = MRI.createGenericVirtualRegister(HalfTy);
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002494 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2495
2496 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
2497 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
2498
2499 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
2500 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
2501 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
2502
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002503 Register ResultRegs[2];
Matt Arsenaultfbec8fe2019-02-07 19:37:44 +00002504 switch (MI.getOpcode()) {
2505 case TargetOpcode::G_SHL: {
2506 // Short: ShAmt < NewBitSize
2507 auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2508
2509 auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2510 auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
2511 auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2512
2513 // Long: ShAmt >= NewBitSize
2514 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
2515 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
2516
2517 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
2518 auto Hi = MIRBuilder.buildSelect(
2519 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
2520
2521 ResultRegs[0] = Lo.getReg(0);
2522 ResultRegs[1] = Hi.getReg(0);
2523 break;
2524 }
2525 case TargetOpcode::G_LSHR: {
2526 // Short: ShAmt < NewBitSize
2527 auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt);
2528
2529 auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2530 auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
2531 auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2532
2533 // Long: ShAmt >= NewBitSize
2534 auto HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
2535 auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2536
2537 auto Lo = MIRBuilder.buildSelect(
2538 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2539 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2540
2541 ResultRegs[0] = Lo.getReg(0);
2542 ResultRegs[1] = Hi.getReg(0);
2543 break;
2544 }
2545 case TargetOpcode::G_ASHR: {
2546 // Short: ShAmt < NewBitSize
2547 auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt);
2548
2549 auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2550 auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack);
2551 auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2552
2553 // Long: ShAmt >= NewBitSize
2554
2555 // Sign of Hi part.
2556 auto HiL = MIRBuilder.buildAShr(
2557 HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1));
2558
2559 auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2560
2561 auto Lo = MIRBuilder.buildSelect(
2562 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2563
2564 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2565
2566 ResultRegs[0] = Lo.getReg(0);
2567 ResultRegs[1] = Hi.getReg(0);
2568 break;
2569 }
2570 default:
2571 llvm_unreachable("not a shift");
2572 }
2573
2574 MIRBuilder.buildMerge(DstReg, ResultRegs);
2575 MI.eraseFromParent();
2576 return Legalized;
2577}
2578
2579LegalizerHelper::LegalizeResult
Matt Arsenault72bcf152019-02-28 00:01:05 +00002580LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2581 LLT MoreTy) {
2582 assert(TypeIdx == 0 && "Expecting only Idx 0");
2583
2584 Observer.changingInstr(MI);
2585 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2586 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2587 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2588 moreElementsVectorSrc(MI, MoreTy, I);
2589 }
2590
2591 MachineBasicBlock &MBB = *MI.getParent();
2592 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2593 moreElementsVectorDst(MI, MoreTy, 0);
2594 Observer.changedInstr(MI);
2595 return Legalized;
2596}
2597
2598LegalizerHelper::LegalizeResult
Matt Arsenault18ec3822019-02-11 22:00:39 +00002599LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
2600 LLT MoreTy) {
2601 MIRBuilder.setInstr(MI);
2602 unsigned Opc = MI.getOpcode();
2603 switch (Opc) {
2604 case TargetOpcode::G_IMPLICIT_DEF: {
2605 Observer.changingInstr(MI);
2606 moreElementsVectorDst(MI, MoreTy, 0);
2607 Observer.changedInstr(MI);
2608 return Legalized;
2609 }
Matt Arsenault26b7e852019-02-19 16:30:19 +00002610 case TargetOpcode::G_AND:
2611 case TargetOpcode::G_OR:
Matt Arsenault0f3ba442019-05-23 17:58:48 +00002612 case TargetOpcode::G_XOR:
2613 case TargetOpcode::G_SMIN:
2614 case TargetOpcode::G_SMAX:
2615 case TargetOpcode::G_UMIN:
2616 case TargetOpcode::G_UMAX: {
Matt Arsenault26b7e852019-02-19 16:30:19 +00002617 Observer.changingInstr(MI);
2618 moreElementsVectorSrc(MI, MoreTy, 1);
2619 moreElementsVectorSrc(MI, MoreTy, 2);
2620 moreElementsVectorDst(MI, MoreTy, 0);
2621 Observer.changedInstr(MI);
2622 return Legalized;
2623 }
Matt Arsenault4d884272019-02-19 16:44:22 +00002624 case TargetOpcode::G_EXTRACT:
2625 if (TypeIdx != 1)
2626 return UnableToLegalize;
2627 Observer.changingInstr(MI);
2628 moreElementsVectorSrc(MI, MoreTy, 1);
2629 Observer.changedInstr(MI);
2630 return Legalized;
Matt Arsenaultc4d07552019-02-20 16:11:22 +00002631 case TargetOpcode::G_INSERT:
2632 if (TypeIdx != 0)
2633 return UnableToLegalize;
2634 Observer.changingInstr(MI);
2635 moreElementsVectorSrc(MI, MoreTy, 1);
2636 moreElementsVectorDst(MI, MoreTy, 0);
2637 Observer.changedInstr(MI);
2638 return Legalized;
Matt Arsenaultb4c95b32019-02-19 17:03:09 +00002639 case TargetOpcode::G_SELECT:
2640 if (TypeIdx != 0)
2641 return UnableToLegalize;
2642 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
2643 return UnableToLegalize;
2644
2645 Observer.changingInstr(MI);
2646 moreElementsVectorSrc(MI, MoreTy, 2);
2647 moreElementsVectorSrc(MI, MoreTy, 3);
2648 moreElementsVectorDst(MI, MoreTy, 0);
2649 Observer.changedInstr(MI);
2650 return Legalized;
Matt Arsenault72bcf152019-02-28 00:01:05 +00002651 case TargetOpcode::G_PHI:
2652 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
Matt Arsenault18ec3822019-02-11 22:00:39 +00002653 default:
2654 return UnableToLegalize;
2655 }
2656}
2657
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002658void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
2659 ArrayRef<Register> Src1Regs,
2660 ArrayRef<Register> Src2Regs,
Petar Avramovic0b17e592019-03-11 10:00:17 +00002661 LLT NarrowTy) {
2662 MachineIRBuilder &B = MIRBuilder;
2663 unsigned SrcParts = Src1Regs.size();
2664 unsigned DstParts = DstRegs.size();
2665
2666 unsigned DstIdx = 0; // Low bits of the result.
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002667 Register FactorSum =
Petar Avramovic0b17e592019-03-11 10:00:17 +00002668 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
2669 DstRegs[DstIdx] = FactorSum;
2670
2671 unsigned CarrySumPrevDstIdx;
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002672 SmallVector<Register, 4> Factors;
Petar Avramovic0b17e592019-03-11 10:00:17 +00002673
2674 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
2675 // Collect low parts of muls for DstIdx.
2676 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
2677 i <= std::min(DstIdx, SrcParts - 1); ++i) {
2678 MachineInstrBuilder Mul =
2679 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
2680 Factors.push_back(Mul.getReg(0));
2681 }
2682 // Collect high parts of muls from previous DstIdx.
2683 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
2684 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
2685 MachineInstrBuilder Umulh =
2686 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
2687 Factors.push_back(Umulh.getReg(0));
2688 }
2689 // Add CarrySum from additons calculated for previous DstIdx.
2690 if (DstIdx != 1) {
2691 Factors.push_back(CarrySumPrevDstIdx);
2692 }
2693
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002694 Register CarrySum;
Petar Avramovic0b17e592019-03-11 10:00:17 +00002695 // Add all factors and accumulate all carries into CarrySum.
2696 if (DstIdx != DstParts - 1) {
2697 MachineInstrBuilder Uaddo =
2698 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
2699 FactorSum = Uaddo.getReg(0);
2700 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
2701 for (unsigned i = 2; i < Factors.size(); ++i) {
2702 MachineInstrBuilder Uaddo =
2703 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
2704 FactorSum = Uaddo.getReg(0);
2705 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
2706 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
2707 }
2708 } else {
2709 // Since value for the next index is not calculated, neither is CarrySum.
2710 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
2711 for (unsigned i = 2; i < Factors.size(); ++i)
2712 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
2713 }
2714
2715 CarrySumPrevDstIdx = CarrySum;
2716 DstRegs[DstIdx] = FactorSum;
2717 Factors.clear();
2718 }
2719}
2720
Matt Arsenault18ec3822019-02-11 22:00:39 +00002721LegalizerHelper::LegalizeResult
Petar Avramovic0b17e592019-03-11 10:00:17 +00002722LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002723 Register DstReg = MI.getOperand(0).getReg();
2724 Register Src1 = MI.getOperand(1).getReg();
2725 Register Src2 = MI.getOperand(2).getReg();
Petar Avramovic0b17e592019-03-11 10:00:17 +00002726
Matt Arsenault211e89d2019-01-27 00:52:51 +00002727 LLT Ty = MRI.getType(DstReg);
2728 if (Ty.isVector())
2729 return UnableToLegalize;
2730
Petar Avramovic0b17e592019-03-11 10:00:17 +00002731 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
2732 unsigned DstSize = Ty.getSizeInBits();
2733 unsigned NarrowSize = NarrowTy.getSizeInBits();
2734 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
Matt Arsenault211e89d2019-01-27 00:52:51 +00002735 return UnableToLegalize;
2736
Petar Avramovic0b17e592019-03-11 10:00:17 +00002737 unsigned NumDstParts = DstSize / NarrowSize;
2738 unsigned NumSrcParts = SrcSize / NarrowSize;
Petar Avramovic5229f472019-03-11 10:08:44 +00002739 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
2740 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
Matt Arsenault211e89d2019-01-27 00:52:51 +00002741
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002742 SmallVector<Register, 2> Src1Parts, Src2Parts, DstTmpRegs;
Petar Avramovic0b17e592019-03-11 10:00:17 +00002743 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
2744 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
Petar Avramovic5229f472019-03-11 10:08:44 +00002745 DstTmpRegs.resize(DstTmpParts);
2746 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
Matt Arsenault211e89d2019-01-27 00:52:51 +00002747
Petar Avramovic5229f472019-03-11 10:08:44 +00002748 // Take only high half of registers if this is high mul.
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002749 ArrayRef<Register> DstRegs(
Petar Avramovic5229f472019-03-11 10:08:44 +00002750 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
Petar Avramovic0b17e592019-03-11 10:00:17 +00002751 MIRBuilder.buildMerge(DstReg, DstRegs);
Matt Arsenault211e89d2019-01-27 00:52:51 +00002752 MI.eraseFromParent();
2753 return Legalized;
2754}
2755
Matt Arsenault1cf713662019-02-12 14:54:52 +00002756LegalizerHelper::LegalizeResult
2757LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2758 LLT NarrowTy) {
2759 if (TypeIdx != 1)
2760 return UnableToLegalize;
2761
2762 uint64_t NarrowSize = NarrowTy.getSizeInBits();
2763
2764 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
2765 // FIXME: add support for when SizeOp1 isn't an exact multiple of
2766 // NarrowSize.
2767 if (SizeOp1 % NarrowSize != 0)
2768 return UnableToLegalize;
2769 int NumParts = SizeOp1 / NarrowSize;
2770
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002771 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00002772 SmallVector<uint64_t, 2> Indexes;
2773 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2774
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002775 Register OpReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00002776 uint64_t OpStart = MI.getOperand(2).getImm();
2777 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2778 for (int i = 0; i < NumParts; ++i) {
2779 unsigned SrcStart = i * NarrowSize;
2780
2781 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
2782 // No part of the extract uses this subregister, ignore it.
2783 continue;
2784 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2785 // The entire subregister is extracted, forward the value.
2786 DstRegs.push_back(SrcRegs[i]);
2787 continue;
2788 }
2789
2790 // OpSegStart is where this destination segment would start in OpReg if it
2791 // extended infinitely in both directions.
2792 int64_t ExtractOffset;
2793 uint64_t SegSize;
2794 if (OpStart < SrcStart) {
2795 ExtractOffset = 0;
2796 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
2797 } else {
2798 ExtractOffset = OpStart - SrcStart;
2799 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
2800 }
2801
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002802 Register SegReg = SrcRegs[i];
Matt Arsenault1cf713662019-02-12 14:54:52 +00002803 if (ExtractOffset != 0 || SegSize != NarrowSize) {
2804 // A genuine extract is needed.
2805 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2806 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
2807 }
2808
2809 DstRegs.push_back(SegReg);
2810 }
2811
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002812 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00002813 if(MRI.getType(DstReg).isVector())
2814 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2815 else
2816 MIRBuilder.buildMerge(DstReg, DstRegs);
2817 MI.eraseFromParent();
2818 return Legalized;
2819}
2820
2821LegalizerHelper::LegalizeResult
2822LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2823 LLT NarrowTy) {
2824 // FIXME: Don't know how to handle secondary types yet.
2825 if (TypeIdx != 0)
2826 return UnableToLegalize;
2827
2828 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
2829 uint64_t NarrowSize = NarrowTy.getSizeInBits();
2830
2831 // FIXME: add support for when SizeOp0 isn't an exact multiple of
2832 // NarrowSize.
2833 if (SizeOp0 % NarrowSize != 0)
2834 return UnableToLegalize;
2835
2836 int NumParts = SizeOp0 / NarrowSize;
2837
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002838 SmallVector<Register, 2> SrcRegs, DstRegs;
Matt Arsenault1cf713662019-02-12 14:54:52 +00002839 SmallVector<uint64_t, 2> Indexes;
2840 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2841
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002842 Register OpReg = MI.getOperand(2).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00002843 uint64_t OpStart = MI.getOperand(3).getImm();
2844 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2845 for (int i = 0; i < NumParts; ++i) {
2846 unsigned DstStart = i * NarrowSize;
2847
2848 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
2849 // No part of the insert affects this subregister, forward the original.
2850 DstRegs.push_back(SrcRegs[i]);
2851 continue;
2852 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2853 // The entire subregister is defined by this insert, forward the new
2854 // value.
2855 DstRegs.push_back(OpReg);
2856 continue;
2857 }
2858
2859 // OpSegStart is where this destination segment would start in OpReg if it
2860 // extended infinitely in both directions.
2861 int64_t ExtractOffset, InsertOffset;
2862 uint64_t SegSize;
2863 if (OpStart < DstStart) {
2864 InsertOffset = 0;
2865 ExtractOffset = DstStart - OpStart;
2866 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
2867 } else {
2868 InsertOffset = OpStart - DstStart;
2869 ExtractOffset = 0;
2870 SegSize =
2871 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
2872 }
2873
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002874 Register SegReg = OpReg;
Matt Arsenault1cf713662019-02-12 14:54:52 +00002875 if (ExtractOffset != 0 || SegSize != OpSize) {
2876 // A genuine extract is needed.
2877 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2878 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
2879 }
2880
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002881 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
Matt Arsenault1cf713662019-02-12 14:54:52 +00002882 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
2883 DstRegs.push_back(DstReg);
2884 }
2885
2886 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002887 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault1cf713662019-02-12 14:54:52 +00002888 if(MRI.getType(DstReg).isVector())
2889 MIRBuilder.buildBuildVector(DstReg, DstRegs);
2890 else
2891 MIRBuilder.buildMerge(DstReg, DstRegs);
2892 MI.eraseFromParent();
2893 return Legalized;
2894}
2895
Matt Arsenault211e89d2019-01-27 00:52:51 +00002896LegalizerHelper::LegalizeResult
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00002897LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
2898 LLT NarrowTy) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002899 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00002900 LLT DstTy = MRI.getType(DstReg);
2901
2902 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
2903
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002904 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
2905 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
2906 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
Matt Arsenault9e0eeba2019-04-10 17:07:56 +00002907 LLT LeftoverTy;
2908 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
2909 Src0Regs, Src0LeftoverRegs))
2910 return UnableToLegalize;
2911
2912 LLT Unused;
2913 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
2914 Src1Regs, Src1LeftoverRegs))
2915 llvm_unreachable("inconsistent extractParts result");
2916
2917 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2918 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
2919 {Src0Regs[I], Src1Regs[I]});
2920 DstRegs.push_back(Inst->getOperand(0).getReg());
2921 }
2922
2923 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2924 auto Inst = MIRBuilder.buildInstr(
2925 MI.getOpcode(),
2926 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
2927 DstLeftoverRegs.push_back(Inst->getOperand(0).getReg());
2928 }
2929
2930 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2931 LeftoverTy, DstLeftoverRegs);
2932
2933 MI.eraseFromParent();
2934 return Legalized;
2935}
2936
2937LegalizerHelper::LegalizeResult
Matt Arsenault81511e52019-02-05 00:13:44 +00002938LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
2939 LLT NarrowTy) {
2940 if (TypeIdx != 0)
2941 return UnableToLegalize;
2942
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002943 Register CondReg = MI.getOperand(1).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00002944 LLT CondTy = MRI.getType(CondReg);
2945 if (CondTy.isVector()) // TODO: Handle vselect
2946 return UnableToLegalize;
2947
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00002948 Register DstReg = MI.getOperand(0).getReg();
Matt Arsenault81511e52019-02-05 00:13:44 +00002949 LLT DstTy = MRI.getType(DstReg);
2950
Matt Arsenaulte3a676e2019-06-24 15:50:29 +00002951 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
2952 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
2953 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
Matt Arsenault81511e52019-02-05 00:13:44 +00002954 LLT LeftoverTy;
2955 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
2956 Src1Regs, Src1LeftoverRegs))
2957 return UnableToLegalize;
2958
2959 LLT Unused;
2960 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
2961 Src2Regs, Src2LeftoverRegs))
2962 llvm_unreachable("inconsistent extractParts result");
2963
2964 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2965 auto Select = MIRBuilder.buildSelect(NarrowTy,
2966 CondReg, Src1Regs[I], Src2Regs[I]);
2967 DstRegs.push_back(Select->getOperand(0).getReg());
2968 }
2969
2970 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2971 auto Select = MIRBuilder.buildSelect(
2972 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
2973 DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
2974 }
2975
2976 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2977 LeftoverTy, DstLeftoverRegs);
2978
2979 MI.eraseFromParent();
2980 return Legalized;
2981}
2982
2983LegalizerHelper::LegalizeResult
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002984LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
2985 unsigned Opc = MI.getOpcode();
2986 auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
Diana Picus0528e2c2018-11-26 11:07:02 +00002987 auto isSupported = [this](const LegalityQuery &Q) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002988 auto QAction = LI.getAction(Q).Action;
Diana Picus0528e2c2018-11-26 11:07:02 +00002989 return QAction == Legal || QAction == Libcall || QAction == Custom;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002990 };
2991 switch (Opc) {
2992 default:
2993 return UnableToLegalize;
2994 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
2995 // This trivially expands to CTLZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00002996 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002997 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00002998 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00002999 return Legalized;
3000 }
3001 case TargetOpcode::G_CTLZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003002 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003003 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003004 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
Diana Picus0528e2c2018-11-26 11:07:02 +00003005 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003006 auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
3007 {Ty}, {SrcReg});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003008 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3009 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3010 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3011 SrcReg, MIBZero);
3012 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3013 MIBCtlzZU);
3014 MI.eraseFromParent();
3015 return Legalized;
3016 }
3017 // for now, we do this:
3018 // NewLen = NextPowerOf2(Len);
3019 // x = x | (x >> 1);
3020 // x = x | (x >> 2);
3021 // ...
3022 // x = x | (x >>16);
3023 // x = x | (x >>32); // for 64-bit input
3024 // Upto NewLen/2
3025 // return Len - popcount(x);
3026 //
3027 // Ref: "Hacker's Delight" by Henry Warren
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003028 Register Op = SrcReg;
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003029 unsigned NewLen = PowerOf2Ceil(Len);
3030 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
3031 auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
3032 auto MIBOp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003033 TargetOpcode::G_OR, {Ty},
3034 {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
3035 {Op, MIBShiftAmt})});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003036 Op = MIBOp->getOperand(0).getReg();
3037 }
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003038 auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
3039 MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3040 {MIRBuilder.buildConstant(Ty, Len), MIBPop});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003041 MI.eraseFromParent();
3042 return Legalized;
3043 }
3044 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
3045 // This trivially expands to CTTZ.
Daniel Sandersd001e0e2018-12-12 23:48:13 +00003046 Observer.changingInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003047 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
Aditya Nandakumarf75d4f32018-12-05 20:14:52 +00003048 Observer.changedInstr(MI);
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003049 return Legalized;
3050 }
3051 case TargetOpcode::G_CTTZ: {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003052 Register SrcReg = MI.getOperand(1).getReg();
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003053 unsigned Len = Ty.getSizeInBits();
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003054 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003055 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
3056 // zero.
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003057 auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
3058 {Ty}, {SrcReg});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003059 auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
3060 auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
3061 auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
3062 SrcReg, MIBZero);
3063 MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
3064 MIBCttzZU);
3065 MI.eraseFromParent();
3066 return Legalized;
3067 }
3068 // for now, we use: { return popcount(~x & (x - 1)); }
3069 // unless the target has ctlz but not ctpop, in which case we use:
3070 // { return 32 - nlz(~x & (x-1)); }
3071 // Ref: "Hacker's Delight" by Henry Warren
3072 auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
3073 auto MIBNot =
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003074 MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003075 auto MIBTmp = MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003076 TargetOpcode::G_AND, {Ty},
3077 {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
3078 {SrcReg, MIBCstNeg1})});
Matt Arsenaultd5684f72019-01-31 02:09:57 +00003079 if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
3080 isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003081 auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
3082 MIRBuilder.buildInstr(
Aditya Nandakumarcef44a22018-12-11 00:48:50 +00003083 TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
3084 {MIBCstLen,
3085 MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
Aditya Nandakumarc0333f72018-08-21 17:30:31 +00003086 MI.eraseFromParent();
3087 return Legalized;
3088 }
3089 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
3090 MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
3091 return Legalized;
3092 }
3093 }
3094}
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003095
3096// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
3097// representation.
3098LegalizerHelper::LegalizeResult
3099LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003100 Register Dst = MI.getOperand(0).getReg();
3101 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003102 const LLT S64 = LLT::scalar(64);
3103 const LLT S32 = LLT::scalar(32);
3104 const LLT S1 = LLT::scalar(1);
3105
3106 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
3107
3108 // unsigned cul2f(ulong u) {
3109 // uint lz = clz(u);
3110 // uint e = (u != 0) ? 127U + 63U - lz : 0;
3111 // u = (u << lz) & 0x7fffffffffffffffUL;
3112 // ulong t = u & 0xffffffffffUL;
3113 // uint v = (e << 23) | (uint)(u >> 40);
3114 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
3115 // return as_float(v + r);
3116 // }
3117
3118 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
3119 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
3120
3121 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
3122
3123 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
3124 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
3125
3126 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
3127 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
3128
3129 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
3130 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
3131
3132 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
3133
3134 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
3135 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
3136
3137 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
3138 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
3139 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
3140
3141 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
3142 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
3143 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
3144 auto One = MIRBuilder.buildConstant(S32, 1);
3145
3146 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
3147 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
3148 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
3149 MIRBuilder.buildAdd(Dst, V, R);
3150
3151 return Legalized;
3152}
3153
3154LegalizerHelper::LegalizeResult
3155LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003156 Register Dst = MI.getOperand(0).getReg();
3157 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003158 LLT DstTy = MRI.getType(Dst);
3159 LLT SrcTy = MRI.getType(Src);
3160
3161 if (SrcTy != LLT::scalar(64))
3162 return UnableToLegalize;
3163
3164 if (DstTy == LLT::scalar(32)) {
3165 // TODO: SelectionDAG has several alternative expansions to port which may
3166 // be more reasonble depending on the available instructions. If a target
3167 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
3168 // intermediate type, this is probably worse.
3169 return lowerU64ToF32BitOps(MI);
3170 }
3171
3172 return UnableToLegalize;
3173}
3174
3175LegalizerHelper::LegalizeResult
3176LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003177 Register Dst = MI.getOperand(0).getReg();
3178 Register Src = MI.getOperand(1).getReg();
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003179 LLT DstTy = MRI.getType(Dst);
3180 LLT SrcTy = MRI.getType(Src);
3181
3182 const LLT S64 = LLT::scalar(64);
3183 const LLT S32 = LLT::scalar(32);
3184 const LLT S1 = LLT::scalar(1);
3185
3186 if (SrcTy != S64)
3187 return UnableToLegalize;
3188
3189 if (DstTy == S32) {
3190 // signed cl2f(long l) {
3191 // long s = l >> 63;
3192 // float r = cul2f((l + s) ^ s);
3193 // return s ? -r : r;
3194 // }
Matt Arsenaultfaeaedf2019-06-24 16:16:12 +00003195 Register L = Src;
Matt Arsenault02b5ca82019-05-17 23:05:13 +00003196 auto SignBit = MIRBuilder.buildConstant(S64, 63);
3197 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
3198
3199 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
3200 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
3201 auto R = MIRBuilder.buildUITOFP(S32, Xor);
3202
3203 auto RNeg = MIRBuilder.buildFNeg(S32, R);
3204 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
3205 MIRBuilder.buildConstant(S64, 0));
3206 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
3207 return Legalized;
3208 }
3209
3210 return UnableToLegalize;
3211}
Matt Arsenault6f74f552019-07-01 17:18:03 +00003212
3213static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
3214 switch (Opc) {
3215 case TargetOpcode::G_SMIN:
3216 return CmpInst::ICMP_SLT;
3217 case TargetOpcode::G_SMAX:
3218 return CmpInst::ICMP_SGT;
3219 case TargetOpcode::G_UMIN:
3220 return CmpInst::ICMP_ULT;
3221 case TargetOpcode::G_UMAX:
3222 return CmpInst::ICMP_UGT;
3223 default:
3224 llvm_unreachable("not in integer min/max");
3225 }
3226}
3227
3228LegalizerHelper::LegalizeResult
3229LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3230 Register Dst = MI.getOperand(0).getReg();
3231 Register Src0 = MI.getOperand(1).getReg();
3232 Register Src1 = MI.getOperand(2).getReg();
3233
3234 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
3235 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
3236
3237 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
3238 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
3239
3240 MI.eraseFromParent();
3241 return Legalized;
3242}
Matt Arsenaultb1843e12019-07-09 23:34:29 +00003243
3244LegalizerHelper::LegalizeResult
3245LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
3246 Register Dst = MI.getOperand(0).getReg();
3247 Register Src0 = MI.getOperand(1).getReg();
3248 Register Src1 = MI.getOperand(2).getReg();
3249
3250 const LLT Src0Ty = MRI.getType(Src0);
3251 const LLT Src1Ty = MRI.getType(Src1);
3252
3253 const int Src0Size = Src0Ty.getScalarSizeInBits();
3254 const int Src1Size = Src1Ty.getScalarSizeInBits();
3255
3256 auto SignBitMask = MIRBuilder.buildConstant(
3257 Src0Ty, APInt::getSignMask(Src0Size));
3258
3259 auto NotSignBitMask = MIRBuilder.buildConstant(
3260 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
3261
3262 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
3263 MachineInstr *Or;
3264
3265 if (Src0Ty == Src1Ty) {
3266 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask);
3267 Or = MIRBuilder.buildOr(Dst, And0, And1);
3268 } else if (Src0Size > Src1Size) {
3269 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
3270 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
3271 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
3272 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
3273 Or = MIRBuilder.buildOr(Dst, And0, And1);
3274 } else {
3275 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
3276 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
3277 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
3278 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
3279 Or = MIRBuilder.buildOr(Dst, And0, And1);
3280 }
3281
3282 // Be careful about setting nsz/nnan/ninf on every instruction, since the
3283 // constants are a nan and -0.0, but the final result should preserve
3284 // everything.
3285 if (unsigned Flags = MI.getFlags())
3286 Or->setFlags(Flags);
3287
3288 MI.eraseFromParent();
3289 return Legalized;
3290}
Matt Arsenault6ce1b4f2019-07-10 16:31:19 +00003291
3292LegalizerHelper::LegalizeResult
3293LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
3294 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
3295 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
3296
3297 Register Dst = MI.getOperand(0).getReg();
3298 Register Src0 = MI.getOperand(1).getReg();
3299 Register Src1 = MI.getOperand(2).getReg();
3300 LLT Ty = MRI.getType(Dst);
3301
3302 if (!MI.getFlag(MachineInstr::FmNoNans)) {
3303 // Insert canonicalizes if it's possible we need to quiet to get correct
3304 // sNaN behavior.
3305
3306 // Note this must be done here, and not as an optimization combine in the
3307 // absence of a dedicate quiet-snan instruction as we're using an
3308 // omni-purpose G_FCANONICALIZE.
3309 if (!isKnownNeverSNaN(Src0, MRI))
3310 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
3311
3312 if (!isKnownNeverSNaN(Src1, MRI))
3313 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
3314 }
3315
3316 // If there are no nans, it's safe to simply replace this with the non-IEEE
3317 // version.
3318 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
3319 MI.eraseFromParent();
3320 return Legalized;
3321}