llvm/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp

//===- MachineIRBuilderTest.cpp -------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "GISelMITest.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"

TEST_F(AArch64GISelMITest, TestBuildConstantFConstant) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  B.buildConstant(LLT::integer(32), 42);
  B.buildFConstant(LLT::float32(), 1.0);

  B.buildConstant(LLT::fixed_vector(2, LLT::integer(32)), 99);
  B.buildFConstant(LLT::fixed_vector(2, LLT::float32()), 2.0);

  // Test APFloat overload.
  APFloat KVal(APFloat::IEEEdouble(), "4.0");
  B.buildFConstant(LLT::float64(), KVal);

  auto CheckStr = R"(
  CHECK: [[CONST0:%[0-9]+]]:_(i32) = G_CONSTANT i32 42
  CHECK: [[FCONST0:%[0-9]+]]:_(f32) = G_FCONSTANT float 1.000000e+00
  CHECK: [[CONST1:%[0-9]+]]:_(i32) = G_CONSTANT i32 99
  CHECK: [[VEC0:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[CONST1]]:_(i32), [[CONST1]]:_(i32)
  CHECK: [[FCONST1:%[0-9]+]]:_(f32) = G_FCONSTANT float 2.000000e+00
  CHECK: [[VEC1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[FCONST1]]:_(f32), [[FCONST1]]:_(f32)
  CHECK: [[FCONST2:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.000000e+00
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, TestBuildFConstantBFloatSemantics) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  // bfloat 1.0 -> 0x3F80
  auto BF16One = B.buildFConstant(LLT::bfloat16(), 1.0);
  const APFloat &BF16APF = BF16One->getOperand(1).getFPImm()->getValueAPF();
  EXPECT_EQ(&BF16APF.getSemantics(), &APFloat::BFloat());
  EXPECT_EQ(BF16APF.bitcastToAPInt().getZExtValue(), 0x3F80u);

  // half 1.0 -> 0x3C00
  auto F16One = B.buildFConstant(LLT::float16(), 1.0);
  const APFloat &F16APF = F16One->getOperand(1).getFPImm()->getValueAPF();
  EXPECT_EQ(&F16APF.getSemantics(), &APFloat::IEEEhalf());
  EXPECT_EQ(F16APF.bitcastToAPInt().getZExtValue(), 0x3C00u);
}

#ifdef GTEST_HAS_DEATH_TEST
#ifndef NDEBUG

TEST_F(AArch64GISelMITest, TestBuildConstantFConstantDeath) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLVMContext &Ctx = MF->getFunction().getContext();
  APInt APV32(32, 12345);

  // Test APInt version breaks
  EXPECT_DEATH(B.buildConstant(LLT::integer(16), APV32),
               "creating constant with the wrong size");
  EXPECT_DEATH(B.buildConstant(LLT::fixed_vector(2, LLT::integer(16)), APV32),
               "creating constant with the wrong size");

  // Test ConstantInt version breaks
  ConstantInt *CI = ConstantInt::get(Ctx, APV32);
  EXPECT_DEATH(B.buildConstant(LLT::integer(16), *CI),
               "creating constant with the wrong size");
  EXPECT_DEATH(B.buildConstant(LLT::fixed_vector(2, LLT::integer(16)), *CI),
               "creating constant with the wrong size");

  APFloat DoubleVal(APFloat::IEEEdouble());
  ConstantFP *CF = ConstantFP::get(Ctx, DoubleVal);
  EXPECT_DEATH(B.buildFConstant(LLT::float16(), *CF),
               "creating fconstant with the wrong size");
  EXPECT_DEATH(B.buildFConstant(LLT::fixed_vector(2, LLT::float16()), *CF),
               "creating fconstant with the wrong size");
}

#endif
#endif

TEST_F(AArch64GISelMITest, DstOpSrcOp) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT s64 = LLT::integer(64);
  auto MIBAdd = B.buildAdd(s64, Copies[0], Copies[1]);

  // Test SrcOp and DstOp can be constructed directly from MachineOperand by
  // copying the instruction
  B.buildAdd(MIBAdd->getOperand(0), MIBAdd->getOperand(1), MIBAdd->getOperand(2));

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[ADD:%[0-9]+]]:_(i64) = G_ADD [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[ADD]]:_(i64) = G_ADD [[COPY0]]:_, [[COPY1]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildUnmerge) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  B.buildUnmerge(LLT::integer(32), Copies[0]);
  B.buildUnmerge(LLT::integer(16), Copies[1]);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[UNMERGE32_0:%[0-9]+]]:_(i32), [[UNMERGE32_1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY0]]
  ; CHECK: [[UNMERGE16_0:%[0-9]+]]:_(i16), [[UNMERGE16_1:%[0-9]+]]:_(i16), [[UNMERGE16_2:%[0-9]+]]:_(i16), [[UNMERGE16_3:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[COPY1]]

  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, TestBuildFPInsts) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT F64 = LLT::float64();
  auto BC0 = B.buildBitcast(F64, Copies[0]);
  auto BC1 = B.buildBitcast(F64, Copies[1]);
  auto BC2 = B.buildBitcast(F64, Copies[2]);

  B.buildFAdd(F64, BC0, BC1);
  B.buildFSub(F64, BC0, BC1);
  B.buildFMA(F64, BC0, BC1, BC2);
  B.buildFMAD(F64, BC0, BC1, BC2);
  B.buildFMAD(F64, BC0, BC1, BC2, MachineInstr::FmNoNans);
  B.buildFNeg(F64, BC0);
  B.buildFAbs(F64, BC0);
  B.buildFCopysign(F64, BC0, BC1);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(f64) = G_BITCAST
  ; CHECK: [[COPY1:%[0-9]+]]:_(f64) = G_BITCAST
  ; CHECK: [[COPY2:%[0-9]+]]:_(f64) = G_BITCAST
  ; CHECK: [[FADD:%[0-9]+]]:_(f64) = G_FADD [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[FSUB:%[0-9]+]]:_(f64) = G_FSUB [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[COPY0]]:_, [[COPY1]]:_, [[COPY2]]:_
  ; CHECK: [[FMAD0:%[0-9]+]]:_(f64) = G_FMAD [[COPY0]]:_, [[COPY1]]:_, [[COPY2]]:_
  ; CHECK: [[FMAD1:%[0-9]+]]:_(f64) = nnan G_FMAD [[COPY0]]:_, [[COPY1]]:_, [[COPY2]]:_
  ; CHECK: [[FNEG:%[0-9]+]]:_(f64) = G_FNEG [[COPY0]]:_
  ; CHECK: [[FABS:%[0-9]+]]:_(f64) = G_FABS [[COPY0]]:_
  ; CHECK: [[FCOPYSIGN:%[0-9]+]]:_(f64) = G_FCOPYSIGN [[COPY0]]:_, [[COPY1]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildIntrinsic) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S64 = LLT::integer(64);

  // Make sure DstOp version works. sqrt is just a placeholder intrinsic.
  B.buildIntrinsic(Intrinsic::sqrt, {S64}).addUse(Copies[0]);

  // Make sure register version works
  SmallVector<Register, 1> Results;
  Results.push_back(MRI->createGenericVirtualRegister(S64));
  B.buildIntrinsic(Intrinsic::sqrt, Results).addUse(Copies[1]);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[SQRT0:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.sqrt), [[COPY0]]:_
  ; CHECK: [[SQRT1:%[0-9]+]]:_(i64) = G_INTRINSIC intrinsic(@llvm.sqrt), [[COPY1]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildXor) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S64 = LLT::integer(64);
  LLT S128 = LLT::integer(128);
  B.buildXor(S64, Copies[0], Copies[1]);
  B.buildNot(S64, Copies[0]);

  // Make sure this works with > 64-bit types
  auto Merge = B.buildMergeLikeInstr(S128, {Copies[0], Copies[1]});
  B.buildNot(S128, Merge);
  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[XOR0:%[0-9]+]]:_(i64) = G_XOR [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[NEGONE64:%[0-9]+]]:_(i64) = G_CONSTANT i64 -1
  ; CHECK: [[XOR1:%[0-9]+]]:_(i64) = G_XOR [[COPY0]]:_, [[NEGONE64]]:_
  ; CHECK: [[MERGE:%[0-9]+]]:_(i128) = G_MERGE_VALUES [[COPY0]]:_(i64), [[COPY1]]:_(i64)
  ; CHECK: [[NEGONE128:%[0-9]+]]:_(i128) = G_CONSTANT i128 -1
  ; CHECK: [[XOR2:%[0-9]+]]:_(i128) = G_XOR [[MERGE]]:_, [[NEGONE128]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildBitCounts) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S32 = LLT::integer(32);

  B.buildCTPOP(S32, Copies[0]);
  B.buildCTLZ(S32, Copies[0]);
  B.buildCTLZ_ZERO_POISON(S32, Copies[1]);
  B.buildCTTZ(S32, Copies[0]);
  B.buildCTTZ_ZERO_POISON(S32, Copies[1]);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[CTPOP:%[0-9]+]]:_(i32) = G_CTPOP [[COPY0]]:_
  ; CHECK: [[CTLZ0:%[0-9]+]]:_(i32) = G_CTLZ [[COPY0]]:_
  ; CHECK: [[CTLZ_POISON0:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[COPY1]]:_
  ; CHECK: [[CTTZ:%[0-9]+]]:_(i32) = G_CTTZ [[COPY0]]:_
  ; CHECK: [[CTTZ_POISON0:%[0-9]+]]:_(i32) = G_CTTZ_ZERO_POISON [[COPY1]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildCasts) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT I32 = LLT::integer(32);
  LLT F32 = LLT::float32();

  auto FPU = B.buildUITOFP(F32, Copies[0]);
  auto FPS = B.buildSITOFP(F32, Copies[0]);
  B.buildFPTOUI(I32, FPU);
  B.buildFPTOSI(I32, FPS);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[UITOFP:%[0-9]+]]:_(f32) = G_UITOFP [[COPY0]]:_
  ; CHECK: [[SITOFP:%[0-9]+]]:_(f32) = G_SITOFP [[COPY0]]:_
  ; CHECK: [[FPTOUI:%[0-9]+]]:_(i32) = G_FPTOUI [[UITOFP]]:_
  ; CHECK: [[FPTOSI:%[0-9]+]]:_(i32) = G_FPTOSI [[SITOFP]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildMinMaxAbs) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S64 = LLT::integer(64);

  B.buildSMin(S64, Copies[0], Copies[1]);
  B.buildSMax(S64, Copies[0], Copies[1]);
  B.buildUMin(S64, Copies[0], Copies[1]);
  B.buildUMax(S64, Copies[0], Copies[1]);
  B.buildAbs(S64, Copies[0]);

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[SMIN0:%[0-9]+]]:_(i64) = G_SMIN [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[SMAX0:%[0-9]+]]:_(i64) = G_SMAX [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[UMIN0:%[0-9]+]]:_(i64) = G_UMIN [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[UMAX0:%[0-9]+]]:_(i64) = G_UMAX [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[UABS0:%[0-9]+]]:_(i64) = G_ABS [[COPY0]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildAtomicRMW) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT F64 = LLT::float64();
  LLT P0 = LLT::pointer(0, 64);

  MachineMemOperand *MMO = MF->getMachineMemOperand(
      MachinePointerInfo(),
      MachineMemOperand::MOLoad | MachineMemOperand::MOStore, 8, Align(8),
      AAMDNodes(), SyncScope::System, AtomicOrdering::Unordered);

  auto Ptr = B.buildUndef(P0);
  auto BC0 = B.buildBitcast(F64, Copies[0]);
  B.buildAtomicRMWFAdd(F64, Ptr, BC0, *MMO);
  B.buildAtomicRMWFSub(F64, Ptr, BC0, *MMO);

  auto CheckStr = R"(
  ; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
  ; CHECK: [[COPY0:%[0-9]+]]:_(f64) = G_BITCAST
  ; CHECK: [[FADD:%[0-9]+]]:_(f64) = G_ATOMICRMW_FADD [[PTR]]:_(p0), [[COPY0]]:_ :: (load store unordered (s64))
  ; CHECK: [[FSUB:%[0-9]+]]:_(f64) = G_ATOMICRMW_FSUB [[PTR]]:_(p0), [[COPY0]]:_ :: (load store unordered (s64))
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildMergeLikeInstr) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S32 = LLT::integer(32);
  Register RegC0 = B.buildConstant(S32, 0).getReg(0);
  Register RegC1 = B.buildConstant(S32, 1).getReg(0);
  Register RegC2 = B.buildConstant(S32, 2).getReg(0);
  Register RegC3 = B.buildConstant(S32, 3).getReg(0);

  // Merging plain constants as one big blob of bit should produce a
  // G_MERGE_VALUES.
  B.buildMergeLikeInstr(LLT::integer(128), {RegC0, RegC1, RegC2, RegC3});
  // Merging plain constants to a vector should produce a G_BUILD_VECTOR.
  LLT V2x32 = LLT::fixed_vector(2, S32);
  Register RegC0C1 = B.buildMergeLikeInstr(V2x32, {RegC0, RegC1}).getReg(0);
  Register RegC2C3 = B.buildMergeLikeInstr(V2x32, {RegC2, RegC3}).getReg(0);
  // Merging vector constants to a vector should produce a G_CONCAT_VECTORS.
  B.buildMergeLikeInstr(LLT::fixed_vector(4, S32), {RegC0C1, RegC2C3});
  // Merging vector constants to a plain type is not allowed.
  // Nothing else to test.

  auto CheckStr = R"(
  ; CHECK: [[C0:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
  ; CHECK: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
  ; CHECK: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
  ; CHECK: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
  ; CHECK: {{%[0-9]+}}:_(i128) = G_MERGE_VALUES [[C0]]:_(i32), [[C1]]:_(i32), [[C2]]:_(i32), [[C3]]:_(i32)
  ; CHECK: [[LOW2x32:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[C0]]:_(i32), [[C1]]:_(i32)
  ; CHECK: [[HIGH2x32:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[C2]]:_(i32), [[C3]]:_(i32)
  ; CHECK: {{%[0-9]+}}:_(<4 x i32>) = G_CONCAT_VECTORS [[LOW2x32]]:_(<2 x i32>), [[HIGH2x32]]:_(<2 x i32>)
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

using MachineIRBuilderDeathTest = AArch64GISelMITest;

TEST_F(MachineIRBuilderDeathTest, BuildMergeValues) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S32 = LLT::integer(32);
  Register RegC0 = B.buildConstant(S32, 0).getReg(0);
  Register RegC1 = B.buildConstant(S32, 1).getReg(0);
  Register RegC2 = B.buildConstant(S32, 2).getReg(0);
  Register RegC3 = B.buildConstant(S32, 3).getReg(0);

  // Merging scalar constants should produce a G_MERGE_VALUES.
  B.buildMergeValues(LLT::integer(128), {RegC0, RegC1, RegC2, RegC3});

  // Using a vector destination type should assert.
  LLT V2x32 = LLT::fixed_vector(2, S32);
  EXPECT_DEBUG_DEATH(
      B.buildMergeValues(V2x32, {RegC0, RegC1}),
      "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");

  auto CheckStr = R"(
  ; CHECK: [[C0:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
  ; CHECK: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
  ; CHECK: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
  ; CHECK: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
  ; CHECK: {{%[0-9]+}}:_(i128) = G_MERGE_VALUES [[C0]]:_(i32), [[C1]]:_(i32), [[C2]]:_(i32), [[C3]]:_(i32)
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildAddoSubo) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S1 = LLT::integer(1);
  LLT S64 = LLT::integer(64);

  auto UAddo = B.buildUAddo(S64, S1, Copies[0], Copies[1]);
  auto USubo = B.buildUSubo(S64, S1, Copies[0], Copies[1]);
  auto SAddo = B.buildSAddo(S64, S1, Copies[0], Copies[1]);
  auto SSubo = B.buildSSubo(S64, S1, Copies[0], Copies[1]);

  B.buildUAdde(S64, S1, Copies[0], Copies[1], UAddo.getReg(1));
  B.buildUSube(S64, S1, Copies[0], Copies[1], USubo.getReg(1));
  B.buildSAdde(S64, S1, Copies[0], Copies[1], SAddo.getReg(1));
  B.buildSSube(S64, S1, Copies[0], Copies[1], SSubo.getReg(1));

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[UADDO:%[0-9]+]]:_(i64), [[UADDO_FLAG:%[0-9]+]]:_(i1) = G_UADDO [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[USUBO:%[0-9]+]]:_(i64), [[USUBO_FLAG:%[0-9]+]]:_(i1) = G_USUBO [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[SADDO:%[0-9]+]]:_(i64), [[SADDO_FLAG:%[0-9]+]]:_(i1) = G_SADDO [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[SSUBO:%[0-9]+]]:_(i64), [[SSUBO_FLAG:%[0-9]+]]:_(i1) = G_SSUBO [[COPY0]]:_, [[COPY1]]:_
  ; CHECK: [[UADDE:%[0-9]+]]:_(i64), [[UADDE_FLAG:%[0-9]+]]:_(i1) = G_UADDE [[COPY0]]:_, [[COPY1]]:_, [[UADDO_FLAG]]
  ; CHECK: [[USUBE:%[0-9]+]]:_(i64), [[USUBE_FLAG:%[0-9]+]]:_(i1) = G_USUBE [[COPY0]]:_, [[COPY1]]:_, [[USUBO_FLAG]]
  ; CHECK: [[SADDE:%[0-9]+]]:_(i64), [[SADDE_FLAG:%[0-9]+]]:_(i1) = G_SADDE [[COPY0]]:_, [[COPY1]]:_, [[SADDO_FLAG]]
  ; CHECK: [[SSUBE:%[0-9]+]]:_(i64), [[SSUBE_FLAG:%[0-9]+]]:_(i1) = G_SSUBE [[COPY0]]:_, [[COPY1]]:_, [[SSUBO_FLAG]]
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildBitfieldExtract) {
  setUp();
  if (!TM)
    GTEST_SKIP();
  LLT S64 = LLT::integer(64);

  auto Ubfx = B.buildUbfx(S64, Copies[0], Copies[1], Copies[2]);
  B.buildSbfx(S64, Ubfx, Copies[0], Copies[2]);

  const auto *CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[COPY2:%[0-9]+]]:_(i64) = COPY $x2
  ; CHECK: [[UBFX:%[0-9]+]]:_(i64) = G_UBFX [[COPY0]]:_, [[COPY1]]:_(i64), [[COPY2]]:_
  ; CHECK: [[SBFX:%[0-9]+]]:_(i64) = G_SBFX [[UBFX]]:_, [[COPY0]]:_(i64), [[COPY2]]:_
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildFPEnv) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT S32 = LLT::integer(32);

  B.buildGetFPEnv(Copies[0]);
  B.buildSetFPEnv(Copies[1]);
  B.buildResetFPEnv();
  auto GetFPMode = B.buildGetFPMode(S32);
  B.buildSetFPMode(GetFPMode);
  B.buildResetFPMode();

  auto CheckStr = R"(
  ; CHECK: [[COPY0:%[0-9]+]]:_(i64) = COPY $x0
  ; CHECK: [[COPY1:%[0-9]+]]:_(i64) = COPY $x1
  ; CHECK: [[COPY2:%[0-9]+]]:_(i64) = COPY $x2
  ; CHECK: [[COPY0]]:_(i64) = G_GET_FPENV
  ; CHECK: G_SET_FPENV [[COPY1]]:_(i64)
  ; CHECK: G_RESET_FPENV
  ; CHECK: [[FPMODE:%[0-9]+]]:_(i32) = G_GET_FPMODE
  ; CHECK: G_SET_FPMODE [[FPMODE]]:_(i32)
  ; CHECK: G_RESET_FPMODE
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
}

TEST_F(AArch64GISelMITest, BuildExtractSubvector) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT Vec4x32 = LLT::fixed_vector(4, LLT::integer(32));
  LLT Vec2x32 = LLT::fixed_vector(2, LLT::integer(32));
  LLT Vec2x64 = LLT::fixed_vector(2, LLT::integer(64));
  LLT SVec4x32 = LLT::scalable_vector(4, LLT::integer(32));
  LLT SVec2x32 = LLT::scalable_vector(2, LLT::integer(32));

  // Fixed-length: extract <2 x s32> from <4 x s32> at index 0.
  auto BigVec = B.buildUndef(Vec4x32);
  B.buildExtractSubvector(Vec2x32, BigVec, 0);

  // Scalable: extract <vscale x 2 x s32> from <vscale x 4 x s32> at index 0.
  auto SBigVec = B.buildUndef(SVec4x32);
  B.buildExtractSubvector(SVec2x32, SBigVec, 0);

  // Mixed: extract fixed-length <2 x s32> from scalable <vscale x 4 x s32>.
  B.buildExtractSubvector(Vec2x32, SBigVec, 0);

  auto CheckStr = R"(
  ; CHECK: [[DEF:%[0-9]+]]:_(<4 x i32>) = G_IMPLICIT_DEF
  ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x i32>) = G_EXTRACT_SUBVECTOR [[DEF]]:_(<4 x i32>), 0
  ; CHECK: [[SDEF:%[0-9]+]]:_(<vscale x 4 x i32>) = G_IMPLICIT_DEF
  ; CHECK: [[SEXTRACT:%[0-9]+]]:_(<vscale x 2 x i32>) = G_EXTRACT_SUBVECTOR [[SDEF]]:_(<vscale x 4 x i32>), 0
  ; CHECK: [[MIXEDEXTRACT:%[0-9]+]]:_(<2 x i32>) = G_EXTRACT_SUBVECTOR [[SDEF]]:_(<vscale x 4 x i32>), 0
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;

  // Different element types between result and source.
  EXPECT_DEBUG_DEATH(B.buildExtractSubvector(Vec2x64, BigVec, 0),
                     "Extract subvector VTs must have the same element type!");

  // Cannot extract a scalable subvector from a fixed-length vector.
  EXPECT_DEBUG_DEATH(
      B.buildExtractSubvector(SVec2x32, BigVec, 0),
      "Cannot extract a scalable vector from a fixed length vector!");

  // Result must not be larger than the source.
  EXPECT_DEBUG_DEATH(
      B.buildExtractSubvector(Vec4x32, B.buildUndef(Vec2x32), 0),
      "Extract subvector must be from larger vector to smaller vector!");

  // Extraction index causes overflow (2 + 4 > 4).
  EXPECT_DEBUG_DEATH(B.buildExtractSubvector(Vec2x32, BigVec, 4),
                     "Extract subvector overflow!");

  // Index must be a multiple of the result vector length (1 % 2 != 0).
  EXPECT_DEBUG_DEATH(
      B.buildExtractSubvector(Vec2x32, BigVec, 1),
      "Extract index is not a multiple of the output vector length");
}

TEST_F(AArch64GISelMITest, BuildInsertSubvector) {
  setUp();
  if (!TM)
    GTEST_SKIP();

  LLT Vec4x32 = LLT::fixed_vector(4, LLT::integer(32));
  LLT Vec2x32 = LLT::fixed_vector(2, LLT::integer(32));
  LLT Vec2x64 = LLT::fixed_vector(2, LLT::integer(64));
  LLT SVec4x32 = LLT::scalable_vector(4, LLT::integer(32));
  LLT SVec2x32 = LLT::scalable_vector(2, LLT::integer(32));

  // Fixed-length: insert <2 x s32> into <4 x s32> at index 0 and 2.
  auto BigVec = B.buildUndef(Vec4x32);
  auto SubVec = B.buildUndef(Vec2x32);
  B.buildInsertSubvector(Vec4x32, BigVec, SubVec, 0);
  B.buildInsertSubvector(Vec4x32, BigVec, SubVec, 2);

  // Scalable: insert <vscale x 2 x s32> into <vscale x 4 x s32> at index 0.
  auto SBigVec = B.buildUndef(SVec4x32);
  auto SSubVec = B.buildUndef(SVec2x32);
  B.buildInsertSubvector(SVec4x32, SBigVec, SSubVec, 0);

  // Mixed: insert fixed-length <2 x s32> into scalable <vscale x 4 x s32>.
  B.buildInsertSubvector(SVec4x32, SBigVec, SubVec, 0);

  auto CheckStr = R"(
  ; CHECK: [[DEF0:%[0-9]+]]:_(<4 x i32>) = G_IMPLICIT_DEF
  ; CHECK: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
  ; CHECK: {{%[0-9]+}}:_(<4 x i32>) = G_INSERT_SUBVECTOR [[DEF0]]:_, [[DEF1]]:_(<2 x i32>), 0
  ; CHECK: {{%[0-9]+}}:_(<4 x i32>) = G_INSERT_SUBVECTOR [[DEF0]]:_, [[DEF1]]:_(<2 x i32>), 2
  ; CHECK: [[SDEF0:%[0-9]+]]:_(<vscale x 4 x i32>) = G_IMPLICIT_DEF
  ; CHECK: [[SDEF1:%[0-9]+]]:_(<vscale x 2 x i32>) = G_IMPLICIT_DEF
  ; CHECK: {{%[0-9]+}}:_(<vscale x 4 x i32>) = G_INSERT_SUBVECTOR [[SDEF0]]:_, [[SDEF1]]:_(<vscale x 2 x i32>), 0
  ; CHECK: {{%[0-9]+}}:_(<vscale x 4 x i32>) = G_INSERT_SUBVECTOR [[SDEF0]]:_, [[DEF1]]:_(<2 x i32>), 0
  )";

  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;

  // Dst type must match the BigVec (Src0) type.
  EXPECT_DEBUG_DEATH(B.buildInsertSubvector(Vec2x32, BigVec, SubVec, 0),
                     "Dest and insert subvector source types must match!");

  // Different element types between dst and subvec.
  EXPECT_DEBUG_DEATH(
      B.buildInsertSubvector(Vec4x32, BigVec, B.buildUndef(Vec2x64), 0),
      "Insert subvector VTs must have the same element type!");

  // Cannot insert a scalable subvector into a fixed-length vector.
  EXPECT_DEBUG_DEATH(
      B.buildInsertSubvector(Vec4x32, BigVec, B.buildUndef(SVec2x32), 0),
      "Cannot insert a scalable vector into a fixed length vector!");

  // Subvector must not be larger than the destination.
  EXPECT_DEBUG_DEATH(
      B.buildInsertSubvector(Vec2x32, B.buildUndef(Vec2x32),
                             B.buildUndef(Vec4x32), 0),
      "Insert subvector must be from smaller vector to larger vector!");

  // Insertion index causes overflow (2 + 4 > 4).
  EXPECT_DEBUG_DEATH(B.buildInsertSubvector(Vec4x32, BigVec, SubVec, 4),
                     "Insert subvector overflow!");

  // Index must be a multiple of the subvector length (1 % 2 != 0).
  EXPECT_DEBUG_DEATH(B.buildInsertSubvector(Vec4x32, BigVec, SubVec, 1),
                     "Insert index is not a multiple of the subvector length");
}