blob: 010ffa1502de17426d1a2afcab281e90ef1a5aec [file] [log] [blame]
//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions.
//
//===----------------------------------------------------------------------===//
def SDT_AArch64Setcc : SDTypeProfile<1, 4, [
SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,
SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>,
SDTCisVT<4, OtherVT>
]>;
def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>;
def SVEPatternOperand : AsmOperandClass {
let Name = "SVEPattern";
let ParserMethod = "tryParseSVEPattern";
let PredicateMethod = "isSVEPattern";
let RenderMethod = "addImmOperands";
let DiagnosticType = "InvalidSVEPattern";
}
def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{
return (((uint32_t)Imm) < 32);
}]> {
let PrintMethod = "printSVEPattern";
let ParserMatchClass = SVEPatternOperand;
}
def SVEPrefetchOperand : AsmOperandClass {
let Name = "SVEPrefetch";
let ParserMethod = "tryParsePrefetch<true>";
let PredicateMethod = "isPrefetch";
let RenderMethod = "addPrefetchOperands";
}
def sve_prfop : Operand<i32>, TImmLeaf<i32, [{
return (((uint32_t)Imm) <= 15);
}]> {
let PrintMethod = "printPrefetchOp<true>";
let ParserMatchClass = SVEPrefetchOperand;
}
class SVELogicalImmOperand<int Width> : AsmOperandClass {
let Name = "SVELogicalImm" # Width;
let DiagnosticType = "LogicalSecondSource";
let PredicateMethod = "isLogicalImm<int" # Width # "_t>";
let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";
}
def sve_logical_imm8 : Operand<i64> {
let ParserMatchClass = SVELogicalImmOperand<8>;
let PrintMethod = "printLogicalImm<int8_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val);
}];
}
def sve_logical_imm16 : Operand<i64> {
let ParserMatchClass = SVELogicalImmOperand<16>;
let PrintMethod = "printLogicalImm<int16_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val);
}];
}
def sve_logical_imm32 : Operand<i64> {
let ParserMatchClass = SVELogicalImmOperand<32>;
let PrintMethod = "printLogicalImm<int32_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val);
}];
}
class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass {
let Name = "SVEPreferredLogicalImm" # Width;
let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>";
let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";
}
def sve_preferred_logical_imm16 : Operand<i64> {
let ParserMatchClass = SVEPreferredLogicalImmOperand<16>;
let PrintMethod = "printSVELogicalImm<int16_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) &&
AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
}];
}
def sve_preferred_logical_imm32 : Operand<i64> {
let ParserMatchClass = SVEPreferredLogicalImmOperand<32>;
let PrintMethod = "printSVELogicalImm<int32_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) &&
AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
}];
}
def sve_preferred_logical_imm64 : Operand<i64> {
let ParserMatchClass = SVEPreferredLogicalImmOperand<64>;
let PrintMethod = "printSVELogicalImm<int64_t>";
let MCOperandPredicate = [{
if (!MCOp.isImm())
return false;
int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) &&
AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
}];
}
class SVELogicalImmNotOperand<int Width> : AsmOperandClass {
let Name = "SVELogicalImm" # Width # "Not";
let DiagnosticType = "LogicalSecondSource";
let PredicateMethod = "isLogicalImm<int" # Width # "_t>";
let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>";
}
def sve_logical_imm8_not : Operand<i64> {
let ParserMatchClass = SVELogicalImmNotOperand<8>;
}
def sve_logical_imm16_not : Operand<i64> {
let ParserMatchClass = SVELogicalImmNotOperand<16>;
}
def sve_logical_imm32_not : Operand<i64> {
let ParserMatchClass = SVELogicalImmNotOperand<32>;
}
class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate>
: AsmOperandClass {
let Name = "SVE" # Infix # "Imm" # ElementWidth;
let DiagnosticType = "Invalid" # Name;
let RenderMethod = "addImmWithOptionalShiftOperands<8>";
let ParserMethod = "tryParseImmWithOptionalShift";
let PredicateMethod = Predicate;
}
def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">;
def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">;
def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">;
def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">;
def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">;
def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">;
def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">;
def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">;
class imm8_opt_lsl<int ElementWidth, string printType,
AsmOperandClass OpndClass>
: Operand<i32> {
let EncoderMethod = "getImm8OptLsl";
let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">";
let PrintMethod = "printImm8OptLsl<" # printType # ">";
let ParserMatchClass = OpndClass;
let MIOperandInfo = (ops i32imm, i32imm);
}
def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>;
def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>;
def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>;
def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>;
def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>;
def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>;
def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>;
def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>;
def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>;
def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>;
def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>;
def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>;
def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>;
def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>;
def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>;
def SVELogicalImm8NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8, true>", []>;
def SVELogicalImm16NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16, true>", []>;
def SVELogicalImm32NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32, true>", []>;
def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>;
def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>;
def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>;
def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>;
def SVEArithUImm64Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>;
def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;
def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>;
def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>;
def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>;
def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>;
def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>;
def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>;
def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>;
def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>;
def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>;
class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass {
let Name = "SVEExactFPImmOperand" # Suffix;
let DiagnosticType = "Invalid" # Name;
let ParserMethod = "tryParseFPImm<false>";
let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">";
let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">";
}
class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> {
let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">";
let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>;
}
def sve_fpimm_half_one
: SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half",
"AArch64ExactFPImm::one">;
def sve_fpimm_half_two
: SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half",
"AArch64ExactFPImm::two">;
def sve_fpimm_zero_one
: SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero",
"AArch64ExactFPImm::one">;
def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{
return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
}]> {
let ParserMatchClass = Imm1_16Operand;
let EncoderMethod = "getSVEIncDecImm";
let DecoderMethod = "DecodeSVEIncDecImm";
}
// This allows i32 immediate extraction from i64 based arithmetic.
def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">;
def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">;
def sve_ext_imm_0_31 : ComplexPattern<i32, 1, "SelectEXTImm<31, 8>">;
def sve_ext_imm_0_63 : ComplexPattern<i32, 1, "SelectEXTImm<63, 4>">;
def sve_ext_imm_0_127 : ComplexPattern<i32, 1, "SelectEXTImm<127, 2>">;
def sve_ext_imm_0_255 : ComplexPattern<i32, 1, "SelectEXTImm<255, 1>">;
def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2),
(int_aarch64_sve_cntp node:$pred, node:$src2), [{
return N->hasOneUse();
}]>;
def step_vector_oneuse : PatFrag<(ops node:$idx),
(step_vector node:$idx), [{
return N->hasOneUse();
}]>;
//===----------------------------------------------------------------------===//
// SVE PTrue - These are used extensively throughout the pattern matching so
// it's important we define them first.
//===----------------------------------------------------------------------===//
class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
ValueType vt, SDPatternOperator op>
: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern),
asm, "\t$Pd, $pattern",
"",
[(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> {
bits<4> Pd;
bits<5> pattern;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = sz8_64;
let Inst{21-19} = 0b011;
let Inst{18-17} = opc{2-1};
let Inst{16} = opc{0};
let Inst{15-10} = 0b111000;
let Inst{9-5} = pattern;
let Inst{4} = 0b0;
let Inst{3-0} = Pd;
let Defs = !if(!eq (opc{0}, 1), [NZCV], []);
let ElementSize = pprty.ElementSize;
let isReMaterializable = 1;
}
multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> {
def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>;
def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>;
def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>;
def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>;
def : InstAlias<asm # "\t$Pd",
(!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>;
def : InstAlias<asm # "\t$Pd",
(!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>;
def : InstAlias<asm # "\t$Pd",
(!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>;
def : InstAlias<asm # "\t$Pd",
(!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>;
}
def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>;
def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>;
let Predicates = [HasSVEorStreamingSVE] in {
defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>;
defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>;
}
//===----------------------------------------------------------------------===//
// SVE pattern match helpers.
//===----------------------------------------------------------------------===//
class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
Instruction inst>
: Pat<(vtd (op vt1:$Op1)),
(inst $Op1)>;
class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
ValueType vts, Instruction inst>
: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)),
(inst $Op3, $Op1, $Op2)>;
multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
ValueType vts, Instruction inst> {
def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))),
(inst (IMPLICIT_DEF), $Op1, $Op2)>;
def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)),
(inst $Op3, $Op1, $Op2)>;
}
// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the
// type of rounding. This is matched by timm0_1 in pattern below and ignored.
class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
ValueType vts, Instruction inst>
: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)),
(inst $Op3, $Op1, $Op2)>;
class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
ValueType it, ComplexPattern cpx, Instruction inst>
: Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))),
(inst $Op1, i32:$imm, i32:$shift)>;
class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
ValueType it, ComplexPattern cpx, Instruction inst>
: Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))),
(inst $Op1, i32:$imm, i32:$shift)>;
class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op,
ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst>
: Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))),
(inst $Op1, i32:$imm)>;
class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
ValueType it, ComplexPattern cpx, Instruction inst>
: Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))),
(inst $Op1, i64:$imm)>;
class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, Instruction inst>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
(inst $Op1, $Op2)>;
class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op,
ValueType pt, ValueType vt1, ValueType vt2,
Instruction inst>
: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)),
(inst $Op1, $Op2)>;
class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op,
ValueType pt, ValueType vt1, ValueType vt2,
Instruction inst>
: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)),
(inst $Op1, $Op2, $Op3)>;
class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, Instruction inst>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)),
(inst $Op1, $Op2, $Op3)>;
multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, Instruction inst> {
def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)),
(inst (IMPLICIT_DEF), $Op1, $Op2)>;
def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)),
(inst $Op1, $Op2, $Op3)>;
}
class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, ValueType vt4,
Instruction inst>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)),
(inst $Op1, $Op2, $Op3, $Op4)>;
class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, Operand ImmTy, Instruction inst>
: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))),
(inst $Op1, ImmTy:$Op2)>;
class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, Operand ImmTy,
Instruction inst>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))),
(inst $Op1, $Op2, ImmTy:$Op3)>;
class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, ValueType vt4,
Operand ImmTy, Instruction inst>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),
(inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;
def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>;
def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>;
let AddedComplexity = 1 in {
class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, ValueType vt3, Instruction inst>
: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))),
(inst $Op1, $Op2, $Op3)>;
class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op,
ValueType vt1, ValueType vt2,
Operand vt3, Instruction inst>
: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))),
(inst $Op1, $Op2, vt3:$Op3)>;
}
//
// Common but less generic patterns.
//
class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
Instruction inst, Instruction ptrue>
: Pat<(vtd (op vt1:$Op1)),
(inst (IMPLICIT_DEF), (ptrue 31), $Op1)>;
class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, Instruction inst, Instruction ptrue>
: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
(inst (ptrue 31), $Op1, $Op2)>;
class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt,
ValueType inreg_vt, Instruction inst>
: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)),
(inst $PassThru, $Pg, $Src)>;
multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt,
ValueType inreg_vt, Instruction inst> {
def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))),
(inst (IMPLICIT_DEF), $Pg, $Src)>;
def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)),
(inst $PassThru, $Pg, $Src)>;
}
class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op,
ValueType pt, ValueType it,
ComplexPattern cast, Instruction inst>
: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))),
(inst $Pg, $Rn, i32:$imm)>;
class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op,
ValueType pt, ValueType it,
ComplexPattern cast, Instruction inst>
: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))),
(inst $Rn, i32:$imm)>;
class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op,
ValueType pt, ValueType it,
FPImmLeaf immL, int imm,
Instruction inst>
: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (AArch64dup (it immL))))),
(inst $Pg, $Zs1, imm)>;
class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op,
ValueType pt, ValueType it,
FPImmLeaf immL, int imm,
Instruction inst>
: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)),
(vt (AArch64dup (it immL))))),
(inst $Pg, $Zs1, imm)>;
//
// Pseudo -> Instruction mappings
//
def getSVEPseudoMap : InstrMapping {
let FilterClass = "SVEPseudo2Instr";
let RowFields = ["PseudoName"];
let ColFields = ["IsInstr"];
let KeyCol = ["0"];
let ValueCols = [["1"]];
}
class SVEPseudo2Instr<string name, bit instr> {
string PseudoName = name;
bit IsInstr = instr;
}
// Lookup e.g. DIV -> DIVR
def getSVERevInstr : InstrMapping {
let FilterClass = "SVEInstr2Rev";
let RowFields = ["InstrName"];
let ColFields = ["isReverseInstr"];
let KeyCol = ["0"];
let ValueCols = [["1"]];
}
// Lookup e.g. DIVR -> DIV
def getSVENonRevInstr : InstrMapping {
let FilterClass = "SVEInstr2Rev";
let RowFields = ["InstrName"];
let ColFields = ["isReverseInstr"];
let KeyCol = ["1"];
let ValueCols = [["0"]];
}
class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> {
string InstrName = !if(name1IsReverseInstr, name1, name2);
bit isReverseInstr = name1IsReverseInstr;
}
//
// Pseudos for destructive operands
//
let hasNoSchedulingInfo = 1 in {
class PredTwoOpPseudo<string name, ZPRRegOp zprty,
FalseLanesEnum flags = FalseLanesNone>
: SVEPseudo2Instr<name, 0>,
Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> {
let FalseLanes = flags;
}
class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty,
FalseLanesEnum flags = FalseLanesNone>
: SVEPseudo2Instr<name, 0>,
Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> {
let FalseLanes = flags;
}
class PredThreeOpPseudo<string name, ZPRRegOp zprty,
FalseLanesEnum flags = FalseLanesNone>
: SVEPseudo2Instr<name, 0>,
Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> {
let FalseLanes = flags;
}
}
//
// Pseudos for passthru operands
//
let hasNoSchedulingInfo = 1 in {
class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty>
: SVEPseudo2Instr<name, 0>,
Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>;
}
//===----------------------------------------------------------------------===//
// SVE Predicate Misc Group
//===----------------------------------------------------------------------===//
class sve_int_pfalse<bits<6> opc, string asm>
: I<(outs PPR8:$Pd), (ins),
asm, "\t$Pd",
"",
[]>, Sched<[]> {
bits<4> Pd;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = opc{5-4};
let Inst{21-19} = 0b011;
let Inst{18-16} = opc{3-1};
let Inst{15-10} = 0b111001;
let Inst{9} = opc{0};
let Inst{8-4} = 0b00000;
let Inst{3-0} = Pd;
let isReMaterializable = 1;
}
class sve_int_ptest<bits<6> opc, string asm>
: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn),
asm, "\t$Pg, $Pn",
"",
[]>, Sched<[]> {
bits<4> Pg;
bits<4> Pn;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = opc{5-4};
let Inst{21-19} = 0b010;
let Inst{18-16} = opc{3-1};
let Inst{15-14} = 0b11;
let Inst{13-10} = Pg;
let Inst{9} = opc{0};
let Inst{8-5} = Pn;
let Inst{4-0} = 0b00000;
let Defs = [NZCV];
let isCompare = 1;
}
class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm,
PPRRegOp pprty>
: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn),
asm, "\t$Pdn, $Pg, $_Pdn",
"",
[]>, Sched<[]> {
bits<4> Pdn;
bits<4> Pg;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = sz8_64;
let Inst{21-19} = 0b011;
let Inst{18-16} = opc{4-2};
let Inst{15-11} = 0b11000;
let Inst{10-9} = opc{1-0};
let Inst{8-5} = Pg;
let Inst{4} = 0;
let Inst{3-0} = Pdn;
let Constraints = "$Pdn = $_Pdn";
let Defs = [NZCV];
let isPTestLike = 1;
let ElementSize = pprty.ElementSize;
}
multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> {
def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>;
def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
}
multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> {
def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>;
def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>;
def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>;
def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>;
def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;
}
//===----------------------------------------------------------------------===//
// SVE Predicate Count Group
//===----------------------------------------------------------------------===//
class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm,
RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty>
: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn),
asm, "\t$Rdn, $Pg",
"",
[]>, Sched<[]> {
bits<5> Rdn;
bits<4> Pg;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = sz8_64;
let Inst{21-19} = 0b101;
let Inst{18-16} = opc{4-2};
let Inst{15-11} = 0b10001;
let Inst{10-9} = opc{1-0};
let Inst{8-5} = Pg;
let Inst{4-0} = Rdn;
// Signed 32bit forms require their GPR operand printed.
let AsmString = !if(!eq(opc{4,2-0}, 0b0000),
!strconcat(asm, "\t$Rdn, $Pg, $_Rdn"),
!strconcat(asm, "\t$Rdn, $Pg"));
let Constraints = "$Rdn = $_Rdn";
}
multiclass sve_int_count_r_s32<bits<5> opc, string asm,
SDPatternOperator op> {
def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>;
def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>;
def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>;
def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>;
def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),
(EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))),
(!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),
(EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))),
(!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),
(EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))),
(!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),
(EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))),
(!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
}
multiclass sve_int_count_r_u32<bits<5> opc, string asm,
SDPatternOperator op> {
def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>;
def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>;
def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>;
def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>;
def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;
def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;
def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;
def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;
}
multiclass sve_int_count_r_x64<bits<5> opc, string asm,
SDPatternOperator op,
SDPatternOperator combine_op = null_frag> {
def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>;
def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>;
def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>;
def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>;
def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;
def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;
def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;
def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))),
(!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;
// Combine cntp with combine_op
def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))),
(!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>;
def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))),
(!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>;
def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))),
(!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>;
def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))),
(!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>;
}
class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm,
ZPRRegOp zprty, PPRRegOp pprty>
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm),
asm, "\t$Zdn, $Pm",
"",
[]>, Sched<[]> {
bits<4> Pm;
bits<5> Zdn;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = sz8_64;
let Inst{21-19} = 0b101;
let Inst{18-16} = opc{4-2};
let Inst{15-11} = 0b10000;
let Inst{10-9} = opc{1-0};
let Inst{8-5} = Pm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = ElementSizeNone;
}
multiclass sve_int_count_v<bits<5> opc, string asm,
SDPatternOperator op = null_frag> {
def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>;
def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>;
def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>;
def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>;
def : InstAlias<asm # "\t$Zdn, $Pm",
(!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>;
def : InstAlias<asm # "\t$Zdn, $Pm",
(!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>;
def : InstAlias<asm # "\t$Zdn, $Pm",
(!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>;
}
class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm,
PPRRegOp pprty>
: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn),
asm, "\t$Rd, $Pg, $Pn",
"",
[]>, Sched<[]> {
bits<4> Pg;
bits<4> Pn;
bits<5> Rd;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = sz8_64;
let Inst{21-19} = 0b100;
let Inst{18-16} = opc{3-1};
let Inst{15-14} = 0b10;
let Inst{13-10} = Pg;
let Inst{9} = opc{0};
let Inst{8-5} = Pn;
let Inst{4-0} = Rd;
}
multiclass sve_int_pcount_pred<bits<4> opc, string asm,
SDPatternOperator int_op> {
def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;
def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;
def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;
def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;
def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;
}
//===----------------------------------------------------------------------===//
// SVE Element Count Group
//===----------------------------------------------------------------------===//
class sve_int_count<bits<3> opc, string asm>
: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
asm, "\t$Rd, $pattern, mul $imm4",
"",
[]>, Sched<[]> {
bits<5> Rd;
bits<4> imm4;
bits<5> pattern;
let Inst{31-24} = 0b00000100;
let Inst{23-22} = opc{2-1};
let Inst{21-20} = 0b10;
let Inst{19-16} = imm4;
let Inst{15-11} = 0b11100;
let Inst{10} = opc{0};
let Inst{9-5} = pattern;
let Inst{4-0} = Rd;
}
multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> {
def NAME : sve_int_count<opc, asm>;
def : InstAlias<asm # "\t$Rd, $pattern",
(!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Rd",
(!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>;
def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))),
(!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))),
(!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
def : Pat<(i64 (op sve_pred_enum:$pattern)),
(!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>;
}
class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
asm, "\t$Zdn, $pattern, mul $imm4",
"",
[]>, Sched<[]> {
bits<5> Zdn;
bits<5> pattern;
bits<4> imm4;
let Inst{31-24} = 0b00000100;
let Inst{23-22} = opc{4-3};
let Inst{21} = 0b1;
let Inst{20} = opc{2};
let Inst{19-16} = imm4;
let Inst{15-12} = 0b1100;
let Inst{11-10} = opc{1-0};
let Inst{9-5} = pattern;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = ElementSizeNone;
}
multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty,
SDPatternOperator op = null_frag,
ValueType vt = OtherVT> {
def NAME : sve_int_countvlv<opc, asm, zprty>;
def : InstAlias<asm # "\t$Zdn, $pattern",
(!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Zdn",
(!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>;
def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
(!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
}
class sve_int_pred_pattern_a<bits<3> opc, string asm>
: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
asm, "\t$Rdn, $pattern, mul $imm4",
"",
[]>, Sched<[]> {
bits<5> Rdn;
bits<5> pattern;
bits<4> imm4;
let Inst{31-24} = 0b00000100;
let Inst{23-22} = opc{2-1};
let Inst{21-20} = 0b11;
let Inst{19-16} = imm4;
let Inst{15-11} = 0b11100;
let Inst{10} = opc{0};
let Inst{9-5} = pattern;
let Inst{4-0} = Rdn;
let Constraints = "$Rdn = $_Rdn";
}
multiclass sve_int_pred_pattern_a<bits<3> opc, string asm,
SDPatternOperator op,
SDPatternOperator opcnt> {
let Predicates = [HasSVEorStreamingSVE] in {
def NAME : sve_int_pred_pattern_a<opc, asm>;
def : InstAlias<asm # "\t$Rdn, $pattern",
(!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Rdn",
(!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>;
}
let Predicates = [HasSVEorStreamingSVE, UseScalarIncVL] in {
def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))),
(!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>;
def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm)))),
(!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;
def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm))))),
(!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>;
def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))),
(i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1),
sub_32))>;
def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm i32:$imm)))),
(i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),
sub_32))>;
def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (i64 (sve_cnt_shl_imm i32:$imm))))),
(i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm),
sub_32))>;
}
}
class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt,
RegisterOperand st>
: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
asm, "\t$Rdn, $pattern, mul $imm4",
"",
[]>, Sched<[]> {
bits<5> Rdn;
bits<5> pattern;
bits<4> imm4;
let Inst{31-24} = 0b00000100;
let Inst{23-22} = opc{4-3};
let Inst{21} = 0b1;
let Inst{20} = opc{2};
let Inst{19-16} = imm4;
let Inst{15-12} = 0b1111;
let Inst{11-10} = opc{1-0};
let Inst{9-5} = pattern;
let Inst{4-0} = Rdn;
// Signed 32bit forms require their GPR operand printed.
let AsmString = !if(!eq(opc{2,0}, 0b00),
!strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"),
!strconcat(asm, "\t$Rdn, $pattern, mul $imm4"));
let Constraints = "$Rdn = $_Rdn";
}
multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm,
SDPatternOperator op> {
def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>;
def : InstAlias<asm # "\t$Rd, $Rn, $pattern",
(!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Rd, $Rn",
(!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>;
// NOTE: Register allocation doesn't like tied operands of differing register
// class, hence the extra INSERT_SUBREG complication.
def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
(EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>;
def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))),
(!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
}
multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm,
SDPatternOperator op> {
def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>;
def : InstAlias<asm # "\t$Rdn, $pattern",
(!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Rdn",
(!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>;
def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
(!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
}
multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm,
SDPatternOperator op> {
def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>;
def : InstAlias<asm # "\t$Rdn, $pattern",
(!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>;
def : InstAlias<asm # "\t$Rdn",
(!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>;
def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
(!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
}
//===----------------------------------------------------------------------===//
// SVE Permute - Cross Lane Group
//===----------------------------------------------------------------------===//
class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
ValueType vt, RegisterClass srcRegType,
SDPatternOperator op>
: I<(outs zprty:$Zd), (ins srcRegType:$Rn),
asm, "\t$Zd, $Rn",
"",
[(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> {
bits<5> Rn;
bits<5> Zd;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21-10} = 0b100000001110;
let Inst{9-5} = Rn;
let Inst{4-0} = Zd;
}
multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> {
def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>;
def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>;
def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>;
def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>;
def : InstAlias<"mov $Zd, $Rn",
(!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>;
def : InstAlias<"mov $Zd, $Rn",
(!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>;
def : InstAlias<"mov $Zd, $Rn",
(!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>;
def : InstAlias<"mov $Zd, $Rn",
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>;
}
class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm,
ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx),
asm, "\t$Zd, $Zn$idx",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<5> Zn;
bits<7> idx;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = {?,?}; // imm3h
let Inst{21} = 0b1;
let Inst{20-16} = tsz;
let Inst{15-10} = 0b001000;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_perm_dup_i<string asm> {
def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> {
let Inst{23-22} = idx{5-4};
let Inst{20-17} = idx{3-0};
}
def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> {
let Inst{23-22} = idx{4-3};
let Inst{20-18} = idx{2-0};
}
def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> {
let Inst{23-22} = idx{3-2};
let Inst{20-19} = idx{1-0};
}
def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> {
let Inst{23-22} = idx{2-1};
let Inst{20} = idx{0};
}
def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> {
let Inst{23-22} = idx{1-0};
}
def : InstAlias<"mov $Zd, $Zn$idx",
(!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>;
def : InstAlias<"mov $Zd, $Zn$idx",
(!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>;
def : InstAlias<"mov $Zd, $Zn$idx",
(!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>;
def : InstAlias<"mov $Zd, $Zn$idx",
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>;
def : InstAlias<"mov $Zd, $Zn$idx",
(!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>;
def : InstAlias<"mov $Zd, $Bn",
(!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>;
def : InstAlias<"mov $Zd, $Hn",
(!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>;
def : InstAlias<"mov $Zd, $Sn",
(!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>;
def : InstAlias<"mov $Zd, $Dn",
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>;
def : InstAlias<"mov $Zd, $Qn",
(!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>;
// Duplicate extracted element of vector into all vector elements
def : Pat<(nxv16i8 (AArch64dup (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))),
(!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>;
def : Pat<(nxv8i16 (AArch64dup (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv4i32 (AArch64dup (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2i64 (AArch64dup (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv8f16 (AArch64dup (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv8bf16 (AArch64dup (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv4f16 (AArch64dup (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2f16 (AArch64dup (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv4f32 (AArch64dup (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2f32 (AArch64dup (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv2f64 (AArch64dup (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
}
class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty,
RegisterOperand VecList>
: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm),
asm, "\t$Zd, $Zn, $Zm",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21} = 0b1;
let Inst{20-16} = Zm;
let Inst{15-13} = 0b001;
let Inst{12-11} = opc;
let Inst{10} = 0b0;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> {
def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>;
def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>;
def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>;
def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>;
def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
(!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>;
def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
(!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>;
def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
(!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>;
def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>;
def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
}
multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> {
def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>;
def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>;
def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>;
def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>;
def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)),
(nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0,
nxv16i8:$Op2, zsub1),
nxv16i8:$Op3))>;
def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)),
(nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0,
nxv8i16:$Op2, zsub1),
nxv8i16:$Op3))>;
def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)),
(nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0,
nxv4i32:$Op2, zsub1),
nxv4i32:$Op3))>;
def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)),
(nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0,
nxv2i64:$Op2, zsub1),
nxv2i64:$Op3))>;
def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)),
(nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0,
nxv8f16:$Op2, zsub1),
nxv8i16:$Op3))>;
def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)),
(nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0,
nxv4f32:$Op2, zsub1),
nxv4i32:$Op3))>;
def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)),
(nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0,
nxv2f64:$Op2, zsub1),
nxv2i64:$Op3))>;
def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)),
(nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0,
nxv8bf16:$Op2, zsub1),
nxv8i16:$Op3))>;
}
class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm),
asm, "\t$Zd, $Zn, $Zm",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21} = 0b1;
let Inst{20-16} = Zm;
let Inst{15-10} = 0b001011;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
let Constraints = "$Zd = $_Zd";
}
multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> {
def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>;
def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>;
def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>;
def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>;
def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
}
class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins zprty:$Zn),
asm, "\t$Zd, $Zn",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<5> Zn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21-10} = 0b111000001110;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> {
def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>;
def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>;
def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>;
def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>;
def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>;
def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>;
def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;
def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>;
def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;
def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>;
def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>;
}
class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty>
: I<(outs pprty:$Pd), (ins pprty:$Pn),
asm, "\t$Pd, $Pn",
"",
[]>, Sched<[]> {
bits<4> Pd;
bits<4> Pn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21-9} = 0b1101000100000;
let Inst{8-5} = Pn;
let Inst{4} = 0b0;
let Inst{3-0} = Pd;
}
multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> {
def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>;
def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>;
def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>;
def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>;
def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>;
def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>;
def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>;
}
class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm,
ZPRRegOp zprty1, ZPRRegOp zprty2>
: I<(outs zprty1:$Zd), (ins zprty2:$Zn),
asm, "\t$Zd, $Zn",
"", []>, Sched<[]> {
bits<5> Zd;
bits<5> Zn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz16_64;
let Inst{21-18} = 0b1100;
let Inst{17-16} = opc;
let Inst{15-10} = 0b001110;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> {
def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>;
def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>;
def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>;
def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>;
def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>;
def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>;
}
class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty,
RegisterClass srcRegType>
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm),
asm, "\t$Zdn, $Rm",
"",
[]>, Sched<[]> {
bits<5> Rm;
bits<5> Zdn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21-10} = 0b100100001110;
let Inst{9-5} = Rm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
}
multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> {
def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>;
def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>;
def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>;
def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>;
def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>;
def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>;
}
class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty,
FPRasZPROperand srcOpType>
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm),
asm, "\t$Zdn, $Vm",
"",
[]>, Sched<[]> {
bits<5> Vm;
bits<5> Zdn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21-10} = 0b110100001110;
let Inst{9-5} = Vm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
}
multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> {
def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>;
def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>;
def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>;
def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>;
def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)),
(!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>;
def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)),
(!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>;
def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)),
(!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>;
def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)),
(!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>;
// Keep integer insertions within the vector unit.
def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))),
(!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>;
def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))),
(!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>;
def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))),
(!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>;
def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))),
(!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>;
}
//===----------------------------------------------------------------------===//
// SVE Permute - Extract Group
//===----------------------------------------------------------------------===//
class sve_int_perm_extract_i<string asm>
: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8),
asm, "\t$Zdn, $_Zdn, $Zm, $imm8",
"", []>, Sched<[]> {
bits<5> Zdn;
bits<5> Zm;
bits<8> imm8;
let Inst{31-21} = 0b00000101001;
let Inst{20-16} = imm8{7-3};
let Inst{15-13} = 0b000;
let Inst{12-10} = imm8{2-0};
let Inst{9-5} = Zm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = ElementSizeNone;
}
multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> {
def NAME : sve_int_perm_extract_i<asm>;
def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255,
!cast<Instruction>(NAME)>;
}
class sve2_int_perm_extract_i_cons<string asm>
: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8),
asm, "\t$Zd, $Zn, $imm8",
"", []>, Sched<[]> {
bits<5> Zd;
bits<5> Zn;
bits<8> imm8;
let Inst{31-21} = 0b00000101011;
let Inst{20-16} = imm8{7-3};
let Inst{15-13} = 0b000;
let Inst{12-10} = imm8{2-0};
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
//===----------------------------------------------------------------------===//
// SVE Vector Select Group
//===----------------------------------------------------------------------===//
class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm),
asm, "\t$Zd, $Pg, $Zn, $Zm",
"",
[]>, Sched<[]> {
bits<4> Pg;
bits<5> Zd;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = sz8_64;
let Inst{21} = 0b1;
let Inst{20-16} = Zm;
let Inst{15-14} = 0b11;
let Inst{13-10} = Pg;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> {
def _B : sve_int_sel_vvv<0b00, asm, ZPR8>;
def _H : sve_int_sel_vvv<0b01, asm, ZPR16>;
def _S : sve_int_sel_vvv<0b10, asm, ZPR32>;
def _D : sve_int_sel_vvv<0b11, asm, ZPR64>;
def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
def : InstAlias<"mov $Zd, $Pg/m, $Zn",
(!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>;
def : InstAlias<"mov $Zd, $Pg/m, $Zn",
(!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>;
def : InstAlias<"mov $Zd, $Pg/m, $Zn",
(!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>;
def : InstAlias<"mov $Zd, $Pg/m, $Zn",
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>;
}
//===----------------------------------------------------------------------===//
// SVE Predicate Logical Operations Group
//===----------------------------------------------------------------------===//
class sve_int_pred_log<bits<4> opc, string asm>
: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),
asm, "\t$Pd, $Pg/z, $Pn, $Pm",
"",
[]>, Sched<[]> {
bits<4> Pd;
bits<4> Pg;
bits<4> Pm;
bits<4> Pn;
let Inst{31-24} = 0b00100101;
let Inst{23-22} = opc{3-2};
let Inst{21-20} = 0b00;
let Inst{19-16} = Pm;
let Inst{15-14} = 0b01;
let Inst{13-10} = Pg;
let Inst{9} = opc{1};
let Inst{8-5} = Pn;
let Inst{4} = opc{0};
let Inst{3-0} = Pd;
// SEL has no predication qualifier.
let AsmString = !if(!eq(opc, 0b0011),
!strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"),
!strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm"));
let Defs = !if(!eq (opc{2}, 1), [NZCV], []);
}
multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,
SDPatternOperator op_nopred = null_frag> {
def NAME : sve_int_pred_log<opc, asm>;
def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>;
def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>;
def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>;
def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1,
!cast<Instruction>(NAME), PTRUE_B>;
def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1,
!cast<Instruction>(NAME), PTRUE_H>;
def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1,
!cast<Instruction>(NAME), PTRUE_S>;
def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1,
!cast<Instruction>(NAME), PTRUE_D>;
}
//===----------------------------------------------------------------------===//
// SVE Logical Mask Immediate Group
//===----------------------------------------------------------------------===//
class sve_int_log_imm<bits<2> opc, string asm>
: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),
asm, "\t$Zdn, $_Zdn, $imms13",
"", []>, Sched<[]> {
bits<5> Zdn;
bits<13> imms13;
let Inst{31-24} = 0b00000101;
let Inst{23-22} = opc;
let Inst{21-18} = 0b0000;
let Inst{17-5} = imms13;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DecoderMethod = "DecodeSVELogicalImmInstruction";
let DestructiveInstType = DestructiveOther;
let ElementSize = ElementSizeNone;
}
multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> {
def NAME : sve_int_log_imm<opc, asm>;
def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>;
def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>;
def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>;
def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>;
def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>;
def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>;
def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>;
def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>;
def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>;
def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>;
def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>;
}
multiclass sve_int_log_imm_bic<SDPatternOperator op> {
def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>;
def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>;
def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>;
def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>;
}
class sve_int_dup_mask_imm<string asm>
: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms),
asm, "\t$Zd, $imms",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<13> imms;
let Inst{31-18} = 0b00000101110000;
let Inst{17-5} = imms;
let Inst{4-0} = Zd;
let isReMaterializable = 1;
let DecoderMethod = "DecodeSVELogicalImmInstruction";
}
multiclass sve_int_dup_mask_imm<string asm> {
def NAME : sve_int_dup_mask_imm<asm>;
def : InstAlias<"dupm $Zd, $imm",
(!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>;
def : InstAlias<"dupm $Zd, $imm",
(!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>;
def : InstAlias<"dupm $Zd, $imm",
(!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>;
// All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here.
def : InstAlias<"mov $Zd, $imm",
(!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>;
def : InstAlias<"mov $Zd, $imm",
(!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>;
def : InstAlias<"mov $Zd, $imm",
(!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>;
}
//===----------------------------------------------------------------------===//
// SVE Integer Arithmetic - Unpredicated Group.
//===----------------------------------------------------------------------===//
class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm,
ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
asm, "\t$Zd, $Zn, $Zm",
"", []>, Sched<[]> {
bits<5> Zd;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b00000100;
let Inst{23-22} = sz8_64;
let Inst{21} = 0b1;
let Inst{20-16} = Zm;
let Inst{15-13} = 0b000;
let Inst{12-10} = opc;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> {
def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>;
def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>;
def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>;
def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>;
def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
}
//===----------------------------------------------------------------------===//
// SVE Floating Point Arithmetic - Predicated Group
//===----------------------------------------------------------------------===//
class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm,
ZPRRegOp zprty,
Operand imm_ty>
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1),
asm, "\t$Zdn, $Pg/m, $_Zdn, $i1",
"",
[]>, Sched<[]> {
bits<3> Pg;
bits<5> Zdn;
bit i1;
let Inst{31-24} = 0b01100101;
let Inst{23-22} = sz;
let Inst{21-19} = 0b011;
let Inst{18-16} = opc;
let Inst{15-13} = 0b100;
let Inst{12-10} = Pg;
let Inst{9-6} = 0b0000;
let Inst{5} = i1;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = zprty.ElementSize;
}
multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> {
let DestructiveInstType = DestructiveBinaryImm in {
def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>;
def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>;
def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>;
}
def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>;
}
class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm,
ZPRRegOp zprty>
: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",
"",
[]>, Sched<[]> {
bits<3> Pg;
bits<5> Zdn;
bits<5> Zm;
let Inst{31-24} = 0b01100101;
let Inst{23-22} = sz;
let Inst{21-20} = 0b00;
let Inst{19-16} = opc;
let Inst{15-13} = 0b100;
let Inst{12-10} = Pg;
let Inst{9-5} = Zm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = zprty.ElementSize;
}
multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps,
SDPatternOperator op, DestructiveInstTypeEnum flags,
string revname="", bit isReverseInstr=0> {
let DestructiveInstType = flags in {
def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>,
SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>,
SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>,
SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
}
def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
}
multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm,
SDPatternOperator op> {
def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>;
def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>;
def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>;
def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
}
multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> {
def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;
def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>;
def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>;
def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>;
}
class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3),
asm, "\t$Zdn, $_Zdn, $Zm, $imm3",
"",
[]>, Sched<[]> {
bits<5> Zdn;
bits<5> Zm;
bits<3> imm3;
let Inst{31-24} = 0b01100101;
let Inst{23-22} = sz;
let Inst{21-19} = 0b010;
let Inst{18-16} = imm3;
let Inst{15-10} = 0b100000;
let Inst{9-5} = Zm;
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
let DestructiveInstType = DestructiveOther;
let ElementSize = ElementSizeNone;
}
multiclass sve_fp_ftmad<string asm, SDPatternOperator op> {
def _H : sve_fp_ftmad<0b01, asm, ZPR16>;
def _S : sve_fp_ftmad<0b10, asm, ZPR32>;
def _D : sve_fp_ftmad<0b11, asm, ZPR64>;
def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))),
(!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>;
def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))),
(!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>;
def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))),
(!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>;
}
multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> {
def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>;
def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>;
def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>;
def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>;
def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>;
}
multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> {
def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>;
def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>;
def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>;
let AddedComplexity = 2 in {
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>;
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>;
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>;
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>;
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>;
def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>;
}
}
//===----------------------------------------------------------------------===//
// SVE Floating Point Arithmetic - Unpredicated Group
//===----------------------------------------------------------------------===//
class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
asm, "\t$Zd, $Zn, $Zm",
"",
[]>, Sched<[]> {
bits<5> Zd;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b01100101;
let Inst{23-22} = sz;
let Inst{21} = 0b0;
let Inst{20-16} = Zm;
let Inst{15-13} = 0b000;
let Inst{12-10} = opc;
let Inst{9-5} = Zn;
let Inst{4-0} = Zd;
}
multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op,
SDPatternOperator predicated_op = null_frag> {
def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;
def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;
def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;
def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
}
multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> {
def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;
def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;
def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;
def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
}
//===----------------------------------------------------------------------===//
// SVE Floating Point Fused Multiply-Add Group
//===----------------------------------------------------------------------===//
class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty>
: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),
asm, "\t$Zda, $Pg/m, $Zn, $Zm",
"",
[]>, Sched<[]> {
bits<3> Pg;
bits<5> Zda;
bits<5> Zm;
bits<5> Zn;
let Inst{31-24} = 0b01100101;
let Inst{23-22} = sz;
let Inst{21} = 0b1;
let Inst{20-16} = Zm;
let Inst{15} = 0b0;
let Inst{14-13} = opc;
let Inst{12-10} = Pg;
let Inst{9-5} = Zn;
let Inst{4-0} = Zda;
let Constraints = "$Zda = $_Zda";
let ElementSize = zprty.ElementSize;
}
multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps,
SDPatternOperator op, string revname,
bit isReverseInstr=0> {
let DestructiveInstType = DestructiveTernaryCommWithRev in {