Google Git
Sign in
llvm / llvm-project / 37d72991c136244fe53be02f5af0ad643247a1e6 / . / llvm / lib / Target / ARM / ARMInstrMVE.td
blob: f53814a80e01e551b34f4629ffd3cc54082e3905 [file] [log] [blame]
//===-- ARMInstrMVE.td - MVE support for ARM ---------------*- tablegen -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file describes the ARM MVE instruction set.
//
//===----------------------------------------------------------------------===//
// VPT condition mask
def vpt_mask : Operand<i32> {
let PrintMethod = "printVPTMask";
let ParserMatchClass = it_mask_asmoperand;
let EncoderMethod = "getVPTMaskOpValue";
let DecoderMethod = "DecodeVPTMaskOperand";
}
// VPT/VCMP restricted predicate for sign invariant types
def pred_restricted_i_asmoperand : AsmOperandClass {
let Name = "CondCodeRestrictedI";
let RenderMethod = "addITCondCodeOperands";
let PredicateMethod = "isITCondCodeRestrictedI";
let ParserMethod = "parseITCondCode";
let DiagnosticString = "condition code for sign-independent integer "#
"comparison must be EQ or NE";
}
// VPT/VCMP restricted predicate for signed types
def pred_restricted_s_asmoperand : AsmOperandClass {
let Name = "CondCodeRestrictedS";
let RenderMethod = "addITCondCodeOperands";
let PredicateMethod = "isITCondCodeRestrictedS";
let ParserMethod = "parseITCondCode";
let DiagnosticString = "condition code for signed integer "#
"comparison must be EQ, NE, LT, GT, LE or GE";
}
// VPT/VCMP restricted predicate for unsigned types
def pred_restricted_u_asmoperand : AsmOperandClass {
let Name = "CondCodeRestrictedU";
let RenderMethod = "addITCondCodeOperands";
let PredicateMethod = "isITCondCodeRestrictedU";
let ParserMethod = "parseITCondCode";
let DiagnosticString = "condition code for unsigned integer "#
"comparison must be EQ, NE, HS or HI";
}
// VPT/VCMP restricted predicate for floating point
def pred_restricted_fp_asmoperand : AsmOperandClass {
let Name = "CondCodeRestrictedFP";
let RenderMethod = "addITCondCodeOperands";
let PredicateMethod = "isITCondCodeRestrictedFP";
let ParserMethod = "parseITCondCode";
let DiagnosticString = "condition code for floating-point "#
"comparison must be EQ, NE, LT, GT, LE or GE";
}
class VCMPPredicateOperand : Operand<i32>;
def pred_basic_i : VCMPPredicateOperand {
let PrintMethod = "printMandatoryRestrictedPredicateOperand";
let ParserMatchClass = pred_restricted_i_asmoperand;
let DecoderMethod = "DecodeRestrictedIPredicateOperand";
let EncoderMethod = "getRestrictedCondCodeOpValue";
}
def pred_basic_u : VCMPPredicateOperand {
let PrintMethod = "printMandatoryRestrictedPredicateOperand";
let ParserMatchClass = pred_restricted_u_asmoperand;
let DecoderMethod = "DecodeRestrictedUPredicateOperand";
let EncoderMethod = "getRestrictedCondCodeOpValue";
}
def pred_basic_s : VCMPPredicateOperand {
let PrintMethod = "printMandatoryRestrictedPredicateOperand";
let ParserMatchClass = pred_restricted_s_asmoperand;
let DecoderMethod = "DecodeRestrictedSPredicateOperand";
let EncoderMethod = "getRestrictedCondCodeOpValue";
}
def pred_basic_fp : VCMPPredicateOperand {
let PrintMethod = "printMandatoryRestrictedPredicateOperand";
let ParserMatchClass = pred_restricted_fp_asmoperand;
let DecoderMethod = "DecodeRestrictedFPPredicateOperand";
let EncoderMethod = "getRestrictedCondCodeOpValue";
}
// Register list operands for interleaving load/stores
def VecList2QAsmOperand : AsmOperandClass {
let Name = "VecListTwoMQ";
let ParserMethod = "parseVectorList";
let RenderMethod = "addMVEVecListOperands";
let DiagnosticString = "operand must be a list of two consecutive "#
"q-registers in range [q0,q7]";
}
def VecList2Q : RegisterOperand<MQQPR, "printMVEVectorListTwoQ"> {
let ParserMatchClass = VecList2QAsmOperand;
let PrintMethod = "printMVEVectorList<2>";
}
def VecList4QAsmOperand : AsmOperandClass {
let Name = "VecListFourMQ";
let ParserMethod = "parseVectorList";
let RenderMethod = "addMVEVecListOperands";
let DiagnosticString = "operand must be a list of four consecutive "#
"q-registers in range [q0,q7]";
}
def VecList4Q : RegisterOperand<MQQQQPR, "printMVEVectorListFourQ"> {
let ParserMatchClass = VecList4QAsmOperand;
let PrintMethod = "printMVEVectorList<4>";
}
// taddrmode_imm7 := reg[r0-r7] +/- (imm7 << shift)
class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
let Name = "TMemImm7Shift"#shift#"Offset";
let PredicateMethod = "isMemImm7ShiftedOffset<"#shift#",ARM::tGPRRegClassID>";
let RenderMethod = "addMemImmOffsetOperands";
}
class taddrmode_imm7<int shift> : MemOperand,
ComplexPattern<i32, 2, "SelectTAddrModeImm7<"#shift#">", []> {
let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;
// They are printed the same way as the T2 imm8 version
let PrintMethod = "printT2AddrModeImm8Operand<false>";
// This can also be the same as the T2 version.
let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
let DecoderMethod = "DecodeTAddrModeImm7<"#shift#">";
let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
}
// t2addrmode_imm7 := reg +/- (imm7)
class MemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
let Name = "MemImm7Shift"#shift#"Offset";
let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
",ARM::GPRnopcRegClassID>";
let RenderMethod = "addMemImmOffsetOperands";
}
def MemImm7Shift0OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<0>;
def MemImm7Shift1OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<1>;
def MemImm7Shift2OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<2>;
class T2AddrMode_Imm7<int shift> : MemOperand,
ComplexPattern<i32, 2, "SelectT2AddrModeImm7<"#shift#">", []> {
let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 0>";
let ParserMatchClass =
!cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetAsmOperand");
let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);
}
class t2addrmode_imm7<int shift> : T2AddrMode_Imm7<shift> {
// They are printed the same way as the imm8 version
let PrintMethod = "printT2AddrModeImm8Operand<false>";
}
class MemImm7ShiftOffsetWBAsmOperand<int shift> : AsmOperandClass {
let Name = "MemImm7Shift"#shift#"OffsetWB";
let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
",ARM::rGPRRegClassID>";
let RenderMethod = "addMemImmOffsetOperands";
}
def MemImm7Shift0OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<0>;
def MemImm7Shift1OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<1>;
def MemImm7Shift2OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<2>;
class t2addrmode_imm7_pre<int shift> : T2AddrMode_Imm7<shift> {
// They are printed the same way as the imm8 version
let PrintMethod = "printT2AddrModeImm8Operand<true>";
let ParserMatchClass =
!cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetWBAsmOperand");
let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 1>";
let MIOperandInfo = (ops rGPR:$base, i32imm:$offsim);
}
class t2am_imm7shiftOffsetAsmOperand<int shift>
: AsmOperandClass { let Name = "Imm7Shift"#shift; }
def t2am_imm7shift0OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<0>;
def t2am_imm7shift1OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<1>;
def t2am_imm7shift2OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<2>;
class t2am_imm7_offset<int shift> : MemOperand,
ComplexPattern<i32, 1, "SelectT2AddrModeImm7Offset<"#shift#">",
[], [SDNPWantRoot]> {
// They are printed the same way as the imm8 version
let PrintMethod = "printT2AddrModeImm8OffsetOperand";
let ParserMatchClass =
!cast<AsmOperandClass>("t2am_imm7shift"#shift#"OffsetAsmOperand");
let EncoderMethod = "getT2ScaledImmOpValue<7,"#shift#">";
let DecoderMethod = "DecodeT2Imm7<"#shift#">";
}
// Operands for gather/scatter loads of the form [Rbase, Qoffsets]
class MemRegRQOffsetAsmOperand<int shift> : AsmOperandClass {
let Name = "MemRegRQS"#shift#"Offset";
let PredicateMethod = "isMemRegRQOffset<"#shift#">";
let RenderMethod = "addMemRegRQOffsetOperands";
}
def MemRegRQS0OffsetAsmOperand : MemRegRQOffsetAsmOperand<0>;
def MemRegRQS1OffsetAsmOperand : MemRegRQOffsetAsmOperand<1>;
def MemRegRQS2OffsetAsmOperand : MemRegRQOffsetAsmOperand<2>;
def MemRegRQS3OffsetAsmOperand : MemRegRQOffsetAsmOperand<3>;
// mve_addr_rq_shift := reg + vreg{ << UXTW #shift}
class mve_addr_rq_shift<int shift> : MemOperand {
let EncoderMethod = "getMveAddrModeRQOpValue";
let PrintMethod = "printMveAddrModeRQOperand<"#shift#">";
let ParserMatchClass =
!cast<AsmOperandClass>("MemRegRQS"#shift#"OffsetAsmOperand");
let DecoderMethod = "DecodeMveAddrModeRQ";
let MIOperandInfo = (ops GPRnopc:$base, MQPR:$offsreg);
}
class MemRegQOffsetAsmOperand<int shift> : AsmOperandClass {
let Name = "MemRegQS"#shift#"Offset";
let PredicateMethod = "isMemRegQOffset<"#shift#">";
let RenderMethod = "addMemImmOffsetOperands";
}
def MemRegQS2OffsetAsmOperand : MemRegQOffsetAsmOperand<2>;
def MemRegQS3OffsetAsmOperand : MemRegQOffsetAsmOperand<3>;
// mve_addr_q_shift := vreg {+ #imm7s2/4}
class mve_addr_q_shift<int shift> : MemOperand {
let EncoderMethod = "getMveAddrModeQOpValue<"#shift#">";
// Can be printed same way as other reg + imm operands
let PrintMethod = "printT2AddrModeImm8Operand<false>";
let ParserMatchClass =
!cast<AsmOperandClass>("MemRegQS"#shift#"OffsetAsmOperand");
let DecoderMethod = "DecodeMveAddrModeQ<"#shift#">";
let MIOperandInfo = (ops MQPR:$base, i32imm:$imm);
}
// A family of classes wrapping up information about the vector types
// used by MVE.
class MVEVectorVTInfo<ValueType vec, ValueType dblvec,
ValueType pred, ValueType dblpred,
bits<2> size, string suffixletter, bit unsigned> {
// The LLVM ValueType representing the vector, so we can use it in
// ISel patterns.
ValueType Vec = vec;
// The LLVM ValueType representing a vector with elements double the size
// of those in Vec, so we can use it in ISel patterns. It is up to the
// invoker of this class to ensure that this is a correct choice.
ValueType DblVec = dblvec;
// An LLVM ValueType representing a corresponding vector of
// predicate bits, for use in ISel patterns that handle an IR
// intrinsic describing the predicated form of the instruction.
//
// Usually, for a vector of N things, this will be vNi1. But for
// vectors of 2 values, we make an exception, and use v4i1 instead
// of v2i1. Rationale: MVE codegen doesn't support doing all the
// auxiliary operations on v2i1 (vector shuffles etc), and also,
// there's no MVE compare instruction that will _generate_ v2i1
// directly.
ValueType Pred = pred;
// Same as Pred but for DblVec rather than Vec.
ValueType DblPred = dblpred;
// The most common representation of the vector element size in MVE
// instruction encodings: a 2-bit value V representing an (8<<V)-bit
// vector element.
bits<2> Size = size;
// For vectors explicitly mentioning a signedness of integers: 0 for
// signed and 1 for unsigned. For anything else, undefined.
bit Unsigned = unsigned;
// The number of bits in a vector element, in integer form.
int LaneBits = !shl(8, Size);
// The suffix used in assembly language on an instruction operating
// on this lane if it only cares about number of bits.
string BitsSuffix = !if(!eq(suffixletter, "p"),
!if(!eq(unsigned, 0b0), "8", "16"),
!cast<string>(LaneBits));
// The suffix used on an instruction that mentions the whole type.
string Suffix = suffixletter # BitsSuffix;
// The letter part of the suffix only.
string SuffixLetter = suffixletter;
}
// Integer vector types that don't treat signed and unsigned differently.
def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>;
def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "i", ?>;
def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "i", ?>;
def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "i", ?>;
// Explicitly signed and unsigned integer vectors. They map to the
// same set of LLVM ValueTypes as above, but are represented
// differently in assembly and instruction encodings.
def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>;
def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "s", 0b0>;
def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "s", 0b0>;
def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "s", 0b0>;
def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>;
def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "u", 0b1>;
def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "u", 0b1>;
def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "u", 0b1>;
// FP vector types.
def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1, v4i1, 0b01, "f", ?>;
def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v4i1, 0b10, "f", ?>;
def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v4i1, ?, 0b11, "f", ?>;
// Polynomial vector types.
def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;
def MVE_v8p16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b11, "p", 0b1>;
multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
dag PredOperands, Instruction Inst,
SDPatternOperator IdentityVec = null_frag> {
// Unpredicated
def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
// Predicated with select
if !ne(VTI.Size, 0b11) then {
def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
(VTI.Vec MQPR:$Qn))),
(VTI.Vec MQPR:$inactive))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$inactive)))>;
// Optionally with the select folded through the op
def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
(VTI.Vec MQPR:$Qn),
(VTI.Vec IdentityVec))))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$Qm)))>;
}
// Predicated with intrinsic
def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),
PredOperands,
(? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$inactive)))>;
}
multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
dag PredOperands, Instruction Inst,
SDPatternOperator IdentityVec = null_frag> {
// Unpredicated
def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;
// Predicated with select
if !ne(VTI.Size, 0b11) then {
def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
(VTI.Vec (ARMvdup rGPR:$Rn)))),
(VTI.Vec MQPR:$inactive))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$inactive)))>;
// Optionally with the select folded through the op
def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
(ARMvdup rGPR:$Rn),
(VTI.Vec IdentityVec))))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$Qm)))>;
}
// Predicated with intrinsic
def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),
PredOperands,
(? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
(VTI.Vec MQPR:$inactive)))>;
}
// --------- Start of base classes for the instructions themselves
class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,
string ops, string cstr, bits<2> vecsize, list<dag> pattern>
: Thumb2XI<oops, iops, AddrModeNone, 4, itin, !strconcat(asm, "\t", ops), cstr,
pattern>,
Requires<[HasMVEInt]> {
let D = MVEDomain;
let DecoderNamespace = "MVE";
let VecSize = vecsize;
}
// MVE_p is used for most predicated instructions, to add the cluster
// of input operands that provides the VPT suffix (none, T or E) and
// the input predicate register.
class MVE_p<dag oops, dag iops, InstrItinClass itin, string iname,
string suffix, string ops, vpred_ops vpred, string cstr,
bits<2> vecsize, list<dag> pattern=[]>
: MVE_MI<oops, !con(iops, (ins vpred:$vp)), itin,
// If the instruction has a suffix, like vadd.f32, then the
// VPT predication suffix goes before the dot, so the full
// name has to be "vadd${vp}.f32".
!strconcat(iname, "${vp}",
!if(!eq(suffix, ""), "", !strconcat(".", suffix))),
ops, !strconcat(cstr, vpred.vpred_constraint), vecsize, pattern> {
let Inst{31-29} = 0b111;
let Inst{27-26} = 0b11;
}
class MVE_f<dag oops, dag iops, InstrItinClass itin, string iname,
string suffix, string ops, vpred_ops vpred, string cstr,
bits<2> vecsize, list<dag> pattern=[]>
: MVE_p<oops, iops, itin, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
let Predicates = [HasMVEFloat];
}
class MVE_MI_with_pred<dag oops, dag iops, InstrItinClass itin, string asm,
string ops, string cstr, list<dag> pattern>
: Thumb2I<oops, iops, AddrModeNone, 4, itin, asm, !strconcat("\t", ops), cstr,
pattern>,
Requires<[HasV8_1MMainline, HasMVEInt]> {
let D = MVEDomain;
let DecoderNamespace = "MVE";
}
class MVE_VMOV_lane_base<dag oops, dag iops, InstrItinClass itin, string asm,
string suffix, string ops, string cstr,
list<dag> pattern>
: Thumb2I<oops, iops, AddrModeNone, 4, itin, asm,
!if(!eq(suffix, ""), "", "." # suffix) # "\t" # ops,
cstr, pattern>,
Requires<[HasV8_1MMainline, HasMVEInt]> {
let D = MVEDomain;
let DecoderNamespace = "MVE";
}
class MVE_ScalarShift<string iname, dag oops, dag iops, string asm, string cstr,
list<dag> pattern=[]>
: MVE_MI_with_pred<oops, iops, NoItinerary, iname, asm, cstr, pattern> {
let Inst{31-20} = 0b111010100101;
let Inst{8} = 0b1;
let validForTailPredication=1;
}
class MVE_ScalarShiftSingleReg<string iname, dag iops, string asm, string cstr,
list<dag> pattern=[]>
: MVE_ScalarShift<iname, (outs rGPR:$RdaDest), iops, asm, cstr, pattern> {
bits<4> RdaDest;
let Inst{19-16} = RdaDest{3-0};
}
class MVE_ScalarShiftSRegImm<string iname, bits<2> op5_4>
: MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, long_shift:$imm),
"$RdaSrc, $imm", "$RdaDest = $RdaSrc",
[(set rGPR:$RdaDest,
(i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
(i32 rGPR:$RdaSrc), (i32 imm:$imm))))]> {
bits<5> imm;
let Inst{15} = 0b0;
let Inst{14-12} = imm{4-2};
let Inst{11-8} = 0b1111;
let Inst{7-6} = imm{1-0};
let Inst{5-4} = op5_4{1-0};
let Inst{3-0} = 0b1111;
}
def MVE_SQSHL : MVE_ScalarShiftSRegImm<"sqshl", 0b11>;
def MVE_SRSHR : MVE_ScalarShiftSRegImm<"srshr", 0b10>;
def MVE_UQSHL : MVE_ScalarShiftSRegImm<"uqshl", 0b00>;
def MVE_URSHR : MVE_ScalarShiftSRegImm<"urshr", 0b01>;
class MVE_ScalarShiftSRegReg<string iname, bits<2> op5_4>
: MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, rGPR:$Rm),
"$RdaSrc, $Rm", "$RdaDest = $RdaSrc",
[(set rGPR:$RdaDest,
(i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
(i32 rGPR:$RdaSrc), (i32 rGPR:$Rm))))]> {
bits<4> Rm;
let Inst{15-12} = Rm{3-0};
let Inst{11-8} = 0b1111;
let Inst{7-6} = 0b00;
let Inst{5-4} = op5_4{1-0};
let Inst{3-0} = 0b1101;
let Unpredictable{8-6} = 0b111;
}
def MVE_SQRSHR : MVE_ScalarShiftSRegReg<"sqrshr", 0b10>;
def MVE_UQRSHL : MVE_ScalarShiftSRegReg<"uqrshl", 0b00>;
class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,
string cstr, list<dag> pattern=[]>
: MVE_ScalarShift<iname, (outs tGPREven:$RdaLo, tGPROdd:$RdaHi),
iops, asm, cstr, pattern> {
bits<4> RdaLo;
bits<4> RdaHi;
let Inst{19-17} = RdaLo{3-1};
let Inst{11-9} = RdaHi{3-1};
let hasSideEffects = 0;
}
class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,
list<dag> pattern=[]>
: MVE_ScalarShiftDoubleReg<
iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, long_shift:$imm),
"$RdaLo, $RdaHi, $imm", "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
pattern> {
bits<5> imm;
let Inst{16} = op16;
let Inst{15} = 0b0;
let Inst{14-12} = imm{4-2};
let Inst{7-6} = imm{1-0};
let Inst{5-4} = op5_4{1-0};
let Inst{3-0} = 0b1111;
}
class MVE_ScalarShiftDRegRegBase<string iname, dag iops, string asm,
bit op5, bit op16, list<dag> pattern=[]>
: MVE_ScalarShiftDoubleReg<
iname, iops, asm, "@earlyclobber $RdaHi,@earlyclobber $RdaLo,"
"$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
pattern> {
bits<4> Rm;
let Inst{16} = op16;
let Inst{15-12} = Rm{3-0};
let Inst{6} = 0b0;
let Inst{5} = op5;
let Inst{4} = 0b0;
let Inst{3-0} = 0b1101;
// Custom decoder method because of the following overlapping encodings:
// ASRL and SQRSHR
// LSLL and UQRSHL
// SQRSHRL and SQRSHR
// UQRSHLL and UQRSHL
let DecoderMethod = "DecodeMVEOverlappingLongShift";
}
class MVE_ScalarShiftDRegReg<string iname, bit op5, list<dag> pattern=[]>
: MVE_ScalarShiftDRegRegBase<
iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm),
"$RdaLo, $RdaHi, $Rm", op5, 0b0, pattern> {
let Inst{7} = 0b0;
}
class MVE_ScalarShiftDRegRegWithSat<string iname, bit op5, list<dag> pattern=[]>
: MVE_ScalarShiftDRegRegBase<
iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm, saturateop:$sat),
"$RdaLo, $RdaHi, $sat, $Rm", op5, 0b1, pattern> {
bit sat;
let Inst{7} = sat;
}
def MVE_ASRLr : MVE_ScalarShiftDRegReg<"asrl", 0b1, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
(ARMasrl tGPREven:$RdaLo_src,
tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
def MVE_ASRLi : MVE_ScalarShiftDRegImm<"asrl", 0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
(ARMasrl tGPREven:$RdaLo_src,
tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
def MVE_LSLLr : MVE_ScalarShiftDRegReg<"lsll", 0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
(ARMlsll tGPREven:$RdaLo_src,
tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
def MVE_LSLLi : MVE_ScalarShiftDRegImm<"lsll", 0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
(ARMlsll tGPREven:$RdaLo_src,
tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
def MVE_LSRL : MVE_ScalarShiftDRegImm<"lsrl", 0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
(ARMlsrl tGPREven:$RdaLo_src,
tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
def MVE_SQRSHRL : MVE_ScalarShiftDRegRegWithSat<"sqrshrl", 0b1>;
def MVE_SQSHLL : MVE_ScalarShiftDRegImm<"sqshll", 0b11, 0b1>;
def MVE_SRSHRL : MVE_ScalarShiftDRegImm<"srshrl", 0b10, 0b1>;
def MVE_UQRSHLL : MVE_ScalarShiftDRegRegWithSat<"uqrshll", 0b0>;
def MVE_UQSHLL : MVE_ScalarShiftDRegImm<"uqshll", 0b00, 0b1>;
def MVE_URSHRL : MVE_ScalarShiftDRegImm<"urshrl", 0b01, 0b1>;
// start of mve_rDest instructions
class MVE_rDest<dag oops, dag iops, InstrItinClass itin,
string iname, string suffix,
string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>
// Always use vpred_n and not vpred_r: with the output register being
// a GPR and not a vector register, there can't be any question of
// what to put in its inactive lanes.
: MVE_p<oops, iops, itin, iname, suffix, ops, vpred_n, cstr, vecsize, pattern> {
let Inst{25-23} = 0b101;
let Inst{11-9} = 0b111;
let Inst{4} = 0b0;
}
class MVE_VABAV<string suffix, bit U, bits<2> size>
: MVE_rDest<(outs rGPR:$Rda), (ins rGPR:$Rda_src, MQPR:$Qn, MQPR:$Qm),
NoItinerary, "vabav", suffix, "$Rda, $Qn, $Qm", "$Rda = $Rda_src",
size, []> {
bits<4> Qm;
bits<4> Qn;
bits<4> Rda;
let Inst{28} = U;
let Inst{22} = 0b0;
let Inst{21-20} = size{1-0};
let Inst{19-17} = Qn{2-0};
let Inst{16} = 0b0;
let Inst{15-12} = Rda{3-0};
let Inst{8} = 0b1;
let Inst{7} = Qn{3};
let Inst{6} = 0b0;
let Inst{5} = Qm{3};
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b1;
let horizontalReduction = 1;
}
multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {
def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
def : Pat<(i32 (int_arm_mve_vabav
(i32 VTI.Unsigned),
(i32 rGPR:$Rda_src),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
(i32 (Inst (i32 rGPR:$Rda_src),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
def : Pat<(i32 (int_arm_mve_vabav_predicated
(i32 VTI.Unsigned),
(i32 rGPR:$Rda_src),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
(VTI.Pred VCCR:$mask))),
(i32 (Inst (i32 rGPR:$Rda_src),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
}
}
defm MVE_VABAVs8 : MVE_VABAV_m<MVE_v16s8>;
defm MVE_VABAVs16 : MVE_VABAV_m<MVE_v8s16>;
defm MVE_VABAVs32 : MVE_VABAV_m<MVE_v4s32>;
defm MVE_VABAVu8 : MVE_VABAV_m<MVE_v16u8>;
defm MVE_VABAVu16 : MVE_VABAV_m<MVE_v8u16>;
defm MVE_VABAVu32 : MVE_VABAV_m<MVE_v4u32>;
class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
bit A, bit U, bits<2> size, list<dag> pattern=[]>
: MVE_rDest<(outs tGPREven:$Rda), iops, NoItinerary,
iname, suffix, "$Rda, $Qm", cstr, size, pattern> {
bits<3> Qm;
bits<4> Rda;
let Inst{28} = U;
let Inst{22-20} = 0b111;
let Inst{19-18} = size{1-0};
let Inst{17-16} = 0b01;
let Inst{15-13} = Rda{3-1};
let Inst{12} = 0b0;
let Inst{8-6} = 0b100;
let Inst{5} = A;
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b0;
let horizontalReduction = 1;
let validForTailPredication = 1;
}
def SDTVecReduceP : SDTypeProfile<1, 2, [ // VADDLVp
SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
]>;
def ARMVADDVs : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
def ARMVADDVu : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
def ARMVADDVps : SDNode<"ARMISD::VADDVps", SDTVecReduceP>;
def ARMVADDVpu : SDNode<"ARMISD::VADDVpu", SDTVecReduceP>;
multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
def acc : MVE_VADDV<"vaddva", VTI.Suffix,
(ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",
0b1, VTI.Unsigned, VTI.Size>;
def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
(ins MQPR:$Qm), "",
0b0, VTI.Unsigned, VTI.Size>;
defvar InstA = !cast<Instruction>(NAME # "acc");
defvar InstN = !cast<Instruction>(NAME # "no_acc");
let Predicates = [HasMVEInt] in {
if VTI.Unsigned then {
def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
(i32 (InstN $vec))>;
def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
(VTI.Vec MQPR:$vec),
(VTI.Vec ARMimmAllZerosV))))),
(i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
(i32 (InstN $vec))>;
def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
(i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec))>;
def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
(VTI.Vec MQPR:$vec),
(VTI.Vec ARMimmAllZerosV))))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec))>;
def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
} else {
def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
(i32 (InstN $vec))>;
def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec))>;
def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
(i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
}
def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
(i32 VTI.Unsigned),
(VTI.Pred VCCR:$pred))),
(i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
(i32 VTI.Unsigned),
(VTI.Pred VCCR:$pred)),
(i32 tGPREven:$acc))),
(i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
}
}
defm MVE_VADDVs8 : MVE_VADDV_A<MVE_v16s8>;
defm MVE_VADDVs16 : MVE_VADDV_A<MVE_v8s16>;
defm MVE_VADDVs32 : MVE_VADDV_A<MVE_v4s32>;
defm MVE_VADDVu8 : MVE_VADDV_A<MVE_v16u8>;
defm MVE_VADDVu16 : MVE_VADDV_A<MVE_v8u16>;
defm MVE_VADDVu32 : MVE_VADDV_A<MVE_v4u32>;
class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
bit A, bit U, list<dag> pattern=[]>
: MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,
suffix, "$RdaLo, $RdaHi, $Qm", cstr, 0b10, pattern> {
bits<3> Qm;
bits<4> RdaLo;
bits<4> RdaHi;
let Inst{28} = U;
let Inst{22-20} = RdaHi{3-1};
let Inst{19-18} = 0b10;
let Inst{17-16} = 0b01;
let Inst{15-13} = RdaLo{3-1};
let Inst{12} = 0b0;
let Inst{8-6} = 0b100;
let Inst{5} = A;
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b0;
let horizontalReduction = 1;
}
def SDTVecReduceL : SDTypeProfile<2, 1, [ // VADDLV
SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
]>;
def SDTVecReduceLA : SDTypeProfile<2, 3, [ // VADDLVA
SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
SDTCisVec<4>
]>;
def SDTVecReduceLP : SDTypeProfile<2, 2, [ // VADDLVp
SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<2>
]>;
def SDTVecReduceLPA : SDTypeProfile<2, 4, [ // VADDLVAp
SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
SDTCisVec<4>, SDTCisVec<5>
]>;
multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
def acc : MVE_VADDLV<"vaddlva", VTI.Suffix,
(ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
"$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
0b1, VTI.Unsigned>;
def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
(ins MQPR:$Qm), "",
0b0, VTI.Unsigned>;
defvar InstA = !cast<Instruction>(NAME # "acc");
defvar InstN = !cast<Instruction>(NAME # "no_acc");
defvar letter = VTI.SuffixLetter;
defvar ARMVADDLV = SDNode<"ARMISD::VADDLV" # letter, SDTVecReduceL>;
defvar ARMVADDLVA = SDNode<"ARMISD::VADDLVA" # letter, SDTVecReduceLA>;
defvar ARMVADDLVp = SDNode<"ARMISD::VADDLVp" # letter, SDTVecReduceLP>;
defvar ARMVADDLVAp = SDNode<"ARMISD::VADDLVAp" # letter, SDTVecReduceLPA>;
let Predicates = [HasMVEInt] in {
def : Pat<(ARMVADDLV (v4i32 MQPR:$vec)),
(InstN (v4i32 MQPR:$vec))>;
def : Pat<(ARMVADDLVA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec)),
(InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec))>;
def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
(InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
def : Pat<(ARMVADDLVAp tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
(VTI.Pred VCCR:$pred)),
(InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
}
}
defm MVE_VADDLVs32 : MVE_VADDLV_A<MVE_v4s32>;
defm MVE_VADDLVu32 : MVE_VADDLV_A<MVE_v4u32>;
class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
bit bit_17, bit bit_7, list<dag> pattern=[]>
: MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),
NoItinerary, iname, suffix, "$RdaSrc, $Qm",
"$RdaDest = $RdaSrc", !if(sz, 0b01, 0b10), pattern> {
bits<3> Qm;
bits<4> RdaDest;
let Inst{28} = sz;
let Inst{22-20} = 0b110;
let Inst{19-18} = 0b11;
let Inst{17} = bit_17;
let Inst{16} = 0b0;
let Inst{15-12} = RdaDest{3-0};
let Inst{8} = 0b1;
let Inst{7} = bit_7;
let Inst{6-5} = 0b00;
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b0;
let horizontalReduction = 1;
let Predicates = [HasMVEFloat];
let hasSideEffects = 0;
}
multiclass MVE_VMINMAXNMV_p<string iname, bit notAbs, bit isMin,
MVEVectorVTInfo VTI, string intrBaseName,
ValueType Scalar, RegisterClass ScalarReg> {
def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;
defvar Inst = !cast<Instruction>(NAME);
defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
defvar pred_intr = !cast<Intrinsic>(intrBaseName#"_predicated");
let Predicates = [HasMVEFloat] in {
def : Pat<(Scalar (unpred_intr (Scalar ScalarReg:$prev),
(VTI.Vec MQPR:$vec))),
(COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
(VTI.Vec MQPR:$vec)),
ScalarReg)>;
def : Pat<(Scalar (pred_intr (Scalar ScalarReg:$prev),
(VTI.Vec MQPR:$vec),
(VTI.Pred VCCR:$pred))),
(COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
(VTI.Vec MQPR:$vec),
ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg),
ScalarReg)>;
}
}
multiclass MVE_VMINMAXNMV_fty<string iname, bit notAbs, bit isMin,
string intrBase> {
defm f32 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v4f32, intrBase,
f32, SPR>;
defm f16 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v8f16, intrBase,
f16, HPR>;
}
defm MVE_VMINNMV : MVE_VMINMAXNMV_fty<"vminnmv", 1, 1, "int_arm_mve_minnmv">;
defm MVE_VMAXNMV : MVE_VMINMAXNMV_fty<"vmaxnmv", 1, 0, "int_arm_mve_maxnmv">;
defm MVE_VMINNMAV: MVE_VMINMAXNMV_fty<"vminnmav", 0, 1, "int_arm_mve_minnmav">;
defm MVE_VMAXNMAV: MVE_VMINMAXNMV_fty<"vmaxnmav", 0, 0, "int_arm_mve_maxnmav">;
class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,
bit bit_17, bit bit_7, list<dag> pattern=[]>
: MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm), NoItinerary,
iname, suffix, "$RdaSrc, $Qm", "$RdaDest = $RdaSrc", size, pattern> {
bits<3> Qm;
bits<4> RdaDest;
let Inst{28} = U;
let Inst{22-20} = 0b110;
let Inst{19-18} = size{1-0};
let Inst{17} = bit_17;
let Inst{16} = 0b0;
let Inst{15-12} = RdaDest{3-0};
let Inst{8} = 0b1;
let Inst{7} = bit_7;
let Inst{6-5} = 0b00;
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b0;
let horizontalReduction = 1;
}
multiclass MVE_VMINMAXV_p<string iname, bit notAbs, bit isMin,
MVEVectorVTInfo VTI, string intrBaseName> {
def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
notAbs, isMin>;
defvar Inst = !cast<Instruction>(NAME);
defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
defvar pred_intr = !cast<Intrinsic>(intrBaseName#"_predicated");
defvar base_args = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));
defvar args = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),
base_args);
let Predicates = [HasMVEInt] in {
def : Pat<(i32 !con(args, (unpred_intr))),
(i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;
def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),
(i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),
ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
}
}
multiclass MVE_VMINMAXV_ty<string iname, bit isMin, string intrBaseName> {
defm s8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16s8, intrBaseName>;
defm s16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8s16, intrBaseName>;
defm s32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4s32, intrBaseName>;
defm u8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16u8, intrBaseName>;
defm u16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8u16, intrBaseName>;
defm u32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4u32, intrBaseName>;
}
def SDTVecReduceR : SDTypeProfile<1, 2, [ // Reduction of an integer and vector into an integer
SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
]>;
def ARMVMINVu : SDNode<"ARMISD::VMINVu", SDTVecReduceR>;
def ARMVMINVs : SDNode<"ARMISD::VMINVs", SDTVecReduceR>;
def ARMVMAXVu : SDNode<"ARMISD::VMAXVu", SDTVecReduceR>;
def ARMVMAXVs : SDNode<"ARMISD::VMAXVs", SDTVecReduceR>;
defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 1, "int_arm_mve_minv">;
defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0, "int_arm_mve_maxv">;
let Predicates = [HasMVEInt] in {
def : Pat<(i32 (vecreduce_smax (v16i8 MQPR:$src))),
(i32 (MVE_VMAXVs8 (t2MVNi (i32 127)), $src))>;
def : Pat<(i32 (vecreduce_smax (v8i16 MQPR:$src))),
(i32 (MVE_VMAXVs16 (t2MOVi32imm (i32 -32768)), $src))>;
def : Pat<(i32 (vecreduce_smax (v4i32 MQPR:$src))),
(i32 (MVE_VMAXVs32 (t2MOVi (i32 -2147483648)), $src))>;
def : Pat<(i32 (vecreduce_umax (v16i8 MQPR:$src))),
(i32 (MVE_VMAXVu8 (t2MOVi (i32 0)), $src))>;
def : Pat<(i32 (vecreduce_umax (v8i16 MQPR:$src))),
(i32 (MVE_VMAXVu16 (t2MOVi (i32 0)), $src))>;
def : Pat<(i32 (vecreduce_umax (v4i32 MQPR:$src))),
(i32 (MVE_VMAXVu32 (t2MOVi (i32 0)), $src))>;
def : Pat<(i32 (vecreduce_smin (v16i8 MQPR:$src))),
(i32 (MVE_VMINVs8 (t2MOVi (i32 127)), $src))>;
def : Pat<(i32 (vecreduce_smin (v8i16 MQPR:$src))),
(i32 (MVE_VMINVs16 (t2MOVi16 (i32 32767)), $src))>;
def : Pat<(i32 (vecreduce_smin (v4i32 MQPR:$src))),
(i32 (MVE_VMINVs32 (t2MVNi (i32 -2147483648)), $src))>;
def : Pat<(i32 (vecreduce_umin (v16i8 MQPR:$src))),
(i32 (MVE_VMINVu8 (t2MOVi (i32 255)), $src))>;
def : Pat<(i32 (vecreduce_umin (v8i16 MQPR:$src))),
(i32 (MVE_VMINVu16 (t2MOVi16 (i32 65535)), $src))>;
def : Pat<(i32 (vecreduce_umin (v4i32 MQPR:$src))),
(i32 (MVE_VMINVu32 (t2MOVi (i32 4294967295)), $src))>;
def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v16i8 MQPR:$src))),
(i32 (MVE_VMINVu8 $x, $src))>;
def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v8i16 MQPR:$src))),
(i32 (MVE_VMINVu16 $x, $src))>;
def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v4i32 MQPR:$src))),
(i32 (MVE_VMINVu32 $x, $src))>;
def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v16i8 MQPR:$src))),
(i32 (MVE_VMINVs8 $x, $src))>;
def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v8i16 MQPR:$src))),
(i32 (MVE_VMINVs16 $x, $src))>;
def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v4i32 MQPR:$src))),
(i32 (MVE_VMINVs32 $x, $src))>;
def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v16i8 MQPR:$src))),
(i32 (MVE_VMAXVu8 $x, $src))>;
def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v8i16 MQPR:$src))),
(i32 (MVE_VMAXVu16 $x, $src))>;
def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v4i32 MQPR:$src))),
(i32 (MVE_VMAXVu32 $x, $src))>;
def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v16i8 MQPR:$src))),
(i32 (MVE_VMAXVs8 $x, $src))>;
def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v8i16 MQPR:$src))),
(i32 (MVE_VMAXVs16 $x, $src))>;
def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v4i32 MQPR:$src))),
(i32 (MVE_VMAXVs32 $x, $src))>;
}
multiclass MVE_VMINMAXAV_ty<string iname, bit isMin, string intrBaseName> {
defm s8 : MVE_VMINMAXV_p<iname, 0, isMin, MVE_v16s8, intrBaseName>;
defm s16: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v8s16, intrBaseName>;
defm s32: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v4s32, intrBaseName>;
}
defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 1, "int_arm_mve_minav">;
defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0, "int_arm_mve_maxav">;
class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
bits<2> vecsize>
: MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,
"$RdaDest, $Qn, $Qm", cstr, vecsize, []> {
bits<4> RdaDest;
bits<3> Qm;
bits<3> Qn;
let Inst{28} = bit_28;
let Inst{22-20} = 0b111;
let Inst{19-17} = Qn{2-0};
let Inst{16} = sz;
let Inst{15-13} = RdaDest{3-1};
let Inst{12} = X;
let Inst{8} = bit_8;
let Inst{7-6} = 0b00;
let Inst{5} = A;
let Inst{3-1} = Qm{2-0};
let Inst{0} = bit_0;
let horizontalReduction = 1;
// Allow tail predication for non-exchanging versions. As this is also a
// horizontalReduction, ARMLowOverheadLoops will also have to check that
// the vector operands contain zeros in their false lanes for the instruction
// to be properly valid.
let validForTailPredication = !eq(X, 0);
}
multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,
bit sz, bit bit_28, bit X, bit bit_8, bit bit_0> {
def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,
(ins MQPR:$Qn, MQPR:$Qm), "",
sz, bit_28, 0b0, X, bit_8, bit_0, VTI.Size>;
def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,
(ins tGPREven:$RdaSrc, MQPR:$Qn, MQPR:$Qm),
"$RdaDest = $RdaSrc",
sz, bit_28, 0b1, X, bit_8, bit_0, VTI.Size>;
let Predicates = [HasMVEInt] in {
def : Pat<(i32 (int_arm_mve_vmldava
(i32 VTI.Unsigned),
(i32 bit_0) /* subtract */,
(i32 X) /* exchange */,
(i32 0) /* accumulator */,
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
(i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
def : Pat<(i32 (int_arm_mve_vmldava_predicated
(i32 VTI.Unsigned),
(i32 bit_0) /* subtract */,
(i32 X) /* exchange */,
(i32 0) /* accumulator */,
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
(VTI.Pred VCCR:$mask))),
(i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
def : Pat<(i32 (int_arm_mve_vmldava
(i32 VTI.Unsigned),
(i32 bit_0) /* subtract */,
(i32 X) /* exchange */,
(i32 tGPREven:$RdaSrc),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
(i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
(i32 tGPREven:$RdaSrc),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
def : Pat<(i32 (int_arm_mve_vmldava_predicated
(i32 VTI.Unsigned),
(i32 bit_0) /* subtract */,
(i32 X) /* exchange */,
(i32 tGPREven:$RdaSrc),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
(VTI.Pred VCCR:$mask))),
(i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
(i32 tGPREven:$RdaSrc),
(VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
}
}
multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,
bit bit_28, bit bit_8, bit bit_0> {
defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,
0b0, bit_8, bit_0>;
defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,
0b1, bit_8, bit_0>;
}
multiclass MVE_VMLADAV_multi<MVEVectorVTInfo SVTI, MVEVectorVTInfo UVTI,
bit sz, bit bit_8> {
defm "" : MVE_VMLAMLSDAV_AX<"vmladav", SVTI,
sz, 0b0, bit_8, 0b0>;
defm "" : MVE_VMLAMLSDAV_A<"vmladav", "", UVTI,
sz, 0b1, 0b0, bit_8, 0b0>;
}
multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {
defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,
sz, bit_28, 0b0, 0b1>;
}
defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v16s8, MVE_v16u8, 0b0, 0b1>;
defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v8s16, MVE_v8u16, 0b0, 0b0>;
defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v4s32, MVE_v4u32, 0b1, 0b0>;
defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v16s8, 0b0, 0b1>;
defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v8s16, 0b0, 0b0>;
defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v4s32, 0b1, 0b0>;
def SDTVecReduce2 : SDTypeProfile<1, 2, [ // VMLAV
SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
]>;
def SDTVecReduce2L : SDTypeProfile<2, 2, [ // VMLALV
SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>
]>;
def SDTVecReduce2LA : SDTypeProfile<2, 4, [ // VMLALVA
SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
SDTCisVec<4>, SDTCisVec<5>
]>;
def SDTVecReduce2P : SDTypeProfile<1, 3, [ // VMLAV
SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>
]>;
def SDTVecReduce2LP : SDTypeProfile<2, 3, [ // VMLALV
SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>
]>;
def SDTVecReduce2LAP : SDTypeProfile<2, 5, [ // VMLALVA
SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
SDTCisVec<4>, SDTCisVec<5>, SDTCisVec<6>
]>;
def ARMVMLAVs : SDNode<"ARMISD::VMLAVs", SDTVecReduce2>;
def ARMVMLAVu : SDNode<"ARMISD::VMLAVu", SDTVecReduce2>;
def ARMVMLALVs : SDNode<"ARMISD::VMLALVs", SDTVecReduce2L>;
def ARMVMLALVu : SDNode<"ARMISD::VMLALVu", SDTVecReduce2L>;
def ARMVMLALVAs : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;
def ARMVMLALVAu : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;
def ARMVMLAVps : SDNode<"ARMISD::VMLAVps", SDTVecReduce2P>;
def ARMVMLAVpu : SDNode<"ARMISD::VMLAVpu", SDTVecReduce2P>;
def ARMVMLALVps : SDNode<"ARMISD::VMLALVps", SDTVecReduce2LP>;
def ARMVMLALVpu : SDNode<"ARMISD::VMLALVpu", SDTVecReduce2LP>;
def ARMVMLALVAps : SDNode<"ARMISD::VMLALVAps", SDTVecReduce2LAP>;
def ARMVMLALVApu : SDNode<"ARMISD::VMLALVApu", SDTVecReduce2LAP>;
let Predicates = [HasMVEInt] in {
def : Pat<(i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
(i32 (MVE_VMLADAVu32 $src1, $src2))>;
def : Pat<(i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
(i32 (MVE_VMLADAVu16 $src1, $src2))>;
def : Pat<(i32 (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
(i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
def : Pat<(i32 (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
(i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
def : Pat<(i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
(i32 (MVE_VMLADAVu8 $src1, $src2))>;
def : Pat<(i32 (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
(i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
def : Pat<(i32 (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
(i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
def : Pat<(i32 (add (i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau32 $src3, $src1, $src2))>;
def : Pat<(i32 (add (i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau16 $src3, $src1, $src2))>;
def : Pat<(i32 (add (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
def : Pat<(i32 (add (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
def : Pat<(i32 (add (i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau8 $src3, $src1, $src2))>;
def : Pat<(i32 (add (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
def : Pat<(i32 (add (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
// Predicated
def : Pat<(i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
(mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
(v4i32 ARMimmAllZerosV)))),
(i32 (MVE_VMLADAVu32 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
(mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
(v8i16 ARMimmAllZerosV)))),
(i32 (MVE_VMLADAVu16 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
(i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
(i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
(mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
(v16i8 ARMimmAllZerosV)))),
(i32 (MVE_VMLADAVu8 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
(i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
(i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
(mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
(v4i32 ARMimmAllZerosV)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau32 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
(mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
(v8i16 ARMimmAllZerosV)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau16 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
(mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
(v16i8 ARMimmAllZerosV)))),
(i32 tGPREven:$src3))),
(i32 (MVE_VMLADAVau8 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
def : Pat<(i32 (add (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
(i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
}
// vmlav aliases vmladav
foreach acc = ["", "a"] in {
foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32"] in {
def : MVEInstAlias<"vmlav"#acc#"${vp}."#suffix#"\t$RdaDest, $Qn, $Qm",
(!cast<Instruction>("MVE_VMLADAV"#acc#suffix)
tGPREven:$RdaDest, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
}
}
// Base class for VMLALDAV and VMLSLDAV, VRMLALDAVH, VRMLSLDAVH
class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
bits<2> vecsize, list<dag> pattern=[]>
: MVE_rDest<(outs tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest), iops, NoItinerary,
iname, suffix, "$RdaLoDest, $RdaHiDest, $Qn, $Qm", cstr, vecsize, pattern> {
bits<4> RdaLoDest;
bits<4> RdaHiDest;
bits<3> Qm;
bits<3> Qn;
let Inst{28} = bit_28;
let Inst{22-20} = RdaHiDest{3-1};
let Inst{19-17} = Qn{2-0};
let Inst{16} = sz;
let Inst{15-13} = RdaLoDest{3-1};
let Inst{12} = X;
let Inst{8} = bit_8;
let Inst{7-6} = 0b00;
let Inst{5} = A;
let Inst{3-1} = Qm{2-0};
let Inst{0} = bit_0;
let horizontalReduction = 1;
// Allow tail predication for non-exchanging versions. As this is also a
// horizontalReduction, ARMLowOverheadLoops will also have to check that
// the vector operands contain zeros in their false lanes for the instruction
// to be properly valid.
let validForTailPredication = !eq(X, 0);
let hasSideEffects = 0;
}
multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,
bit sz, bit bit_28, bit X, bit bit_8, bit bit_0,
bits<2> vecsize, list<dag> pattern=[]> {
def ""#x#suffix : MVE_VMLALDAVBase<
iname # x, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",
sz, bit_28, 0b0, X, bit_8, bit_0, vecsize, pattern>;
def "a"#x#suffix : MVE_VMLALDAVBase<
iname # "a" # x, suffix,
(ins tGPREven:$RdaLoSrc, tGPROdd:$RdaHiSrc, MQPR:$Qn, MQPR:$Qm),
"$RdaLoDest = $RdaLoSrc,$RdaHiDest = $RdaHiSrc",
sz, bit_28, 0b1, X, bit_8, bit_0, vecsize, pattern>;
}
multiclass MVE_VMLALDAVBase_AX<string iname, string suffix, bit sz, bit bit_28,
bit bit_8, bit bit_0, bits<2> vecsize, list<dag> pattern=[]> {
defm "" : MVE_VMLALDAVBase_A<iname, "", suffix, sz,
bit_28, 0b0, bit_8, bit_0, vecsize, pattern>;
defm "" : MVE_VMLALDAVBase_A<iname, "x", suffix, sz,
bit_28, 0b1, bit_8, bit_0, vecsize, pattern>;
}
multiclass MVE_VRMLALDAVH_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
defm "" : MVE_VMLALDAVBase_AX<"vrmlaldavh", "s"#VTI.BitsSuffix,
0b0, 0b0, 0b1, 0b0, VTI.Size, pattern>;
defm "" : MVE_VMLALDAVBase_A<"vrmlaldavh", "", "u"#VTI.BitsSuffix,
0b0, 0b1, 0b0, 0b1, 0b0, VTI.Size, pattern>;
}
defm MVE_VRMLALDAVH : MVE_VRMLALDAVH_multi<MVE_v4i32>;
// vrmlalvh aliases for vrmlaldavh
def : MVEInstAlias<"vrmlalvh${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
(MVE_VRMLALDAVHs32
tGPREven:$RdaLo, tGPROdd:$RdaHi,
MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
def : MVEInstAlias<"vrmlalvha${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
(MVE_VRMLALDAVHas32
tGPREven:$RdaLo, tGPROdd:$RdaHi,
MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
def : MVEInstAlias<"vrmlalvh${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
(MVE_VRMLALDAVHu32
tGPREven:$RdaLo, tGPROdd:$RdaHi,
MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
def : MVEInstAlias<"vrmlalvha${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
(MVE_VRMLALDAVHau32
tGPREven:$RdaLo, tGPROdd:$RdaHi,
MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
multiclass MVE_VMLALDAV_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
defm "" : MVE_VMLALDAVBase_AX<"vmlaldav", "s"#VTI.BitsSuffix,
VTI.Size{1}, 0b0, 0b0, 0b0, VTI.Size, pattern>;
defm "" : MVE_VMLALDAVBase_A<"vmlaldav", "", "u"#VTI.BitsSuffix,
VTI.Size{1}, 0b1, 0b0, 0b0, 0b0, VTI.Size, pattern>;
}
defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v8i16>;
defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v4i32>;
let Predicates = [HasMVEInt] in {
def : Pat<(ARMVMLALVs (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
(MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
def : Pat<(ARMVMLALVu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
(MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
def : Pat<(ARMVMLALVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
(MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
def : Pat<(ARMVMLALVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
(MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
(MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
(MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
(MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
(MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
// Predicated
def : Pat<(ARMVMLALVps (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
(MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVpu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
(MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
(MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
(MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
(MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
(MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
(MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
(MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
}
// vmlalv aliases vmlaldav
foreach acc = ["", "a"] in {
foreach suffix = ["s16", "s32", "u16", "u32"] in {
def : MVEInstAlias<"vmlalv" # acc # "${vp}." # suffix #
"\t$RdaLoDest, $RdaHiDest, $Qn, $Qm",
(!cast<Instruction>("MVE_VMLALDAV"#acc#suffix)
tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest,
MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
}
}
multiclass MVE_VMLSLDAV_multi<string iname, string suffix, bit sz,
bit bit_28, bits<2> vecsize, list<dag> pattern=[]> {
defm "" : MVE_VMLALDAVBase_AX<iname, suffix, sz, bit_28, 0b0, 0b1, vecsize, pattern>;
}
defm MVE_VMLSLDAV : MVE_VMLSLDAV_multi<"vmlsldav", "s16", 0b0, 0b0, 0b01>;
defm MVE_VMLSLDAV : MVE_VMLSLDAV_multi<"vmlsldav", "s32", 0b1, 0b0, 0b10>;
defm MVE_VRMLSLDAVH : MVE_VMLSLDAV_multi<"vrmlsldavh", "s32", 0b0, 0b1, 0b10>;
// end of mve_rDest instructions
// start of mve_comp instructions
class MVE_comp<InstrItinClass itin, string iname, string suffix,
string cstr, bits<2> vecsize, list<dag> pattern=[]>
: MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), itin, iname, suffix,
"$Qd, $Qn, $Qm", vpred_r, cstr, vecsize, pattern> {
bits<4> Qd;
bits<4> Qn;
bits<4> Qm;
let Inst{22} = Qd{3};
let Inst{19-17} = Qn{2-0};
let Inst{16} = 0b0;
let Inst{15-13} = Qd{2-0};
let Inst{12} = 0b0;
let Inst{10-9} = 0b11;
let Inst{7} = Qn{3};
let Inst{5} = Qm{3};
let Inst{3-1} = Qm{2-0};
let Inst{0} = 0b0;
}
class MVE_VMINMAXNM<string iname, string suffix, bits<2> sz, bit bit_21,
list<dag> pattern=[]>
: MVE_comp<NoItinerary, iname, suffix, "", sz, pattern> {
let Inst{28} = 0b1;
let Inst{25-24} = 0b11;
let Inst{23} = 0b0;
let Inst{21} = bit_21;
let Inst{20} = sz{0};
let Inst{11} = 0b1;
let Inst{8} = 0b1;
let Inst{6} = 0b1;
let Inst{4} = 0b1;
let Predicates = [HasMVEFloat];
let validForTailPredication = 1;
}
multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {
def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size, bit_4>;
let Predicates = [HasMVEFloat] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;
}
}
defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, fmaxnum, int_arm_mve_max_predicated>;
defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, fmaxnum, int_arm_mve_max_predicated>;
defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, fminnum, int_arm_mve_min_predicated>;
defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, fminnum, int_arm_mve_min_predicated>;
class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
bit bit_4, list<dag> pattern=[]>
: MVE_comp<NoItinerary, iname, suffix, "", size, pattern> {
let Inst{28} = U;
let Inst{25-24} = 0b11;
let Inst{23} = 0b0;
let Inst{21-20} = size{1-0};
let Inst{11} = 0b0;
let Inst{8} = 0b0;
let Inst{6} = 0b1;
let Inst{4} = bit_4;
let validForTailPredication = 1;
}
multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
SDNode Op, Intrinsic PredInt> {
def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
}
}
multiclass MVE_VMAX<MVEVectorVTInfo VTI>
: MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;
multiclass MVE_VMIN<MVEVectorVTInfo VTI>
: MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;
defm MVE_VMINs8 : MVE_VMIN<MVE_v16s8>;
defm MVE_VMINs16 : MVE_VMIN<MVE_v8s16>;
defm MVE_VMINs32 : MVE_VMIN<MVE_v4s32>;
defm MVE_VMINu8 : MVE_VMIN<MVE_v16u8>;
defm MVE_VMINu16 : MVE_VMIN<MVE_v8u16>;
defm MVE_VMINu32 : MVE_VMIN<MVE_v4u32>;
defm MVE_VMAXs8 : MVE_VMAX<MVE_v16s8>;
defm MVE_VMAXs16 : MVE_VMAX<MVE_v8s16>;
defm MVE_VMAXs32 : MVE_VMAX<MVE_v4s32>;
defm MVE_VMAXu8 : MVE_VMAX<MVE_v16u8>;
defm MVE_VMAXu16 : MVE_VMAX<MVE_v8u16>;
defm MVE_VMAXu32 : MVE_VMAX<MVE_v4u32>;
// end of mve_comp instructions
// start of mve_bit instructions
class MVE_bit_arith<dag oops, dag iops, string iname, string suffix,
string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>
: MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred_r, cstr, vecsize, pattern> {
bits<4> Qd;
bits<4> Qm;
let Inst{22} = Qd{3};
let Inst{15-13} = Qd{2-0};
let Inst{5} = Qm{3};
let Inst{3-1} = Qm{2-0};
}
def MVE_VBIC : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
"vbic", "", "$Qd, $Qn, $Qm", "", 0b00> {
bits<4> Qn;
let Inst{28} = 0b0;
let Inst{25-23} = 0b110;
let Inst{21-20} = 0b01;
let Inst{19-17} = Qn{2-0};
let Inst{16} = 0b0;
let Inst{12-8} = 0b00001;
let Inst{7} = Qn{3};
let Inst{6} = 0b1;
let Inst{4} = 0b1;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
class MVE_VREV<string iname, string suffix, bits<2> size, bits<2> bit_8_7,
bits<2> vecsize, string cstr="">
: MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm), iname,
suffix, "$Qd, $Qm", cstr, vecsize> {
let Inst{28} = 0b1;
let Inst{25-23} = 0b111;
let Inst{21-20} = 0b11;
let Inst{19-18} = size;
let Inst{17-16} = 0b00;
let Inst{12-9} = 0b0000;
let Inst{8-7} = bit_8_7;
let Inst{6} = 0b1;
let Inst{4} = 0b0;
let Inst{0} = 0b0;
}
def MVE_VREV64_8 : MVE_VREV<"vrev64", "8", 0b00, 0b00, 0b11, "@earlyclobber $Qd">;
def MVE_VREV64_16 : MVE_VREV<"vrev64", "16", 0b01, 0b00, 0b11, "@earlyclobber $Qd">;
def MVE_VREV64_32 : MVE_VREV<"vrev64", "32", 0b10, 0b00, 0b11, "@earlyclobber $Qd">;
def MVE_VREV32_8 : MVE_VREV<"vrev32", "8", 0b00, 0b01, 0b10>;
def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01, 0b10>;
def MVE_VREV16_8 : MVE_VREV<"vrev16", "8", 0b00, 0b10, 0b01>;
let Predicates = [HasMVEInt] in {
def : Pat<(v8i16 (bswap (v8i16 MQPR:$src))),
(v8i16 (MVE_VREV16_8 (v8i16 MQPR:$src)))>;
def : Pat<(v4i32 (bswap (v4i32 MQPR:$src))),
(v4i32 (MVE_VREV32_8 (v4i32 MQPR:$src)))>;
}
multiclass MVE_VREV_basic_patterns<int revbits, list<MVEVectorVTInfo> VTIs,
Instruction Inst> {
defvar unpred_op = !cast<SDNode>("ARMvrev" # revbits);
foreach VTI = VTIs in {
def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),
(VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;
def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),
revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
(VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,
(VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
}
}
let Predicates = [HasMVEInt] in {
defm: MVE_VREV_basic_patterns<64, [MVE_v4i32, MVE_v4f32], MVE_VREV64_32>;
defm: MVE_VREV_basic_patterns<64, [MVE_v8i16, MVE_v8f16], MVE_VREV64_16>;
defm: MVE_VREV_basic_patterns<64, [MVE_v16i8 ], MVE_VREV64_8>;
defm: MVE_VREV_basic_patterns<32, [MVE_v8i16, MVE_v8f16], MVE_VREV32_16>;
defm: MVE_VREV_basic_patterns<32, [MVE_v16i8 ], MVE_VREV32_8>;
defm: MVE_VREV_basic_patterns<16, [MVE_v16i8 ], MVE_VREV16_8>;
}
def MVE_VMVN : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm),
"vmvn", "", "$Qd, $Qm", "", 0b00> {
let Inst{28} = 0b1;
let Inst{25-23} = 0b111;
let Inst{21-16} = 0b110000;
let Inst{12-6} = 0b0010111;
let Inst{4} = 0b0;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
let Predicates = [HasMVEInt] in {
foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {
def : Pat<(VTI.Vec (vnotq (VTI.Vec MQPR:$val1))),
(VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;
def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),
(VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
(VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,
(VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
}
}
class MVE_bit_ops<string iname, bits<2> bit_21_20, bit bit_28>
: MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
iname, "", "$Qd, $Qn, $Qm", "", 0b00> {
bits<4> Qn;
let Inst{28} = bit_28;
let Inst{25-23} = 0b110;
let Inst{21-20} = bit_21_20;
let Inst{19-17} = Qn{2-0};
let Inst{16} = 0b0;
let Inst{12-8} = 0b00001;
let Inst{7} = Qn{3};
let Inst{6} = 0b1;
let Inst{4} = 0b1;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
def MVE_VEOR : MVE_bit_ops<"veor", 0b00, 0b1>;
def MVE_VORN : MVE_bit_ops<"vorn", 0b11, 0b0>;
def MVE_VORR : MVE_bit_ops<"vorr", 0b10, 0b0>;
def MVE_VAND : MVE_bit_ops<"vand", 0b00, 0b0>;
// add ignored suffixes as aliases
foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f32"] in {
def : MVEInstAlias<"vbic${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
(MVE_VBIC MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
def : MVEInstAlias<"veor${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
(MVE_VEOR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
def : MVEInstAlias<"vorn${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
(MVE_VORN MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
def : MVEInstAlias<"vorr${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
(MVE_VORR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
def : MVEInstAlias<"vand${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
(MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
}
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<MVE_v16i8, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
defm : MVE_TwoOpPattern<MVE_v8i16, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
defm : MVE_TwoOpPattern<MVE_v4i32, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
defm : MVE_TwoOpPattern<MVE_v2i64, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
defm : MVE_TwoOpPattern<MVE_v16i8, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v8i16, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v4i32, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v2i64, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v16i8, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v8i16, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v4i32, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v2i64, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_orn_predicated, (? ), MVE_VORN>;
defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_orn_predicated, (? ), MVE_VORN>;
defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_orn_predicated, (? ), MVE_VORN>;
defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
int_arm_mve_orn_predicated, (? ), MVE_VORN>;
}
class MVE_bit_cmode<string iname, string suffix, bit halfword, dag inOps, bits<2> vecsize>
: MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,
iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src", vecsize> {
bits<12> imm;
bits<4> Qd;
let Inst{28} = imm{7};
let Inst{27-23} = 0b11111;
let Inst{22} = Qd{3};
let Inst{21-19} = 0b000;
let Inst{18-16} = imm{6-4};
let Inst{15-13} = Qd{2-0};
let Inst{12} = 0b0;
let Inst{11} = halfword;
let Inst{10} = !if(halfword, 0, imm{10});
let Inst{9} = imm{9};
let Inst{8} = 0b1;
let Inst{7-6} = 0b01;
let Inst{4} = 0b1;
let Inst{3-0} = imm{3-0};
}
multiclass MVE_bit_cmode_p<string iname, bit opcode,
MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {
def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},
(ins MQPR:$Qd_src, imm_type:$imm), VTI.Size> {
let Inst{5} = opcode;
let validForTailPredication = 1;
}
defvar Inst = !cast<Instruction>(NAME);
defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));
let Predicates = [HasMVEInt] in {
def : Pat<UnpredPat,
(VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;
def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
UnpredPat, (VTI.Vec MQPR:$src))),
(VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,
ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
}
}
multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {
defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;
}
multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {
defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;
}
defm MVE_VORRimmi16 : MVE_VORRimm<MVE_v8i16, nImmSplatI16>;
defm MVE_VORRimmi32 : MVE_VORRimm<MVE_v4i32, nImmSplatI32>;
defm MVE_VBICimmi16 : MVE_VBICimm<MVE_v8i16, nImmSplatI16>;
defm MVE_VBICimmi32 : MVE_VBICimm<MVE_v4i32, nImmSplatI32>;
def MVE_VORNimmi16 : MVEInstAlias<"vorn${vp}.i16\t$Qd, $imm",
(MVE_VORRimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
def MVE_VORNimmi32 : MVEInstAlias<"vorn${vp}.i32\t$Qd, $imm",
(MVE_VORRimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
def MVE_VANDimmi16 : MVEInstAlias<"vand${vp}.i16\t$Qd, $imm",
(MVE_VBICimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
def MVE_VANDimmi32 : MVEInstAlias<"vand${vp}.i32\t$Qd, $imm",
(MVE_VBICimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
def MVE_VMOV : MVEInstAlias<"vmov${vp}\t$Qd, $Qm",
(MVE_VORR MQPR:$Qd, MQPR:$Qm, MQPR:$Qm, vpred_r:$vp)>;
class MVE_VMOV_lane_direction {
bit bit_20;
dag oops;
dag iops;
string ops;
string cstr;
}
def MVE_VMOV_from_lane : MVE_VMOV_lane_direction {
let bit_20 = 0b1;
let oops = (outs rGPR:$Rt);
let iops = (ins MQPR:$Qd);
let ops = "$Rt, $Qd$Idx";
let cstr = "";
}
def MVE_VMOV_to_lane : MVE_VMOV_lane_direction {
let bit_20 = 0b0;
let oops = (outs MQPR:$Qd);
let iops = (ins MQPR:$Qd_src, rGPR:$Rt);
let ops = "$Qd$Idx, $Rt";
let cstr = "$Qd = $Qd_src";
}
class MVE_VMOV_lane<string suffix, bit U, dag indexop,
MVE_VMOV_lane_direction dir>
: MVE_VMOV_lane_base<dir.oops, !con(dir.iops, indexop), NoItinerary,
"vmov", suffix, dir.ops, dir.cstr, []> {
bits<4> Qd;
bits<4> Rt;
let Inst{31-24} = 0b11101110;
let Inst{23} = U;
let Inst{20} = dir.bit_20;
let Inst{19-17} = Qd{2-0};
let Inst{15-12} = Rt{3-0};
let Inst{11-8} = 0b1011;
let Inst{7} = Qd{3};
let Inst{4-0} = 0b10000;
let hasSideEffects = 0;
}
class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>
: MVE_VMOV_lane<"32", 0b0, (ins MVEVectorIndex<4>:$Idx), dir> {
bits<2> Idx;
let Inst{22} = 0b0;
let Inst{6-5} = 0b00;
let Inst{16} = Idx{1};
let Inst{21} = Idx{0};
let VecSize = 0b10;
let Predicates = [HasFPRegsV8_1M];
}
class MVE_VMOV_lane_16<string suffix, bit U, MVE_VMOV_lane_direction dir>
: MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<8>:$Idx), dir> {
bits<3> Idx;
let Inst{22} = 0b0;
let Inst{5} = 0b1;
let Inst{16} = Idx{2};
let Inst{21} = Idx{1};
let Inst{6} = Idx{0};
let VecSize = 0b01;
}
class MVE_VMOV_lane_8<string suffix, bit U, MVE_VMOV_lane_direction dir>
: MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<16>:$Idx), dir> {
bits<4> Idx;
let Inst{22} = 0b1;
let Inst{16} = Idx{3};
let Inst{21} = Idx{2};
let Inst{6} = Idx{1};
let Inst{5} = Idx{0};
let VecSize = 0b00;
}
def MVE_VMOV_from_lane_32 : MVE_VMOV_lane_32< MVE_VMOV_from_lane>;
def MVE_VMOV_from_lane_s16 : MVE_VMOV_lane_16<"s16", 0b0, MVE_VMOV_from_lane>;
def MVE_VMOV_from_lane_u16 : MVE_VMOV_lane_16<"u16", 0b1, MVE_VMOV_from_lane>;
def MVE_VMOV_from_lane_s8 : MVE_VMOV_lane_8 < "s8", 0b0, MVE_VMOV_from_lane>;
def MVE_VMOV_from_lane_u8 : MVE_VMOV_lane_8 < "u8", 0b1, MVE_VMOV_from_lane>;
let isInsertSubreg = 1 in
def MVE_VMOV_to_lane_32 : MVE_VMOV_lane_32< MVE_VMOV_to_lane>;
def MVE_VMOV_to_lane_16 : MVE_VMOV_lane_16< "16", 0b0, MVE_VMOV_to_lane>;
def MVE_VMOV_to_lane_8 : MVE_VMOV_lane_8 < "8", 0b0, MVE_VMOV_to_lane>;
// This is the same as insertelt but allows the inserted value to be an i32 as
// will be used when it is the only legal type.
def ARMVecInsert : SDTypeProfile<1, 3, [
SDTCisVT<2, i32>, SDTCisSameAs<0, 1>, SDTCisPtrTy<3>
]>;
def ARMinsertelt : SDNode<"ISD::INSERT_VECTOR_ELT", ARMVecInsert>;
let Predicates = [HasMVEInt] in {
def : Pat<(extractelt (v2f64 MQPR:$src), imm:$lane),
(f64 (EXTRACT_SUBREG MQPR:$src, (DSubReg_f64_reg imm:$lane)))>;
def : Pat<(insertelt (v2f64 MQPR:$src1), DPR:$src2, imm:$lane),
(INSERT_SUBREG (v2f64 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), DPR:$src2, (DSubReg_f64_reg imm:$lane))>;
def : Pat<(extractelt (v4i32 MQPR:$src), imm:$lane),
(COPY_TO_REGCLASS
(i32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), rGPR)>;
def : Pat<(insertelt (v4i32 MQPR:$src1), rGPR:$src2, imm:$lane),
(MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$src2, imm:$lane)>;
// This tries to copy from one lane to another, without going via GPR regs
def : Pat<(insertelt (v4i32 MQPR:$src1), (extractelt (v4i32 MQPR:$src2), imm:$extlane), imm:$inslane),
(v4i32 (COPY_TO_REGCLASS
(INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src1), MQPR)),
(f32 (EXTRACT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src2), MQPR)),
(SSubReg_f32_reg imm:$extlane))),
(SSubReg_f32_reg imm:$inslane)),
MQPR))>;
def : Pat<(vector_insert (v16i8 MQPR:$src1), rGPR:$src2, imm:$lane),
(MVE_VMOV_to_lane_8 MQPR:$src1, rGPR:$src2, imm:$lane)>;
def : Pat<(vector_insert (v8i16 MQPR:$src1), rGPR:$src2, imm:$lane),
(MVE_VMOV_to_lane_16 MQPR:$src1, rGPR:$src2, imm:$lane)>;
def : Pat<(ARMvgetlanes (v16i8 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_s8 MQPR:$src, imm:$lane)>;
def : Pat<(ARMvgetlanes (v8i16 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
def : Pat<(ARMvgetlanes (v8f16 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
def : Pat<(ARMvgetlaneu (v16i8 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane)>;
def : Pat<(ARMvgetlaneu (v8i16 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
def : Pat<(ARMvgetlaneu (v8f16 MQPR:$src), imm:$lane),
(MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
// For i16's inserts being extracted from low lanes, then may use VINS.
def : Pat<(ARMinsertelt (v8i16 MQPR:$src1),
(ARMvgetlaneu (v8i16 MQPR:$src2), imm_even:$extlane),
imm_odd:$inslane),
(COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),
(VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$inslane)),
(EXTRACT_SUBREG MQPR:$src2, (SSubReg_f16_reg imm_even:$extlane))),
(SSubReg_f16_reg imm_odd:$inslane)), MQPR)>;
def : Pat<(v16i8 (scalar_to_vector GPR:$src)),
(MVE_VMOV_to_lane_8 (v16i8 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
def : Pat<(v8i16 (scalar_to_vector GPR:$src)),
(MVE_VMOV_to_lane_16 (v8i16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
def : Pat<(v4i32 (scalar_to_vector GPR:$src)),
(MVE_VMOV_to_lane_32 (v4i32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
// Floating point patterns, still enabled under HasMVEInt
def : Pat<(extractelt (v4f32 MQPR:$src), imm:$lane),
(COPY_TO_REGCLASS (f32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), SPR)>;
def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),
(INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;
def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_even:$lane),
(MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS (f16 HPR:$src2), rGPR), imm:$lane)>;
def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_odd:$lane),
(COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),
(VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$lane)),
(COPY_TO_REGCLASS HPR:$src2, SPR)),
(SSubReg_f16_reg imm_odd:$lane)), MQPR)>;
def : Pat<(extractelt (v8f16 MQPR:$src), imm_even:$lane),
(EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_even:$lane))>;
def : Pat<(extractelt (v8f16 MQPR:$src), imm_odd:$lane),
(COPY_TO_REGCLASS
(VMOVH (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_odd:$lane))),
HPR)>;
def : Pat<(v2f64 (scalar_to_vector (f64 DPR:$src))),
(INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), DPR:$src, dsub_0)>;
def : Pat<(v4f32 (scalar_to_vector SPR:$src)),
(INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, ssub_0)>;
def : Pat<(v4f32 (scalar_to_vector GPR:$src)),
(MVE_VMOV_to_lane_32 (v4f32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
def : Pat<(v8f16 (scalar_to_vector (f16 HPR:$src))),
(INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), (f16 HPR:$src), ssub_0)>;
def : Pat<(v8f16 (scalar_to_vector GPR:$src)),
(MVE_VMOV_to_lane_16 (v8f16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
}
// end of mve_bit instructions
// start of MVE Integer instructions
class MVE_int<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
: MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
iname, suffix, "$Qd, $Qn, $Qm", vpred_r, "", size, pattern> {
bits<4> Qd;
bits<4> Qn;
bits<4> Qm;
let Inst{22} = Qd{3};
let Inst{21-20} = size;
let Inst{19-17} = Qn{2-0};
let Inst{15-13} = Qd{2-0};
let Inst{7} = Qn{3};
let Inst{6} = 0b1;
let Inst{5} = Qm{3};
let Inst{3-1} = Qm{2-0};
}
class MVE_VMULt1<string iname, string suffix, bits<2> size,
list<dag> pattern=[]>
: MVE_int<iname, suffix, size, pattern> {
let Inst{28} = 0b0;
let Inst{25-23} = 0b110;
let Inst{16} = 0b0;
let Inst{12-8} = 0b01001;
let Inst{4} = 0b1;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {
def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),
!cast<Instruction>(NAME), ARMimmOneV>;
}
}
defm MVE_VMULi8 : MVE_VMUL_m<MVE_v16i8>;
defm MVE_VMULi16 : MVE_VMUL_m<MVE_v8i16>;
defm MVE_VMULi32 : MVE_VMUL_m<MVE_v4i32>;
class MVE_VQxDMULH_Base<string iname, string suffix, bits<2> size, bit rounding,
list<dag> pattern=[]>
: MVE_int<iname, suffix, size, pattern> {
let Inst{28} = rounding;
let Inst{25-23} = 0b110;
let Inst{16} = 0b0;
let Inst{12-8} = 0b01011;
let Inst{4} = 0b0;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
def MVEvqdmulh : SDNode<"ARMISD::VQDMULH", SDTIntBinOp>;
multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
SDNode Op, Intrinsic unpred_int, Intrinsic pred_int,
bit rounding> {
def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, pred_int, (? ), Inst>;
// Extra unpredicated multiply intrinsic patterns
def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
}
}
multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>
: MVE_VQxDMULH_m<iname, VTI, !if(rounding, null_frag,
MVEvqdmulh),
!if(rounding, int_arm_mve_vqrdmulh,
int_arm_mve_vqdmulh),
!if(rounding, int_arm_mve_qrdmulh_predicated,
int_arm_mve_qdmulh_predicated),
rounding>;
defm MVE_VQDMULHi8 : MVE_VQxDMULH<"vqdmulh", MVE_v16s8, 0b0>;
defm MVE_VQDMULHi16 : MVE_VQxDMULH<"vqdmulh", MVE_v8s16, 0b0>;
defm MVE_VQDMULHi32 : MVE_VQxDMULH<"vqdmulh", MVE_v4s32, 0b0>;
defm MVE_VQRDMULHi8 : MVE_VQxDMULH<"vqrdmulh", MVE_v16s8, 0b1>;
defm MVE_VQRDMULHi16 : MVE_VQxDMULH<"vqrdmulh", MVE_v8s16, 0b1>;
defm MVE_VQRDMULHi32 : MVE_VQxDMULH<"vqrdmulh", MVE_v4s32, 0b1>;
class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,
list<dag> pattern=[]>
: MVE_int<iname, suffix, size, pattern> {
let Inst{28} = subtract;
let Inst{25-23} = 0b110;
let Inst{16} = 0b0;
let Inst{12-8} = 0b01000;
let Inst{4} = 0b0;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
SDNode Op, Intrinsic PredInt> {
def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
}
}
multiclass MVE_VADD<MVEVectorVTInfo VTI>
: MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
multiclass MVE_VSUB<MVEVectorVTInfo VTI>
: MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
defm MVE_VADDi8 : MVE_VADD<MVE_v16i8>;
defm MVE_VADDi16 : MVE_VADD<MVE_v8i16>;
defm MVE_VADDi32 : MVE_VADD<MVE_v4i32>;
defm MVE_VSUBi8 : MVE_VSUB<MVE_v16i8>;
defm MVE_VSUBi16 : MVE_VSUB<MVE_v8i16>;
defm MVE_VSUBi32 : MVE_VSUB<MVE_v4i32>;
class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,
bits<2> size>
: MVE_int<iname, suffix, size, []> {
let Inst{28} = U;
let Inst{25-23} = 0b110;
let Inst{16} = 0b0;
let Inst{12-10} = 0b000;
let Inst{9} = subtract;
let Inst{8} = 0b0;
let Inst{4} = 0b1;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
class MVE_VQADD_<string suffix, bit U, bits<2> size>
: MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size>;
class MVE_VQSUB_<string suffix, bit U, bits<2> size>
: MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size>;
multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
SDNode Op, Intrinsic PredInt> {
def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
!cast<Instruction>(NAME)>;
}
}
multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>
: MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;
defm MVE_VQADDs8 : MVE_VQADD<MVE_v16s8, saddsat>;
defm MVE_VQADDs16 : MVE_VQADD<MVE_v8s16, saddsat>;
defm MVE_VQADDs32 : MVE_VQADD<MVE_v4s32, saddsat>;
defm MVE_VQADDu8 : MVE_VQADD<MVE_v16u8, uaddsat>;
defm MVE_VQADDu16 : MVE_VQADD<MVE_v8u16, uaddsat>;
defm MVE_VQADDu32 : MVE_VQADD<MVE_v4u32, uaddsat>;
multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
SDNode Op, Intrinsic PredInt> {
def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
!cast<Instruction>(NAME)>;
}
}
multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>
: MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;
defm MVE_VQSUBs8 : MVE_VQSUB<MVE_v16s8, ssubsat>;
defm MVE_VQSUBs16 : MVE_VQSUB<MVE_v8s16, ssubsat>;
defm MVE_VQSUBs32 : MVE_VQSUB<MVE_v4s32, ssubsat>;
defm MVE_VQSUBu8 : MVE_VQSUB<MVE_v16u8, usubsat>;
defm MVE_VQSUBu16 : MVE_VQSUB<MVE_v8u16, usubsat>;
defm MVE_VQSUBu32 : MVE_VQSUB<MVE_v4u32, usubsat>;
class MVE_VABD_int<string suffix, bit U, bits<2> size,
list<dag> pattern=[]>
: MVE_int<"vabd", suffix, size, pattern> {
let Inst{28} = U;
let Inst{25-23} = 0b110;
let Inst{16} = 0b0;
let Inst{12-8} = 0b00111;
let Inst{4} = 0b0;
let Inst{0} = 0b0;
let validForTailPredication = 1;
}
multiclass MVE_VABD_m<MVEVectorVTInfo VTI, SDNode Op,
Intrinsic unpred_int, Intrinsic PredInt> {
def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;
defvar Inst = !cast<Instruction>(NAME);
let Predicates = [HasMVEInt] in {
defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
!cast<Instruction>(NAME)>;
// Unpredicated absolute difference
def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
(i32 VTI.Unsigned))),
(VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
}
}
multiclass MVE_VABD<MVEVectorVTInfo VTI, SDNode Op>
: MVE_VABD_m<VTI, Op, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
defm MVE_VABDs8 : MVE_VABD<MVE_v16s8, abds>;
defm MVE_VABDs16 : MVE_VABD<MVE_v8s16, abds>;
defm MVE_VABDs32 : MVE_VABD<MVE_v4s32, abds>;
defm MVE_VABDu8 : MVE_VABD<MVE_v16u8, abdu>;
defm MVE_VABDu16 : MVE_VABD<MVE_v8u16, abdu>;
defm MVE_VABDu32 : MVE_VABD<MVE_v4u32, abdu>;
class MVE_VRHADD_Base<string suffix, bi