//==- SystemZInstrVector.td - SystemZ Vector instructions ------*- tblgen-*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

//===----------------------------------------------------------------------===//
// Move instructions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Register move.
  let isMoveReg = 1 in {
    def VLR : UnaryVRRa<"vlr", 0xE756, null_frag, v128any, v128any>;
    def VLR16 : UnaryAliasVRR<null_frag, v16hb, v16hb>;
    def VLR32 : UnaryAliasVRR<null_frag, v32sb, v32sb>;
    def VLR64 : UnaryAliasVRR<null_frag, v64db, v64db>;
  }

  // Load GR from VR element.
  def VLGV  : BinaryVRScGeneric<"vlgv", 0xE721>;
  def VLGVB : BinaryVRSc<"vlgvb", 0xE721, null_frag, v128b, 0>;
  def VLGVH : BinaryVRSc<"vlgvh", 0xE721, null_frag, v128h, 1>;
  def VLGVF : BinaryVRSc<"vlgvf", 0xE721, null_frag, v128f, 2>;
  def VLGVG : BinaryVRSc<"vlgvg", 0xE721, z_vector_extract, v128g, 3>;

  // Load VR element from GR.
  def VLVG  : TernaryVRSbGeneric<"vlvg", 0xE722>;
  def VLVGB : TernaryVRSb<"vlvgb", 0xE722, z_vector_insert,
                          v128b, v128b, GR32, 0>;
  def VLVGH : TernaryVRSb<"vlvgh", 0xE722, z_vector_insert,
                          v128h, v128h, GR32, 1>;
  def VLVGF : TernaryVRSb<"vlvgf", 0xE722, z_vector_insert,
                          v128f, v128f, GR32, 2>;
  def VLVGG : TernaryVRSb<"vlvgg", 0xE722, z_vector_insert,
                          v128g, v128g, GR64, 3>;

  // Load VR from GRs disjoint.
  def VLVGP : BinaryVRRf<"vlvgp", 0xE762, z_join_dwords, v128g>;
  def VLVGP32 : BinaryAliasVRRf<GR32>;
}

// Extractions always assign to the full GR64, even if the element would
// fit in the lower 32 bits.  Sub-i64 extracts therefore need to take a
// subreg of the result.
class VectorExtractSubreg<ValueType type, Instruction insn>
  : Pat<(i32 (z_vector_extract (type VR128:$vec), shift12only:$index)),
        (EXTRACT_SUBREG (insn VR128:$vec, shift12only:$index), subreg_l32)>;

def : VectorExtractSubreg<v16i8, VLGVB>;
def : VectorExtractSubreg<v8i16, VLGVH>;
def : VectorExtractSubreg<v4i32, VLGVF>;

//===----------------------------------------------------------------------===//
// Immediate instructions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  let isAsCheapAsAMove = 1, isMoveImm = 1, isReMaterializable = 1 in {

    // Generate byte mask.
    def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
    def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16_timm>;

    // Generate mask.
    def VGM  : BinaryVRIbGeneric<"vgm", 0xE746>;
    def VGMB : BinaryVRIb<"vgmb", 0xE746, z_rotate_mask, v128b, 0>;
    def VGMH : BinaryVRIb<"vgmh", 0xE746, z_rotate_mask, v128h, 1>;
    def VGMF : BinaryVRIb<"vgmf", 0xE746, z_rotate_mask, v128f, 2>;
    def VGMG : BinaryVRIb<"vgmg", 0xE746, z_rotate_mask, v128g, 3>;

    // Replicate immediate.
    def VREPI  : UnaryVRIaGeneric<"vrepi", 0xE745, imm32sx16>;
    def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16_timm, 0>;
    def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16_timm, 1>;
    def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16_timm, 2>;
    def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16_timm, 3>;
  }

  // Load element immediate.
  //
  // We want these instructions to be used ahead of VLVG* where possible.
  // However, VLVG* takes a variable BD-format index whereas VLEI takes
  // a plain immediate index.  This means that VLVG* has an extra "base"
  // register operand and is 3 units more complex.  Bumping the complexity
  // of the VLEI* instructions by 4 means that they are strictly better
  // than VLVG* in cases where both forms match.
  let AddedComplexity = 4 in {
    def VLEIB : TernaryVRIa<"vleib", 0xE740, z_vector_insert,
                            v128b, v128b, imm32sx16trunc, imm32zx4>;
    def VLEIH : TernaryVRIa<"vleih", 0xE741, z_vector_insert,
                            v128h, v128h, imm32sx16trunc, imm32zx3>;
    def VLEIF : TernaryVRIa<"vleif", 0xE743, z_vector_insert,
                            v128f, v128f, imm32sx16, imm32zx2>;
    def VLEIG : TernaryVRIa<"vleig", 0xE742, z_vector_insert,
                            v128g, v128g, imm64sx16, imm32zx1>;
  }
}

//===----------------------------------------------------------------------===//
// Loads
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Load.
  let SimpleBDXLoad = 1 in
      defm VL : UnaryVRXAlign<"vl", 0xE706>;

  // Load to block boundary.  The number of loaded bytes is only known
  // at run time.  The instruction is really polymorphic, but v128b matches
  // the return type of the associated intrinsic.
  def VLBB : BinaryVRX<"vlbb", 0xE707, int_s390_vlbb, v128b, 0>;

  // Load count to block boundary.
  let Defs = [CC] in
    def LCBB : InstRXE<0xE727, (outs GR32:$R1),
                               (ins (bdxaddr12only $B2, $D2, $X2):$XBD2, imm32zx4:$M3),
                       "lcbb\t$R1, $XBD2, $M3",
                       [(set GR32:$R1, (int_s390_lcbb bdxaddr12only:$XBD2,
                                                      imm32zx4_timm:$M3))]>;

  // Load with length.  The number of loaded bytes is only known at run time.
  def VLL : BinaryVRSb<"vll", 0xE737, int_s390_vll, 0>;

  // Load multiple.
  defm VLM : LoadMultipleVRSaAlign<"vlm", 0xE736>;

  // Load and replicate
  def VLREP  : UnaryVRXGeneric<"vlrep", 0xE705>;
  def VLREPB : UnaryVRX<"vlrepb", 0xE705, z_replicate_loadi8,  v128b, 1, 0>;
  def VLREPH : UnaryVRX<"vlreph", 0xE705, z_replicate_loadi16, v128h, 2, 1>;
  def VLREPF : UnaryVRX<"vlrepf", 0xE705, z_replicate_loadi32, v128f, 4, 2>;
  def VLREPG : UnaryVRX<"vlrepg", 0xE705, z_replicate_loadi64, v128g, 8, 3>;
  def : Pat<(v8f16 (z_replicate_loadf16 bdxaddr12only:$addr)),
            (VLREPH bdxaddr12only:$addr)>;
  def : Pat<(v4f32 (z_replicate_loadf32 bdxaddr12only:$addr)),
            (VLREPF bdxaddr12only:$addr)>;
  def : Pat<(v2f64 (z_replicate_loadf64 bdxaddr12only:$addr)),
            (VLREPG bdxaddr12only:$addr)>;

  // Use VLREP to load subvectors.  These patterns use "12pair" because
  // LEY and LDY offer full 20-bit displacement fields.  It's often better
  // to use those instructions rather than force a 20-bit displacement
  // into a GPR temporary.
  let mayLoad = 1, SimpleBDXLoad = 1, canFoldAsLoad = 1 in {
    def VL16 : UnaryAliasVRX<z_load, v16hb, bdxaddr12pair>;
    def VL32 : UnaryAliasVRX<z_load, v32sb, bdxaddr12pair>;
    def VL64 : UnaryAliasVRX<z_load, v64db, bdxaddr12pair>;
  }

  // Load logical element and zero.
  def VLLEZ  : UnaryVRXGeneric<"vllez", 0xE704>;
  def VLLEZB : UnaryVRX<"vllezb", 0xE704, z_vllezi8,  v128b, 1, 0>;
  def VLLEZH : UnaryVRX<"vllezh", 0xE704, z_vllezi16, v128h, 2, 1>;
  def VLLEZF : UnaryVRX<"vllezf", 0xE704, z_vllezi32, v128f, 4, 2>;
  def VLLEZG : UnaryVRX<"vllezg", 0xE704, z_vllezi64, v128g, 8, 3>;
  def : Pat<(z_vllezf32 bdxaddr12only:$addr),
            (VLLEZF bdxaddr12only:$addr)>;
  def : Pat<(z_vllezf64 bdxaddr12only:$addr),
            (VLLEZG bdxaddr12only:$addr)>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VLLEZLF : UnaryVRX<"vllezlf", 0xE704, z_vllezli32, v128f, 4, 6>;
    def : Pat<(z_vllezlf32 bdxaddr12only:$addr),
              (VLLEZLF bdxaddr12only:$addr)>;
  }

  // Load element.
  def VLEB : TernaryVRX<"vleb", 0xE700, z_vlei8,  v128b, v128b, 1, imm32zx4>;
  def VLEH : TernaryVRX<"vleh", 0xE701, z_vlei16, v128h, v128h, 2, imm32zx3>;
  def VLEF : TernaryVRX<"vlef", 0xE703, z_vlei32, v128f, v128f, 4, imm32zx2>;
  def VLEG : TernaryVRX<"vleg", 0xE702, z_vlei64, v128g, v128g, 8, imm32zx1>;
  def : Pat<(z_vlef32 (v4f32 VR128:$val), bdxaddr12only:$addr, imm32zx2:$index),
            (VLEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
  def : Pat<(z_vlef64 (v2f64 VR128:$val), bdxaddr12only:$addr, imm32zx1:$index),
            (VLEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;

  // Gather element.
  def VGEF : TernaryVRV<"vgef", 0xE713, 4, imm32zx2>;
  def VGEG : TernaryVRV<"vgeg", 0xE712, 8, imm32zx1>;
}

let Predicates = [FeatureVectorPackedDecimal] in {
  // Load rightmost with length.  The number of loaded bytes is only known
  // at run time.  Note that while the instruction will accept immediate
  // lengths larger that 15 at runtime, those will always result in a trap,
  // so we never emit them here.
  def VLRL : BinaryVSI<"vlrl", 0xE635, null_frag, 0>;
  def VLRLR : BinaryVRSd<"vlrlr", 0xE637, int_s390_vlrl, 0>;
  def : Pat<(int_s390_vlrl imm32zx4:$len, bdaddr12only:$addr),
            (VLRL bdaddr12only:$addr, imm32zx4:$len)>;
}

// Use replicating loads if we're inserting a single element into an
// undefined vector.  This avoids a false dependency on the previous
// register contents.
multiclass ReplicatePeephole<Instruction vlrep, ValueType vectype,
                             SDPatternOperator load, ValueType scalartype> {
  def : Pat<(vectype (z_vector_insert
                      (undef), (scalartype (load bdxaddr12only:$addr)), 0)),
            (vlrep bdxaddr12only:$addr)>;
  def : Pat<(vectype (scalar_to_vector
                      (scalartype (load bdxaddr12only:$addr)))),
            (vlrep bdxaddr12only:$addr)>;
}
defm : ReplicatePeephole<VLREPB, v16i8, z_anyextloadi8, i32>;
defm : ReplicatePeephole<VLREPH, v8i16, z_anyextloadi16, i32>;
defm : ReplicatePeephole<VLREPF, v4i32, z_load, i32>;
defm : ReplicatePeephole<VLREPG, v2i64, z_load, i64>;
defm : ReplicatePeephole<VLREPF, v4f32, z_load, f32>;
defm : ReplicatePeephole<VLREPG, v2f64, z_load, f64>;

//===----------------------------------------------------------------------===//
// Stores
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Store.
  let SimpleBDXStore = 1 in
      defm VST : StoreVRXAlign<"vst", 0xE70E>;

  // Store with length.  The number of stored bytes is only known at run time.
  def VSTL : StoreLengthVRSb<"vstl", 0xE73F, int_s390_vstl, 0>;

  // Store multiple.
  defm VSTM : StoreMultipleVRSaAlign<"vstm", 0xE73E>;

  // Store element.
  def VSTEB : StoreBinaryVRX<"vsteb", 0xE708, z_vstei8,  v128b, 1, imm32zx4>;
  def VSTEH : StoreBinaryVRX<"vsteh", 0xE709, z_vstei16, v128h, 2, imm32zx3>;
  def VSTEF : StoreBinaryVRX<"vstef", 0xE70B, z_vstei32, v128f, 4, imm32zx2>;
  def VSTEG : StoreBinaryVRX<"vsteg", 0xE70A, z_vstei64, v128g, 8, imm32zx1>;
  def : Pat<(z_vstef16 (v8f16 VR128:$val), bdxaddr12only:$addr,
                       imm32zx3:$index),
            (VSTEH VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
  def : Pat<(z_vstef32 (v4f32 VR128:$val), bdxaddr12only:$addr,
                       imm32zx2:$index),
            (VSTEF VR128:$val, bdxaddr12only:$addr, imm32zx2:$index)>;
  def : Pat<(z_vstef64 (v2f64 VR128:$val), bdxaddr12only:$addr,
                       imm32zx1:$index),
            (VSTEG VR128:$val, bdxaddr12only:$addr, imm32zx1:$index)>;

  // Use VSTE to store subvectors.  These patterns use "12pair" because
  // STEY and STDY offer full 20-bit displacement fields.  It's often better
  // to use those instructions rather than force a 20-bit displacement
  // into a GPR temporary.
  let mayStore = 1, SimpleBDXStore = 1 in {
    def VST16 : StoreAliasVRX<store, v16hb, bdxaddr12pair>;
    def VST32 : StoreAliasVRX<store, v32sb, bdxaddr12pair>;
    def VST64 : StoreAliasVRX<store, v64db, bdxaddr12pair>;
  }

  // Scatter element.
  def VSCEF : StoreBinaryVRV<"vscef", 0xE71B, 4, imm32zx2>;
  def VSCEG : StoreBinaryVRV<"vsceg", 0xE71A, 8, imm32zx1>;
}

let Predicates = [FeatureVectorPackedDecimal] in {
  // Store rightmost with length.  The number of stored bytes is only known
  // at run time.  Note that while the instruction will accept immediate
  // lengths larger that 15 at runtime, those will always result in a trap,
  // so we never emit them here.
  def VSTRL : StoreLengthVSI<"vstrl", 0xE63D, null_frag, 0>;
  def VSTRLR : StoreLengthVRSd<"vstrlr", 0xE63F, int_s390_vstrl, 0>;
  def : Pat<(int_s390_vstrl VR128:$val, imm32zx4:$len, bdaddr12only:$addr),
            (VSTRL VR128:$val, bdaddr12only:$addr, imm32zx4:$len)>;
}

//===----------------------------------------------------------------------===//
// Byte swaps
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVectorEnhancements2] in {
  // Load byte-reversed elements.
  def VLBR  : UnaryVRXGeneric<"vlbr", 0xE606>;
  def VLBRH : UnaryVRX<"vlbrh", 0xE606, z_loadbswap, v128h, 16, 1>;
  def VLBRF : UnaryVRX<"vlbrf", 0xE606, z_loadbswap, v128f, 16, 2>;
  def VLBRG : UnaryVRX<"vlbrg", 0xE606, z_loadbswap, v128g, 16, 3>;
  def VLBRQ : UnaryVRX<"vlbrq", 0xE606, z_loadbswap, v128q, 16, 4>;

  // Load elements reversed.
  def VLER  : UnaryVRXGeneric<"vler", 0xE607>;
  def VLERH : UnaryVRX<"vlerh", 0xE607, z_loadeswap, v128h, 16, 1>;
  def VLERF : UnaryVRX<"vlerf", 0xE607, z_loadeswap, v128f, 16, 2>;
  def VLERG : UnaryVRX<"vlerg", 0xE607, z_loadeswap, v128g, 16, 3>;
  def : Pat<(v8f16 (z_loadeswap bdxaddr12only:$addr)),
            (VLERH bdxaddr12only:$addr)>;
  def : Pat<(v4f32 (z_loadeswap bdxaddr12only:$addr)),
            (VLERF bdxaddr12only:$addr)>;
  def : Pat<(v2f64 (z_loadeswap bdxaddr12only:$addr)),
            (VLERG bdxaddr12only:$addr)>;
  def : Pat<(v16i8 (z_loadeswap bdxaddr12only:$addr)),
            (VLBRQ bdxaddr12only:$addr)>;

  // Load byte-reversed element.
  def VLEBRH : TernaryVRX<"vlebrh", 0xE601, z_vlebri16, v128h, v128h, 2, imm32zx3>;
  def VLEBRF : TernaryVRX<"vlebrf", 0xE603, z_vlebri32, v128f, v128f, 4, imm32zx2>;
  def VLEBRG : TernaryVRX<"vlebrg", 0xE602, z_vlebri64, v128g, v128g, 8, imm32zx1>;

  // Load byte-reversed element and zero.
  def VLLEBRZ  : UnaryVRXGeneric<"vllebrz", 0xE604>;
  def VLLEBRZH : UnaryVRX<"vllebrzh", 0xE604, z_vllebrzi16, v128h, 2, 1>;
  def VLLEBRZF : UnaryVRX<"vllebrzf", 0xE604, z_vllebrzi32, v128f, 4, 2>;
  def VLLEBRZG : UnaryVRX<"vllebrzg", 0xE604, z_vllebrzi64, v128g, 8, 3>;
  def VLLEBRZE : UnaryVRX<"vllebrze", 0xE604, z_vllebrzli32, v128f, 4, 6>;
  def : InstAlias<"lerv\t$V1, $XBD2",
                  (VLLEBRZE VR128:$V1, bdxaddr12only:$XBD2), 0>;
  def : InstAlias<"ldrv\t$V1, $XBD2",
                  (VLLEBRZG VR128:$V1, bdxaddr12only:$XBD2), 0>;

  // Load byte-reversed element and replicate.
  def VLBRREP  : UnaryVRXGeneric<"vlbrrep", 0xE605>;
  def VLBRREPH : UnaryVRX<"vlbrreph", 0xE605, z_replicate_loadbswapi16, v128h, 2, 1>;
  def VLBRREPF : UnaryVRX<"vlbrrepf", 0xE605, z_replicate_loadbswapi32, v128f, 4, 2>;
  def VLBRREPG : UnaryVRX<"vlbrrepg", 0xE605, z_replicate_loadbswapi64, v128g, 8, 3>;

  // Store byte-reversed elements.
  def VSTBR  : StoreVRXGeneric<"vstbr", 0xE60E>;
  def VSTBRH : StoreVRX<"vstbrh", 0xE60E, z_storebswap, v128h, 16, 1>;
  def VSTBRF : StoreVRX<"vstbrf", 0xE60E, z_storebswap, v128f, 16, 2>;
  def VSTBRG : StoreVRX<"vstbrg", 0xE60E, z_storebswap, v128g, 16, 3>;
  def VSTBRQ : StoreVRX<"vstbrq", 0xE60E, z_storebswap, v128q, 16, 4>;

  // Store elements reversed.
  def VSTER  : StoreVRXGeneric<"vster", 0xE60F>;
  def VSTERH : StoreVRX<"vsterh", 0xE60F, z_storeeswap, v128h, 16, 1>;
  def VSTERF : StoreVRX<"vsterf", 0xE60F, z_storeeswap, v128f, 16, 2>;
  def VSTERG : StoreVRX<"vsterg", 0xE60F, z_storeeswap, v128g, 16, 3>;
  def : Pat<(z_storeeswap (v8f16 VR128:$val), bdxaddr12only:$addr),
            (VSTERH VR128:$val, bdxaddr12only:$addr)>;
  def : Pat<(z_storeeswap (v4f32 VR128:$val), bdxaddr12only:$addr),
            (VSTERF VR128:$val, bdxaddr12only:$addr)>;
  def : Pat<(z_storeeswap (v2f64 VR128:$val), bdxaddr12only:$addr),
            (VSTERG VR128:$val, bdxaddr12only:$addr)>;
  def : Pat<(z_storeeswap (v16i8 VR128:$val), bdxaddr12only:$addr),
            (VSTBRQ VR128:$val, bdxaddr12only:$addr)>;

  // Store byte-reversed element.
  def VSTEBRH : StoreBinaryVRX<"vstebrh", 0xE609, z_vstebri16, v128h, 2, imm32zx3>;
  def VSTEBRF : StoreBinaryVRX<"vstebrf", 0xE60B, z_vstebri32, v128f, 4, imm32zx2>;
  def VSTEBRG : StoreBinaryVRX<"vstebrg", 0xE60A, z_vstebri64, v128g, 8, imm32zx1>;
  def : InstAlias<"sterv\t$V1, $XBD2",
                  (VSTEBRF VR128:$V1, bdxaddr12only:$XBD2, 0), 0>;
  def : InstAlias<"stdrv\t$V1, $XBD2",
                  (VSTEBRG VR128:$V1, bdxaddr12only:$XBD2, 0), 0>;
}

//===----------------------------------------------------------------------===//
// Selects and permutes
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Merge high.
  def VMRH:   BinaryVRRcGeneric<"vmrh", 0xE761>;
  def VMRHB : BinaryVRRc<"vmrhb", 0xE761, z_merge_high, v128b, v128b, 0>;
  def VMRHH : BinaryVRRc<"vmrhh", 0xE761, z_merge_high, v128h, v128h, 1>;
  def VMRHF : BinaryVRRc<"vmrhf", 0xE761, z_merge_high, v128f, v128f, 2>;
  def VMRHG : BinaryVRRc<"vmrhg", 0xE761, z_merge_high, v128g, v128g, 3>;
  def : BinaryRRWithType<VMRHH, VR128, z_merge_high, v8f16>;
  def : BinaryRRWithType<VMRHF, VR128, z_merge_high, v4f32>;
  def : BinaryRRWithType<VMRHG, VR128, z_merge_high, v2f64>;

  // Merge low.
  def VMRL:   BinaryVRRcGeneric<"vmrl", 0xE760>;
  def VMRLB : BinaryVRRc<"vmrlb", 0xE760, z_merge_low, v128b, v128b, 0>;
  def VMRLH : BinaryVRRc<"vmrlh", 0xE760, z_merge_low, v128h, v128h, 1>;
  def VMRLF : BinaryVRRc<"vmrlf", 0xE760, z_merge_low, v128f, v128f, 2>;
  def VMRLG : BinaryVRRc<"vmrlg", 0xE760, z_merge_low, v128g, v128g, 3>;
  def : BinaryRRWithType<VMRLH, VR128, z_merge_low, v8f16>;
  def : BinaryRRWithType<VMRLF, VR128, z_merge_low, v4f32>;
  def : BinaryRRWithType<VMRLG, VR128, z_merge_low, v2f64>;

  // Permute.
  def VPERM : TernaryVRRe<"vperm", 0xE78C, z_permute, v128b, v128b>;

  // Permute doubleword immediate.
  def VPDI : TernaryVRRc<"vpdi", 0xE784, z_permute_dwords, v128g, v128g>;

  // Bit Permute.
  let Predicates = [FeatureVectorEnhancements1] in
    def VBPERM : BinaryVRRc<"vbperm", 0xE785, int_s390_vbperm, v128g, v128b>;

  // Replicate.
  def VREP:   BinaryVRIcGeneric<"vrep", 0xE74D>;
  def VREPB : BinaryVRIc<"vrepb", 0xE74D, z_splat, v128b, v128b, 0>;
  def VREPH : BinaryVRIc<"vreph", 0xE74D, z_splat, v128h, v128h, 1>;
  def VREPF : BinaryVRIc<"vrepf", 0xE74D, z_splat, v128f, v128f, 2>;
  def VREPG : BinaryVRIc<"vrepg", 0xE74D, z_splat, v128g, v128g, 3>;
  def : Pat<(v8f16 (z_splat VR128:$vec, imm32zx16_timm:$index)),
            (VREPH VR128:$vec, imm32zx16:$index)>;
  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16_timm:$index)),
            (VREPF VR128:$vec, imm32zx16:$index)>;
  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16_timm:$index)),
            (VREPG VR128:$vec, imm32zx16:$index)>;

  // Select.
  def VSEL : TernaryVRRe<"vsel", 0xE78D, null_frag, v128any, v128any>;

  // Blend.
  let Predicates = [FeatureVectorEnhancements3] in {
    def VBLEND  : TernaryVRRdGeneric<"vblend", 0xE789>;
    def VBLENDB : TernaryVRRd<"vblendb", 0xE789, null_frag, v128b, v128b, 0>;
    def VBLENDH : TernaryVRRd<"vblendh", 0xE789, null_frag, v128h, v128h, 1>;
    def VBLENDF : TernaryVRRd<"vblendf", 0xE789, null_frag, v128f, v128f, 2>;
    def VBLENDG : TernaryVRRd<"vblendg", 0xE789, null_frag, v128g, v128g, 3>;
    def VBLENDQ : TernaryVRRd<"vblendq", 0xE789, null_frag, v128q, v128q, 4>;
  }
}

//===----------------------------------------------------------------------===//
// Widening and narrowing
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Pack
  def VPK  : BinaryVRRcGeneric<"vpk", 0xE794>;
  def VPKH : BinaryVRRc<"vpkh", 0xE794, z_pack, v128b, v128h, 1>;
  def VPKF : BinaryVRRc<"vpkf", 0xE794, z_pack, v128h, v128f, 2>;
  def VPKG : BinaryVRRc<"vpkg", 0xE794, z_pack, v128f, v128g, 3>;

  // Pack saturate.
  def  VPKS  : BinaryVRRbSPairGeneric<"vpks", 0xE797>;
  defm VPKSH : BinaryVRRbSPair<"vpksh", 0xE797, int_s390_vpksh, z_packs_cc,
                               v128b, v128h, 1>;
  defm VPKSF : BinaryVRRbSPair<"vpksf", 0xE797, int_s390_vpksf, z_packs_cc,
                               v128h, v128f, 2>;
  defm VPKSG : BinaryVRRbSPair<"vpksg", 0xE797, int_s390_vpksg, z_packs_cc,
                               v128f, v128g, 3>;

  // Pack saturate logical.
  def  VPKLS  : BinaryVRRbSPairGeneric<"vpkls", 0xE795>;
  defm VPKLSH : BinaryVRRbSPair<"vpklsh", 0xE795, int_s390_vpklsh, z_packls_cc,
                                v128b, v128h, 1>;
  defm VPKLSF : BinaryVRRbSPair<"vpklsf", 0xE795, int_s390_vpklsf, z_packls_cc,
                                v128h, v128f, 2>;
  defm VPKLSG : BinaryVRRbSPair<"vpklsg", 0xE795, int_s390_vpklsg, z_packls_cc,
                                v128f, v128g, 3>;

  // Sign-extend to doubleword.
  def VSEG  : UnaryVRRaGeneric<"vseg", 0xE75F>;
  def VSEGB : UnaryVRRa<"vsegb", 0xE75F, z_vsei8,  v128g, v128g, 0>;
  def VSEGH : UnaryVRRa<"vsegh", 0xE75F, z_vsei16, v128g, v128g, 1>;
  def VSEGF : UnaryVRRa<"vsegf", 0xE75F, z_vsei32, v128g, v128g, 2>;
  def : Pat<(z_vsei8_by_parts  (v16i8 VR128:$src)), (VSEGB VR128:$src)>;
  def : Pat<(z_vsei16_by_parts (v8i16 VR128:$src)), (VSEGH VR128:$src)>;
  def : Pat<(z_vsei32_by_parts (v4i32 VR128:$src)), (VSEGF VR128:$src)>;

  // Generate element masks.
  let Predicates = [FeatureVectorEnhancements3] in {
    def VGEM  : UnaryVRRaGeneric<"vgem", 0xE754>;
    def VGEMB : UnaryVRRa<"vgemb", 0xE754, int_s390_vgemb, v128b, v128h, 0>;
    def VGEMH : UnaryVRRa<"vgemh", 0xE754, int_s390_vgemh, v128h, v128b, 1>;
    def VGEMF : UnaryVRRa<"vgemf", 0xE754, int_s390_vgemf, v128f, v128b, 2>;
    def VGEMG : UnaryVRRa<"vgemg", 0xE754, int_s390_vgemg, v128g, v128b, 3>;
    def VGEMQ : UnaryVRRa<"vgemq", 0xE754, int_s390_vgemq, v128q, v128b, 4>;
  }

  // Unpack high.
  def VUPH  : UnaryVRRaGeneric<"vuph", 0xE7D7>;
  def VUPHB : UnaryVRRa<"vuphb", 0xE7D7, z_unpack_high, v128h, v128b, 0>;
  def VUPHH : UnaryVRRa<"vuphh", 0xE7D7, z_unpack_high, v128f, v128h, 1>;
  def VUPHF : UnaryVRRa<"vuphf", 0xE7D7, z_unpack_high, v128g, v128f, 2>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VUPHG : UnaryVRRa<"vuphg", 0xE7D7, z_unpack_high, v128q, v128g, 3>;

  // Unpack logical high.
  def VUPLH  : UnaryVRRaGeneric<"vuplh", 0xE7D5>;
  def VUPLHB : UnaryVRRa<"vuplhb", 0xE7D5, z_unpackl_high, v128h, v128b, 0>;
  def VUPLHH : UnaryVRRa<"vuplhh", 0xE7D5, z_unpackl_high, v128f, v128h, 1>;
  def VUPLHF : UnaryVRRa<"vuplhf", 0xE7D5, z_unpackl_high, v128g, v128f, 2>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VUPLHG : UnaryVRRa<"vuplhg", 0xE7D5, z_unpackl_high, v128q, v128g, 3>;

  // Unpack low.
  def VUPL   : UnaryVRRaGeneric<"vupl", 0xE7D6>;
  def VUPLB  : UnaryVRRa<"vuplb",  0xE7D6, z_unpack_low, v128h, v128b, 0>;
  def VUPLHW : UnaryVRRa<"vuplhw", 0xE7D6, z_unpack_low, v128f, v128h, 1>;
  def VUPLF  : UnaryVRRa<"vuplf",  0xE7D6, z_unpack_low, v128g, v128f, 2>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VUPLG  : UnaryVRRa<"vuplg",  0xE7D6, z_unpack_low, v128q, v128g, 3>;

  // Unpack logical low.
  def VUPLL  : UnaryVRRaGeneric<"vupll", 0xE7D4>;
  def VUPLLB : UnaryVRRa<"vupllb", 0xE7D4, z_unpackl_low, v128h, v128b, 0>;
  def VUPLLH : UnaryVRRa<"vupllh", 0xE7D4, z_unpackl_low, v128f, v128h, 1>;
  def VUPLLF : UnaryVRRa<"vupllf", 0xE7D4, z_unpackl_low, v128g, v128f, 2>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VUPLLG : UnaryVRRa<"vupllg", 0xE7D4, z_unpackl_low, v128q, v128g, 3>;
}

//===----------------------------------------------------------------------===//
// Instantiating generic operations for specific types.
//===----------------------------------------------------------------------===//

multiclass GenericVectorOps<ValueType type, ValueType inttype> {
  let Predicates = [FeatureVector] in {
    def : Pat<(type (load bdxaddr12only:$addr)),
              (VL bdxaddr12only:$addr)>;
    def : Pat<(store (type VR128:$src), bdxaddr12only:$addr),
              (VST VR128:$src, bdxaddr12only:$addr)>;
    def : Pat<(type (vselect (inttype VR128:$x), VR128:$y, VR128:$z)),
              (VSEL VR128:$y, VR128:$z, VR128:$x)>;
    def : Pat<(type (vselect (inttype (z_vnot VR128:$x)), VR128:$y, VR128:$z)),
              (VSEL VR128:$z, VR128:$y, VR128:$x)>;
  }
}

defm : GenericVectorOps<v16i8, v16i8>;
defm : GenericVectorOps<v8i16, v8i16>;
defm : GenericVectorOps<v4i32, v4i32>;
defm : GenericVectorOps<v2i64, v2i64>;
defm : GenericVectorOps<v8f16, v8i16>;
defm : GenericVectorOps<v4f32, v4i32>;
defm : GenericVectorOps<v2f64, v2i64>;

multiclass BlendVectorOps<ValueType type, ValueType inttype,
                          Instruction blend> {
  let Predicates = [FeatureVectorEnhancements3] in {
    def : Pat<(type (vselect (inttype (z_vicmpl_zero VR128:$x)),
                             VR128:$y, VR128:$z)),
              (blend VR128:$y, VR128:$z, VR128:$x)>;
    def : Pat<(type (vselect (inttype (z_vnot (z_vicmpl_zero VR128:$x))),
                             VR128:$y, VR128:$z)),
              (blend VR128:$z, VR128:$y, VR128:$x)>;
  }
}

defm : BlendVectorOps<v16i8, v16i8, VBLENDB>;
defm : BlendVectorOps<v8i16, v8i16, VBLENDH>;
defm : BlendVectorOps<v4i32, v4i32, VBLENDF>;
defm : BlendVectorOps<v2i64, v2i64, VBLENDG>;
defm : BlendVectorOps<v8f16, v8i16, VBLENDH>;
defm : BlendVectorOps<v4f32, v4i32, VBLENDF>;
defm : BlendVectorOps<v2f64, v2i64, VBLENDG>;

let Predicates = [FeatureVectorEnhancements3] in {
    def : Pat<(i128 (or (and VR128:$y, (z_vicmph 0, VR128:$x)),
                        (and VR128:$z, (not (z_vicmph 0, VR128:$x))))),
              (VBLENDQ VR128:$y, VR128:$z, VR128:$x)>;
}

//===----------------------------------------------------------------------===//
// Integer arithmetic
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  let isCommutable = 1 in {
    // Add.
    def VA  : BinaryVRRcGeneric<"va", 0xE7F3>;
    def VAB : BinaryVRRc<"vab", 0xE7F3, add, v128b, v128b, 0>;
    def VAH : BinaryVRRc<"vah", 0xE7F3, add, v128h, v128h, 1>;
    def VAF : BinaryVRRc<"vaf", 0xE7F3, add, v128f, v128f, 2>;
    def VAG : BinaryVRRc<"vag", 0xE7F3, add, v128g, v128g, 3>;
    def VAQ : BinaryVRRc<"vaq", 0xE7F3, add, v128q, v128q, 4>;
  }

  let isCommutable = 1 in {
    // Add compute carry.
    def VACC  : BinaryVRRcGeneric<"vacc", 0xE7F1>;
    def VACCB : BinaryVRRc<"vaccb", 0xE7F1, z_vacc, v128b, v128b, 0>;
    def VACCH : BinaryVRRc<"vacch", 0xE7F1, z_vacc, v128h, v128h, 1>;
    def VACCF : BinaryVRRc<"vaccf", 0xE7F1, z_vacc, v128f, v128f, 2>;
    def VACCG : BinaryVRRc<"vaccg", 0xE7F1, z_vacc, v128g, v128g, 3>;
    def VACCQ : BinaryVRRc<"vaccq", 0xE7F1, z_vacc, v128q, v128q, 4>;

    // Add with carry.
    def VAC  : TernaryVRRdGeneric<"vac", 0xE7BB>;
    def VACQ : TernaryVRRd<"vacq", 0xE7BB, z_vac, v128q, v128q, 4>;

    // Add with carry compute carry.
    def VACCC  : TernaryVRRdGeneric<"vaccc", 0xE7B9>;
    def VACCCQ : TernaryVRRd<"vacccq", 0xE7B9, z_vaccc, v128q, v128q, 4>;
  }

  // And.
  let isCommutable = 1 in
    def VN : BinaryVRRc<"vn", 0xE768, null_frag, v128any, v128any>;

  // And with complement.
  def VNC : BinaryVRRc<"vnc", 0xE769, null_frag, v128any, v128any>;

  let isCommutable = 1 in {
    // Average.
    def VAVG  : BinaryVRRcGeneric<"vavg", 0xE7F2>;
    def VAVGB : BinaryVRRc<"vavgb", 0xE7F2, int_s390_vavgb, v128b, v128b, 0>;
    def VAVGH : BinaryVRRc<"vavgh", 0xE7F2, int_s390_vavgh, v128h, v128h, 1>;
    def VAVGF : BinaryVRRc<"vavgf", 0xE7F2, int_s390_vavgf, v128f, v128f, 2>;
    def VAVGG : BinaryVRRc<"vavgg", 0xE7F2, int_s390_vavgg, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VAVGQ : BinaryVRRc<"vavgq", 0xE7F2, int_s390_vavgq, v128q, v128q, 4>;

    // Average logical.
    def VAVGL  : BinaryVRRcGeneric<"vavgl", 0xE7F0>;
    def VAVGLB : BinaryVRRc<"vavglb", 0xE7F0, int_s390_vavglb, v128b, v128b, 0>;
    def VAVGLH : BinaryVRRc<"vavglh", 0xE7F0, int_s390_vavglh, v128h, v128h, 1>;
    def VAVGLF : BinaryVRRc<"vavglf", 0xE7F0, int_s390_vavglf, v128f, v128f, 2>;
    def VAVGLG : BinaryVRRc<"vavglg", 0xE7F0, int_s390_vavglg, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VAVGLQ : BinaryVRRc<"vavglq", 0xE7F0, int_s390_vavglq, v128q, v128q, 4>;
  }

  // Checksum.
  def VCKSM : BinaryVRRc<"vcksm", 0xE766, int_s390_vcksm, v128f, v128f>;

  // Count leading zeros.
  def VCLZ  : UnaryVRRaGeneric<"vclz", 0xE753>;
  def VCLZB : UnaryVRRa<"vclzb", 0xE753, ctlz, v128b, v128b, 0>;
  def VCLZH : UnaryVRRa<"vclzh", 0xE753, ctlz, v128h, v128h, 1>;
  def VCLZF : UnaryVRRa<"vclzf", 0xE753, ctlz, v128f, v128f, 2>;
  def VCLZG : UnaryVRRa<"vclzg", 0xE753, ctlz, v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VCLZQ : UnaryVRRa<"vclzq", 0xE753, ctlz, v128q, v128q, 4>;

  // Count trailing zeros.
  def VCTZ  : UnaryVRRaGeneric<"vctz", 0xE752>;
  def VCTZB : UnaryVRRa<"vctzb", 0xE752, cttz, v128b, v128b, 0>;
  def VCTZH : UnaryVRRa<"vctzh", 0xE752, cttz, v128h, v128h, 1>;
  def VCTZF : UnaryVRRa<"vctzf", 0xE752, cttz, v128f, v128f, 2>;
  def VCTZG : UnaryVRRa<"vctzg", 0xE752, cttz, v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VCTZQ : UnaryVRRa<"vctzq", 0xE752, cttz, v128q, v128q, 4>;

  // Divide.
  let Predicates = [FeatureVectorEnhancements3] in {
    let hasSideEffects = 1 in {
      def VD  : TernaryVRRcIntGeneric<"vd", 0xE7B2>;
      def VDF : TernaryVRRcInt<"vdf", 0xE7B2, null_frag, v128f, v128f, 2>;
      def VDG : TernaryVRRcInt<"vdg", 0xE7B2, null_frag, v128g, v128g, 3>;
      def VDQ : TernaryVRRcInt<"vdq", 0xE7B2, null_frag, v128q, v128q, 4>;
    }
    def : Pat<(v4i32 (sdiv VR128:$x, VR128:$y)), (VDF VR128:$x, VR128:$y, 0)>;
    def : Pat<(v2i64 (sdiv VR128:$x, VR128:$y)), (VDG VR128:$x, VR128:$y, 0)>;
    def : Pat<(i128 (sdiv VR128:$x, VR128:$y)), (VDQ VR128:$x, VR128:$y, 0)>;
  }

  // Divide logical.
  let Predicates = [FeatureVectorEnhancements3] in {
    let hasSideEffects = 1 in {
      def VDL  : TernaryVRRcIntGeneric<"vdl", 0xE7B0>;
      def VDLF : TernaryVRRcInt<"vdlf", 0xE7B0, null_frag, v128f, v128f, 2>;
      def VDLG : TernaryVRRcInt<"vdlg", 0xE7B0, null_frag, v128g, v128g, 3>;
      def VDLQ : TernaryVRRcInt<"vdlq", 0xE7B0, null_frag, v128q, v128q, 4>;
    }
    def : Pat<(v4i32 (udiv VR128:$x, VR128:$y)), (VDLF VR128:$x, VR128:$y, 0)>;
    def : Pat<(v2i64 (udiv VR128:$x, VR128:$y)), (VDLG VR128:$x, VR128:$y, 0)>;
    def : Pat<(i128 (udiv VR128:$x, VR128:$y)), (VDLQ VR128:$x, VR128:$y, 0)>;
  }

  // Evaluate.
  let Predicates = [FeatureVectorEnhancements3] in
    def VEVAL : QuaternaryVRIk<"veval", 0xE788, int_s390_veval, v128b>;

  let isCommutable = 1 in {
    // Not exclusive or.
    let Predicates = [FeatureVectorEnhancements1] in
      def VNX : BinaryVRRc<"vnx", 0xE76C, null_frag, v128any, v128any>;

    // Exclusive or.
    def VX : BinaryVRRc<"vx", 0xE76D, null_frag, v128any, v128any>;
  }

  // Galois field multiply sum.
  def VGFM  : BinaryVRRcGeneric<"vgfm", 0xE7B4>;
  def VGFMB : BinaryVRRc<"vgfmb", 0xE7B4, int_s390_vgfmb, v128h, v128b, 0>;
  def VGFMH : BinaryVRRc<"vgfmh", 0xE7B4, int_s390_vgfmh, v128f, v128h, 1>;
  def VGFMF : BinaryVRRc<"vgfmf", 0xE7B4, int_s390_vgfmf, v128g, v128f, 2>;
  def VGFMG : BinaryVRRc<"vgfmg", 0xE7B4, int_s390_vgfmg, v128q, v128g, 3>;

  // Galois field multiply sum and accumulate.
  def VGFMA  : TernaryVRRdGeneric<"vgfma", 0xE7BC>;
  def VGFMAB : TernaryVRRd<"vgfmab", 0xE7BC, int_s390_vgfmab, v128h, v128b, 0>;
  def VGFMAH : TernaryVRRd<"vgfmah", 0xE7BC, int_s390_vgfmah, v128f, v128h, 1>;
  def VGFMAF : TernaryVRRd<"vgfmaf", 0xE7BC, int_s390_vgfmaf, v128g, v128f, 2>;
  def VGFMAG : TernaryVRRd<"vgfmag", 0xE7BC, int_s390_vgfmag, v128q, v128g, 3>;

  // Load complement.
  def VLC  : UnaryVRRaGeneric<"vlc", 0xE7DE>;
  def VLCB : UnaryVRRa<"vlcb", 0xE7DE, z_vneg, v128b, v128b, 0>;
  def VLCH : UnaryVRRa<"vlch", 0xE7DE, z_vneg, v128h, v128h, 1>;
  def VLCF : UnaryVRRa<"vlcf", 0xE7DE, z_vneg, v128f, v128f, 2>;
  def VLCG : UnaryVRRa<"vlcg", 0xE7DE, z_vneg, v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VLCQ : UnaryVRRa<"vlcq", 0xE7DE, ineg, v128q, v128q, 4>;

  // Load positive.
  def VLP  : UnaryVRRaGeneric<"vlp", 0xE7DF>;
  def VLPB : UnaryVRRa<"vlpb", 0xE7DF, abs, v128b, v128b, 0>;
  def VLPH : UnaryVRRa<"vlph", 0xE7DF, abs, v128h, v128h, 1>;
  def VLPF : UnaryVRRa<"vlpf", 0xE7DF, abs, v128f, v128f, 2>;
  def VLPG : UnaryVRRa<"vlpg", 0xE7DF, abs, v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    def VLPQ : UnaryVRRa<"vlpq", 0xE7DF, abs, v128q, v128q, 4>;

  let isCommutable = 1 in {
    // Maximum.
    def VMX  : BinaryVRRcGeneric<"vmx", 0xE7FF>;
    def VMXB : BinaryVRRc<"vmxb", 0xE7FF, smax, v128b, v128b, 0>;
    def VMXH : BinaryVRRc<"vmxh", 0xE7FF, smax, v128h, v128h, 1>;
    def VMXF : BinaryVRRc<"vmxf", 0xE7FF, smax, v128f, v128f, 2>;
    def VMXG : BinaryVRRc<"vmxg", 0xE7FF, smax, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMXQ : BinaryVRRc<"vmxq", 0xE7FF, smax, v128q, v128q, 4>;

    // Maximum logical.
    def VMXL  : BinaryVRRcGeneric<"vmxl", 0xE7FD>;
    def VMXLB : BinaryVRRc<"vmxlb", 0xE7FD, umax, v128b, v128b, 0>;
    def VMXLH : BinaryVRRc<"vmxlh", 0xE7FD, umax, v128h, v128h, 1>;
    def VMXLF : BinaryVRRc<"vmxlf", 0xE7FD, umax, v128f, v128f, 2>;
    def VMXLG : BinaryVRRc<"vmxlg", 0xE7FD, umax, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMXLQ : BinaryVRRc<"vmxlq", 0xE7FD, umax, v128q, v128q, 4>;
  }

  let isCommutable = 1 in {
    // Minimum.
    def VMN  : BinaryVRRcGeneric<"vmn", 0xE7FE>;
    def VMNB : BinaryVRRc<"vmnb", 0xE7FE, smin, v128b, v128b, 0>;
    def VMNH : BinaryVRRc<"vmnh", 0xE7FE, smin, v128h, v128h, 1>;
    def VMNF : BinaryVRRc<"vmnf", 0xE7FE, smin, v128f, v128f, 2>;
    def VMNG : BinaryVRRc<"vmng", 0xE7FE, smin, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMNQ : BinaryVRRc<"vmnq", 0xE7FE, smin, v128q, v128q, 4>;

    // Minimum logical.
    def VMNL  : BinaryVRRcGeneric<"vmnl", 0xE7FC>;
    def VMNLB : BinaryVRRc<"vmnlb", 0xE7FC, umin, v128b, v128b, 0>;
    def VMNLH : BinaryVRRc<"vmnlh", 0xE7FC, umin, v128h, v128h, 1>;
    def VMNLF : BinaryVRRc<"vmnlf", 0xE7FC, umin, v128f, v128f, 2>;
    def VMNLG : BinaryVRRc<"vmnlg", 0xE7FC, umin, v128g, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMNLQ : BinaryVRRc<"vmnlq", 0xE7FC, umin, v128q, v128q, 4>;
  }

  let isCommutable = 1 in {
    // Multiply and add low.
    def VMAL   : TernaryVRRdGeneric<"vmal", 0xE7AA>;
    def VMALB  : TernaryVRRd<"vmalb",  0xE7AA, z_muladd<mul>, v128b, v128b, 0>;
    def VMALHW : TernaryVRRd<"vmalhw", 0xE7AA, z_muladd<mul>, v128h, v128h, 1>;
    def VMALF  : TernaryVRRd<"vmalf",  0xE7AA, z_muladd<mul>, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMALG : TernaryVRRd<"vmalg",  0xE7AA, z_muladd<mul>, v128g, v128g, 3>;
      def VMALQ : TernaryVRRd<"vmalq",  0xE7AA, z_muladd<mul>, v128q, v128q, 4>;
    }

    // Multiply and add high.
    def VMAH  : TernaryVRRdGeneric<"vmah", 0xE7AB>;
    def VMAHB : TernaryVRRd<"vmahb", 0xE7AB, z_vmah, v128b, v128b, 0>;
    def VMAHH : TernaryVRRd<"vmahh", 0xE7AB, z_vmah, v128h, v128h, 1>;
    def VMAHF : TernaryVRRd<"vmahf", 0xE7AB, z_vmah, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMAHG : TernaryVRRd<"vmahg", 0xE7AB, z_vmah, v128g, v128g, 3>;
      def VMAHQ : TernaryVRRd<"vmahq", 0xE7AB, z_vmah, v128q, v128q, 4>;
    }

    // Multiply and add logical high.
    def VMALH  : TernaryVRRdGeneric<"vmalh", 0xE7A9>;
    def VMALHB : TernaryVRRd<"vmalhb", 0xE7A9, z_vmalh, v128b, v128b, 0>;
    def VMALHH : TernaryVRRd<"vmalhh", 0xE7A9, z_vmalh, v128h, v128h, 1>;
    def VMALHF : TernaryVRRd<"vmalhf", 0xE7A9, z_vmalh, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMALHG : TernaryVRRd<"vmalhg", 0xE7A9, z_vmalh, v128g, v128g, 3>;
      def VMALHQ : TernaryVRRd<"vmalhq", 0xE7A9, z_vmalh, v128q, v128q, 4>;
    }

    // Multiply and add even.
    def VMAE  : TernaryVRRdGeneric<"vmae", 0xE7AE>;
    def VMAEB : TernaryVRRd<"vmaeb", 0xE7AE, z_muladd<z_vme>, v128h, v128b, 0>;
    def VMAEH : TernaryVRRd<"vmaeh", 0xE7AE, z_muladd<z_vme>, v128f, v128h, 1>;
    def VMAEF : TernaryVRRd<"vmaef", 0xE7AE, z_muladd<z_vme>, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMAEG : TernaryVRRd<"vmaeg", 0xE7AE, z_muladd<z_vme>, v128q, v128g, 3>;

    // Multiply and add logical even.
    def VMALE  : TernaryVRRdGeneric<"vmale", 0xE7AC>;
    def VMALEB : TernaryVRRd<"vmaleb", 0xE7AC, z_muladd<z_vmle>, v128h, v128b, 0>;
    def VMALEH : TernaryVRRd<"vmaleh", 0xE7AC, z_muladd<z_vmle>, v128f, v128h, 1>;
    def VMALEF : TernaryVRRd<"vmalef", 0xE7AC, z_muladd<z_vmle>, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMALEG : TernaryVRRd<"vmaleg", 0xE7AC, z_muladd<z_vmle>, v128q, v128g, 3>;

    // Multiply and add odd.
    def VMAO  : TernaryVRRdGeneric<"vmao", 0xE7AF>;
    def VMAOB : TernaryVRRd<"vmaob", 0xE7AF, z_muladd<z_vmo>, v128h, v128b, 0>;
    def VMAOH : TernaryVRRd<"vmaoh", 0xE7AF, z_muladd<z_vmo>, v128f, v128h, 1>;
    def VMAOF : TernaryVRRd<"vmaof", 0xE7AF, z_muladd<z_vmo>, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMAOG : TernaryVRRd<"vmaog", 0xE7AF, z_muladd<z_vmo>, v128q, v128g, 3>;

    // Multiply and add logical odd.
    def VMALO  : TernaryVRRdGeneric<"vmalo", 0xE7AD>;
    def VMALOB : TernaryVRRd<"vmalob", 0xE7AD, z_muladd<z_vmlo>, v128h, v128b, 0>;
    def VMALOH : TernaryVRRd<"vmaloh", 0xE7AD, z_muladd<z_vmlo>, v128f, v128h, 1>;
    def VMALOF : TernaryVRRd<"vmalof", 0xE7AD, z_muladd<z_vmlo>, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMALOG : TernaryVRRd<"vmalog", 0xE7AD, z_muladd<z_vmlo>, v128q, v128g, 3>;
  }

  let isCommutable = 1 in {
    // Multiply high.
    def VMH  : BinaryVRRcGeneric<"vmh", 0xE7A3>;
    def VMHB : BinaryVRRc<"vmhb", 0xE7A3, mulhs, v128b, v128b, 0>;
    def VMHH : BinaryVRRc<"vmhh", 0xE7A3, mulhs, v128h, v128h, 1>;
    def VMHF : BinaryVRRc<"vmhf", 0xE7A3, mulhs, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMHG : BinaryVRRc<"vmhg", 0xE7A3, mulhs, v128g, v128g, 3>;
      def VMHQ : BinaryVRRc<"vmhq", 0xE7A3, mulhs, v128q, v128q, 4>;
    }

    // Multiply logical high.
    def VMLH  : BinaryVRRcGeneric<"vmlh", 0xE7A1>;
    def VMLHB : BinaryVRRc<"vmlhb", 0xE7A1, mulhu, v128b, v128b, 0>;
    def VMLHH : BinaryVRRc<"vmlhh", 0xE7A1, mulhu, v128h, v128h, 1>;
    def VMLHF : BinaryVRRc<"vmlhf", 0xE7A1, mulhu, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMLHG : BinaryVRRc<"vmlhg", 0xE7A1, mulhu, v128g, v128g, 3>;
      def VMLHQ : BinaryVRRc<"vmlhq", 0xE7A1, mulhu, v128q, v128q, 4>;
    }

    // Multiply low.
    def VML   : BinaryVRRcGeneric<"vml", 0xE7A2>;
    def VMLB  : BinaryVRRc<"vmlb",  0xE7A2, mul, v128b, v128b, 0>;
    def VMLHW : BinaryVRRc<"vmlhw", 0xE7A2, mul, v128h, v128h, 1>;
    def VMLF  : BinaryVRRc<"vmlf",  0xE7A2, mul, v128f, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in {
      def VMLG : BinaryVRRc<"vmlg",  0xE7A2, mul, v128g, v128g, 3>;
      def VMLQ : BinaryVRRc<"vmlq",  0xE7A2, mul, v128q, v128q, 4>;
    }

    // Multiply even.
    def VME  : BinaryVRRcGeneric<"vme", 0xE7A6>;
    def VMEB : BinaryVRRc<"vmeb", 0xE7A6, z_vme, v128h, v128b, 0>;
    def VMEH : BinaryVRRc<"vmeh", 0xE7A6, z_vme, v128f, v128h, 1>;
    def VMEF : BinaryVRRc<"vmef", 0xE7A6, z_vme, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMEG : BinaryVRRc<"vmeg", 0xE7A6, z_vme, v128q, v128g, 3>;

    // Multiply logical even.
    def VMLE  : BinaryVRRcGeneric<"vmle", 0xE7A4>;
    def VMLEB : BinaryVRRc<"vmleb", 0xE7A4, z_vmle, v128h, v128b, 0>;
    def VMLEH : BinaryVRRc<"vmleh", 0xE7A4, z_vmle, v128f, v128h, 1>;
    def VMLEF : BinaryVRRc<"vmlef", 0xE7A4, z_vmle, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMLEG : BinaryVRRc<"vmleg", 0xE7A4, z_vmle, v128q, v128g, 3>;

    // Multiply odd.
    def VMO  : BinaryVRRcGeneric<"vmo", 0xE7A7>;
    def VMOB : BinaryVRRc<"vmob", 0xE7A7, z_vmo, v128h, v128b, 0>;
    def VMOH : BinaryVRRc<"vmoh", 0xE7A7, z_vmo, v128f, v128h, 1>;
    def VMOF : BinaryVRRc<"vmof", 0xE7A7, z_vmo, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMOG : BinaryVRRc<"vmog", 0xE7A7, z_vmo, v128q, v128g, 3>;

    // Multiply logical odd.
    def VMLO  : BinaryVRRcGeneric<"vmlo", 0xE7A5>;
    def VMLOB : BinaryVRRc<"vmlob", 0xE7A5, z_vmlo, v128h, v128b, 0>;
    def VMLOH : BinaryVRRc<"vmloh", 0xE7A5, z_vmlo, v128f, v128h, 1>;
    def VMLOF : BinaryVRRc<"vmlof", 0xE7A5, z_vmlo, v128g, v128f, 2>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VMLOG : BinaryVRRc<"vmlog", 0xE7A5, z_vmlo, v128q, v128g, 3>;
  }

  // Multiply sum logical.
  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in {
    def VMSL  : QuaternaryVRRdGeneric<"vmsl", 0xE7B8>;
    def VMSLG : QuaternaryVRRd<"vmslg", 0xE7B8, int_s390_vmslg,
                               v128q, v128g, v128g, v128q, 3>;
  }

  // Nand.
  let Predicates = [FeatureVectorEnhancements1], isCommutable = 1 in
    def VNN : BinaryVRRc<"vnn", 0xE76E, null_frag, v128any, v128any>;

  // Nor.
  let isCommutable = 1 in
    def VNO : BinaryVRRc<"vno", 0xE76B, null_frag, v128any, v128any>;
  def : InstAlias<"vnot\t$V1, $V2", (VNO VR128:$V1, VR128:$V2, VR128:$V2), 0>;

  // Or.
  let isCommutable = 1 in
    def VO : BinaryVRRc<"vo", 0xE76A, null_frag, v128any, v128any>;

  // Or with complement.
  let Predicates = [FeatureVectorEnhancements1] in
    def VOC : BinaryVRRc<"voc", 0xE76F, null_frag, v128any, v128any>;

  // Population count.
  def VPOPCT : UnaryVRRaGeneric<"vpopct", 0xE750>;
  def : Pat<(v16i8 (z_popcnt VR128:$x)), (VPOPCT VR128:$x, 0)>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VPOPCTB : UnaryVRRa<"vpopctb", 0xE750, ctpop, v128b, v128b, 0>;
    def VPOPCTH : UnaryVRRa<"vpopcth", 0xE750, ctpop, v128h, v128h, 1>;
    def VPOPCTF : UnaryVRRa<"vpopctf", 0xE750, ctpop, v128f, v128f, 2>;
    def VPOPCTG : UnaryVRRa<"vpopctg", 0xE750, ctpop, v128g, v128g, 3>;
  }

  // Remainder.
  let Predicates = [FeatureVectorEnhancements3] in {
    let hasSideEffects = 1 in {
      def VR  : TernaryVRRcIntGeneric<"vr", 0xE7B3>;
      def VRF : TernaryVRRcInt<"vrf", 0xE7B3, null_frag, v128f, v128f, 2>;
      def VRG : TernaryVRRcInt<"vrg", 0xE7B3, null_frag, v128g, v128g, 3>;
      def VRQ : TernaryVRRcInt<"vrq", 0xE7B3, null_frag, v128q, v128q, 4>;
    }
    def : Pat<(v4i32 (srem VR128:$x, VR128:$y)), (VRF VR128:$x, VR128:$y, 0)>;
    def : Pat<(v2i64 (srem VR128:$x, VR128:$y)), (VRG VR128:$x, VR128:$y, 0)>;
    def : Pat<(i128 (srem VR128:$x, VR128:$y)), (VRQ VR128:$x, VR128:$y, 0)>;
  }

  // Remainder logical.
  let Predicates = [FeatureVectorEnhancements3] in {
    let hasSideEffects = 1 in {
      def VRL  : TernaryVRRcIntGeneric<"vrl", 0xE7B1>;
      def VRLF : TernaryVRRcInt<"vrlf", 0xE7B1, null_frag, v128f, v128f, 2>;
      def VRLG : TernaryVRRcInt<"vrlg", 0xE7B1, null_frag, v128g, v128g, 3>;
      def VRLQ : TernaryVRRcInt<"vrlq", 0xE7B1, null_frag, v128q, v128q, 4>;
    }
    def : Pat<(v4i32 (urem VR128:$x, VR128:$y)), (VRLF VR128:$x, VR128:$y, 0)>;
    def : Pat<(v2i64 (urem VR128:$x, VR128:$y)), (VRLG VR128:$x, VR128:$y, 0)>;
    def : Pat<(i128 (urem VR128:$x, VR128:$y)), (VRLQ VR128:$x, VR128:$y, 0)>;
  }

  // Element rotate left logical (with vector shift amount).
  def VERLLV  : BinaryVRRcGeneric<"verllv", 0xE773>;
  def VERLLVB : BinaryVRRc<"verllvb", 0xE773, rotl, v128b, v128b, 0>;
  def VERLLVH : BinaryVRRc<"verllvh", 0xE773, rotl, v128h, v128h, 1>;
  def VERLLVF : BinaryVRRc<"verllvf", 0xE773, rotl, v128f, v128f, 2>;
  def VERLLVG : BinaryVRRc<"verllvg", 0xE773, rotl, v128g, v128g, 3>;

  // Element rotate left logical (with scalar shift amount).
  def VERLL  : BinaryVRSaGeneric<"verll", 0xE733>;
  def VERLLB : BinaryVRSa<"verllb", 0xE733, z_vrotl_by_scalar, v128b, v128b, 0>;
  def VERLLH : BinaryVRSa<"verllh", 0xE733, z_vrotl_by_scalar, v128h, v128h, 1>;
  def VERLLF : BinaryVRSa<"verllf", 0xE733, z_vrotl_by_scalar, v128f, v128f, 2>;
  def VERLLG : BinaryVRSa<"verllg", 0xE733, z_vrotl_by_scalar, v128g, v128g, 3>;

  // Element rotate and insert under mask.
  def VERIM  : QuaternaryVRIdGeneric<"verim", 0xE772>;
  def VERIMB : QuaternaryVRId<"verimb", 0xE772, int_s390_verimb, v128b, v128b, 0>;
  def VERIMH : QuaternaryVRId<"verimh", 0xE772, int_s390_verimh, v128h, v128h, 1>;
  def VERIMF : QuaternaryVRId<"verimf", 0xE772, int_s390_verimf, v128f, v128f, 2>;
  def VERIMG : QuaternaryVRId<"verimg", 0xE772, int_s390_verimg, v128g, v128g, 3>;

  // Element shift left (with vector shift amount).
  def VESLV  : BinaryVRRcGeneric<"veslv", 0xE770>;
  def VESLVB : BinaryVRRc<"veslvb", 0xE770, z_vshl, v128b, v128b, 0>;
  def VESLVH : BinaryVRRc<"veslvh", 0xE770, z_vshl, v128h, v128h, 1>;
  def VESLVF : BinaryVRRc<"veslvf", 0xE770, z_vshl, v128f, v128f, 2>;
  def VESLVG : BinaryVRRc<"veslvg", 0xE770, z_vshl, v128g, v128g, 3>;

  // Element shift left (with scalar shift amount).
  def VESL  : BinaryVRSaGeneric<"vesl", 0xE730>;
  def VESLB : BinaryVRSa<"veslb", 0xE730, z_vshl_by_scalar, v128b, v128b, 0>;
  def VESLH : BinaryVRSa<"veslh", 0xE730, z_vshl_by_scalar, v128h, v128h, 1>;
  def VESLF : BinaryVRSa<"veslf", 0xE730, z_vshl_by_scalar, v128f, v128f, 2>;
  def VESLG : BinaryVRSa<"veslg", 0xE730, z_vshl_by_scalar, v128g, v128g, 3>;

  // Element shift right arithmetic (with vector shift amount).
  def VESRAV  : BinaryVRRcGeneric<"vesrav", 0xE77A>;
  def VESRAVB : BinaryVRRc<"vesravb", 0xE77A, z_vsra, v128b, v128b, 0>;
  def VESRAVH : BinaryVRRc<"vesravh", 0xE77A, z_vsra, v128h, v128h, 1>;
  def VESRAVF : BinaryVRRc<"vesravf", 0xE77A, z_vsra, v128f, v128f, 2>;
  def VESRAVG : BinaryVRRc<"vesravg", 0xE77A, z_vsra, v128g, v128g, 3>;

  // Element shift right arithmetic (with scalar shift amount).
  def VESRA  : BinaryVRSaGeneric<"vesra", 0xE73A>;
  def VESRAB : BinaryVRSa<"vesrab", 0xE73A, z_vsra_by_scalar, v128b, v128b, 0>;
  def VESRAH : BinaryVRSa<"vesrah", 0xE73A, z_vsra_by_scalar, v128h, v128h, 1>;
  def VESRAF : BinaryVRSa<"vesraf", 0xE73A, z_vsra_by_scalar, v128f, v128f, 2>;
  def VESRAG : BinaryVRSa<"vesrag", 0xE73A, z_vsra_by_scalar, v128g, v128g, 3>;

  // Element shift right logical (with vector shift amount).
  def VESRLV  : BinaryVRRcGeneric<"vesrlv", 0xE778>;
  def VESRLVB : BinaryVRRc<"vesrlvb", 0xE778, z_vsrl, v128b, v128b, 0>;
  def VESRLVH : BinaryVRRc<"vesrlvh", 0xE778, z_vsrl, v128h, v128h, 1>;
  def VESRLVF : BinaryVRRc<"vesrlvf", 0xE778, z_vsrl, v128f, v128f, 2>;
  def VESRLVG : BinaryVRRc<"vesrlvg", 0xE778, z_vsrl, v128g, v128g, 3>;

  // Element shift right logical (with scalar shift amount).
  def VESRL  : BinaryVRSaGeneric<"vesrl", 0xE738>;
  def VESRLB : BinaryVRSa<"vesrlb", 0xE738, z_vsrl_by_scalar, v128b, v128b, 0>;
  def VESRLH : BinaryVRSa<"vesrlh", 0xE738, z_vsrl_by_scalar, v128h, v128h, 1>;
  def VESRLF : BinaryVRSa<"vesrlf", 0xE738, z_vsrl_by_scalar, v128f, v128f, 2>;
  def VESRLG : BinaryVRSa<"vesrlg", 0xE738, z_vsrl_by_scalar, v128g, v128g, 3>;

  // Shift left.
  def VSL : BinaryVRRc<"vsl", 0xE774, int_s390_vsl, v128b, v128b>;

  // Shift left by byte.
  def VSLB : BinaryVRRc<"vslb", 0xE775, int_s390_vslb, v128b, v128b>;

  // Shift left double by byte.
  def VSLDB : TernaryVRId<"vsldb", 0xE777, z_shl_double, v128b, v128b, 0>;
  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8_timm:$z),
            (VSLDB VR128:$x, VR128:$y, imm32zx8:$z)>;

  // Shift left double by bit.
  let Predicates = [FeatureVectorEnhancements2] in {
    def VSLD : TernaryVRId<"vsld", 0xE786, z_shl_double_bit, v128b, v128b, 0>;
    def : Pat<(int_s390_vsld VR128:$x, VR128:$y, imm32zx8_timm:$z),
              (VSLD VR128:$x, VR128:$y, imm32zx8:$z)>;
  }

  // Shift right arithmetic.
  def VSRA : BinaryVRRc<"vsra", 0xE77E, int_s390_vsra, v128b, v128b>;

  // Shift right arithmetic by byte.
  def VSRAB : BinaryVRRc<"vsrab", 0xE77F, int_s390_vsrab, v128b, v128b>;

  // Shift right logical.
  def VSRL : BinaryVRRc<"vsrl", 0xE77C, int_s390_vsrl, v128b, v128b>;

  // Shift right logical by byte.
  def VSRLB : BinaryVRRc<"vsrlb", 0xE77D, int_s390_vsrlb, v128b, v128b>;

  // Shift right double by bit.
  let Predicates = [FeatureVectorEnhancements2] in {
    def VSRD : TernaryVRId<"vsrd", 0xE787, z_shr_double_bit, v128b, v128b, 0>;
    def : Pat<(int_s390_vsrd VR128:$x, VR128:$y, imm32zx8_timm:$z),
              (VSRD VR128:$x, VR128:$y, imm32zx8:$z)>;
  }

  // Subtract.
  def VS  : BinaryVRRcGeneric<"vs", 0xE7F7>;
  def VSB : BinaryVRRc<"vsb", 0xE7F7, sub, v128b, v128b, 0>;
  def VSH : BinaryVRRc<"vsh", 0xE7F7, sub, v128h, v128h, 1>;
  def VSF : BinaryVRRc<"vsf", 0xE7F7, sub, v128f, v128f, 2>;
  def VSG : BinaryVRRc<"vsg", 0xE7F7, sub, v128g, v128g, 3>;
  def VSQ : BinaryVRRc<"vsq", 0xE7F7, sub, v128q, v128q, 4>;

  // Subtract compute borrow indication.
  def VSCBI  : BinaryVRRcGeneric<"vscbi", 0xE7F5>;
  def VSCBIB : BinaryVRRc<"vscbib", 0xE7F5, z_vscbi, v128b, v128b, 0>;
  def VSCBIH : BinaryVRRc<"vscbih", 0xE7F5, z_vscbi, v128h, v128h, 1>;
  def VSCBIF : BinaryVRRc<"vscbif", 0xE7F5, z_vscbi, v128f, v128f, 2>;
  def VSCBIG : BinaryVRRc<"vscbig", 0xE7F5, z_vscbi, v128g, v128g, 3>;
  def VSCBIQ : BinaryVRRc<"vscbiq", 0xE7F5, z_vscbi, v128q, v128q, 4>;

  // Subtract with borrow indication.
  def VSBI  : TernaryVRRdGeneric<"vsbi", 0xE7BF>;
  def VSBIQ : TernaryVRRd<"vsbiq", 0xE7BF, z_vsbi, v128q, v128q, 4>;

  // Subtract with borrow compute borrow indication.
  def VSBCBI  : TernaryVRRdGeneric<"vsbcbi", 0xE7BD>;
  def VSBCBIQ : TernaryVRRd<"vsbcbiq", 0xE7BD, z_vsbcbi, v128q, v128q, 4>;

  // Sum across doubleword.
  def VSUMG  : BinaryVRRcGeneric<"vsumg", 0xE765>;
  def VSUMGH : BinaryVRRc<"vsumgh", 0xE765, z_vsum, v128g, v128h, 1>;
  def VSUMGF : BinaryVRRc<"vsumgf", 0xE765, z_vsum, v128g, v128f, 2>;

  // Sum across quadword.
  def VSUMQ  : BinaryVRRcGeneric<"vsumq", 0xE767>;
  def VSUMQF : BinaryVRRc<"vsumqf", 0xE767, z_vsum, v128q, v128f, 2>;
  def VSUMQG : BinaryVRRc<"vsumqg", 0xE767, z_vsum, v128q, v128g, 3>;

  // Sum across word.
  def VSUM  : BinaryVRRcGeneric<"vsum", 0xE764>;
  def VSUMB : BinaryVRRc<"vsumb", 0xE764, z_vsum, v128f, v128b, 0>;
  def VSUMH : BinaryVRRc<"vsumh", 0xE764, z_vsum, v128f, v128h, 1>;
}

// Instantiate the bitwise ops for type TYPE.
multiclass BitwiseVectorOps<ValueType type, SDPatternOperator not_op> {
  let Predicates = [FeatureVector] in {
    def : Pat<(type (and VR128:$x, VR128:$y)), (VN VR128:$x, VR128:$y)>;
    def : Pat<(type (and VR128:$x, (not_op VR128:$y))),
              (VNC VR128:$x, VR128:$y)>;
    def : Pat<(type (or VR128:$x, VR128:$y)), (VO VR128:$x, VR128:$y)>;
    def : Pat<(type (xor VR128:$x, VR128:$y)), (VX VR128:$x, VR128:$y)>;
    def : Pat<(type (or (and VR128:$x, VR128:$z),
                        (and VR128:$y, (not_op VR128:$z)))),
              (VSEL VR128:$x, VR128:$y, VR128:$z)>;
    def : Pat<(type (not_op (or VR128:$x, VR128:$y))),
              (VNO VR128:$x, VR128:$y)>;
    def : Pat<(type (not_op VR128:$x)), (VNO VR128:$x, VR128:$x)>;
  }
  let Predicates = [FeatureVectorEnhancements1] in {
    def : Pat<(type (not_op (xor VR128:$x, VR128:$y))),
              (VNX VR128:$x, VR128:$y)>;
    def : Pat<(type (not_op (and VR128:$x, VR128:$y))),
              (VNN VR128:$x, VR128:$y)>;
    def : Pat<(type (or VR128:$x, (not_op VR128:$y))),
              (VOC VR128:$x, VR128:$y)>;
  }
  let Predicates = [FeatureVectorEnhancements3] in {
    def : Pat<(type (and VR128:$x, (and VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 1)>;
    def : Pat<(type (and (not_op VR128:$z), (and VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 2)>;
    def : Pat<(type (and VR128:$x, (xor VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 6)>;
    def : Pat<(type (and VR128:$x, (or VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 7)>;
    def : Pat<(type (and VR128:$x, (not_op (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 8)>;
    def : Pat<(type (and VR128:$x, (not_op (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 9)>;
    def : Pat<(type (and VR128:$x, (or VR128:$y, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 11)>;
    def : Pat<(type (and VR128:$x, (not_op (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 14)>;
    def : Pat<(type (and (or VR128:$x, VR128:$y), (xor VR128:$z, (and VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 22)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (and VR128:$z, (or VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 23)>;
    def : Pat<(type (and (xor VR128:$x, VR128:$y), (xor VR128:$x, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 24)>;
    def : Pat<(type (and (or VR128:$x, VR128:$y), (not_op (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 25)>;
    def : Pat<(type (and (or VR128:$x, VR128:$y), (xor VR128:$x, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 26)>;
    def : Pat<(type (and (or VR128:$x, VR128:$z), (or VR128:$y, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 27)>;
    def : Pat<(type (xor VR128:$x, (and VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 30)>;
    def : Pat<(type (or VR128:$x, (and VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 31)>;
    def : Pat<(type (and (not_op VR128:$z), (xor VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 40)>;
    def : Pat<(type (and (or VR128:$x, VR128:$y), (not_op (xor VR128:$z, (and VR128:$x, VR128:$y))))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 41)>;
    def : Pat<(type (and (not_op VR128:$z), (or VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 42)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (and (not_op VR128:$z), (or VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 43)>;
    def : Pat<(type (xor VR128:$y, (or VR128:$x, (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 44)>;
    def : Pat<(type (xor VR128:$x, (and VR128:$y, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 45)>;
    def : Pat<(type (and (or VR128:$x, VR128:$y), (not_op (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 46)>;
    def : Pat<(type (or VR128:$x, (and VR128:$y, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 47)>;
    def : Pat<(type (or (xor VR128:$x, VR128:$y), (and VR128:$x, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 61)>;
    def : Pat<(type (or (xor VR128:$x, VR128:$y), (and VR128:$x, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 62)>;
    def : Pat<(type (xor (or VR128:$x, VR128:$y), (or VR128:$z, (and VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 104)>;
    def : Pat<(type (xor VR128:$x, (xor VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 105)>;
    def : Pat<(type (xor VR128:$z, (or VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 106)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (xor VR128:$z, (or VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 107)>;
    def : Pat<(type (or (xor VR128:$y, VR128:$z), (and VR128:$x, (not_op VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 110)>;
    def : Pat<(type (or VR128:$x, (xor VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 111)>;
    def : Pat<(type (or (xor VR128:$x, VR128:$y), (xor VR128:$x, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 126)>;
    def : Pat<(type (or VR128:$x, (or VR128:$y, VR128:$z))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 127)>;
    def : Pat<(type (not_op (or VR128:$x, (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 128)>;
    def : Pat<(type (not_op (or (xor VR128:$x, VR128:$y), (xor VR128:$x, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 129)>;
    def : Pat<(type (not_op (or VR128:$z, (xor VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 130)>;
    def : Pat<(type (and (not_op (xor VR128:$x, VR128:$y)), (or VR128:$x, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 131)>;
    def : Pat<(type (xor (or VR128:$y, VR128:$z), (or (not_op VR128:$x), (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 134)>;
    def : Pat<(type (not_op (xor VR128:$x, (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 135)>;
    def : Pat<(type (or (not_op (or VR128:$y, VR128:$z)), (and VR128:$x, (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 137)>;
    def : Pat<(type (and (not_op VR128:$z), (or VR128:$x, (not_op VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 138)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (not_op (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 139)>;
    def : Pat<(type (or (not_op (or VR128:$y, VR128:$z)), (and VR128:$x, (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 142)>;
    def : Pat<(type (or VR128:$x, (not_op (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 143)>;
    def : Pat<(type (not_op (xor VR128:$x, (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 150)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (not_op (xor VR128:$z, (or VR128:$x, VR128:$y))))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 151)>;
    def : Pat<(type (not_op (or (and VR128:$x, VR128:$y), (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 152)>;
    def : Pat<(type (xor VR128:$z, (or VR128:$x, (not_op VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 154)>;
    def : Pat<(type (or (and VR128:$x, VR128:$y), (not_op (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 155)>;
    def : Pat<(type (or (not_op (or VR128:$y, VR128:$z)), (xor VR128:$x, (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 158)>;
    def : Pat<(type (or VR128:$x, (not_op (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 159)>;
    def : Pat<(type (not_op (or VR128:$z, (and VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 168)>;
    def : Pat<(type (not_op (xor VR128:$z, (and VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 169)>;
    def : Pat<(type (or (not_op VR128:$z), (and VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 171)>;
    def : Pat<(type (and (not_op (and VR128:$x, VR128:$y)), (or VR128:$x, (not_op VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 172)>;
    def : Pat<(type (not_op (and (xor VR128:$x, VR128:$z), (or VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 173)>;
    def : Pat<(type (or (not_op VR128:$z), (and VR128:$x, (not_op VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 174)>;
    def : Pat<(type (or (xor VR128:$x, VR128:$y), (not_op (or VR128:$x, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 188)>;
    def : Pat<(type (not_op (and (xor VR128:$x, VR128:$z), (xor VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 189)>;
    def : Pat<(type (or (not_op VR128:$z), (xor VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 190)>;
    def : Pat<(type (or (not_op VR128:$z), (or VR128:$x, VR128:$y))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 191)>;
    def : Pat<(type (or (not_op (or VR128:$x, VR128:$y)), (and (not_op VR128:$z), (xor VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 232)>;
    def : Pat<(type (xor (not_op (and VR128:$x, VR128:$y)), (and VR128:$z, (or VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 233)>;
    def : Pat<(type (not_op (and VR128:$z, (or VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 234)>;
    def : Pat<(type (not_op (and VR128:$z, (xor VR128:$x, VR128:$y)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 235)>;
    def : Pat<(type (or VR128:$x, (not_op (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 239)>;
    def : Pat<(type (not_op (and VR128:$x, (and VR128:$y, VR128:$z)))),
              (VEVAL VR128:$x, VR128:$y, VR128:$z, 254)>;
  }
}

defm : BitwiseVectorOps<v16i8, z_vnot>;
defm : BitwiseVectorOps<v8i16, z_vnot>;
defm : BitwiseVectorOps<v4i32, z_vnot>;
defm : BitwiseVectorOps<v2i64, z_vnot>;
defm : BitwiseVectorOps<i128, not>;

// Instantiate additional patterns for absolute-related expressions on
// type TYPE.  LC is the negate instruction for TYPE and LP is the absolute
// instruction.
multiclass IntegerAbsoluteVectorOps<ValueType type, Instruction lc,
                                    Instruction lp, int shift> {
  let Predicates = [FeatureVector] in {
    def : Pat<(type (vselect (type (z_vicmph_zero VR128:$x)),
                             (z_vneg VR128:$x), VR128:$x)),
              (lc (lp VR128:$x))>;
    def : Pat<(type (vselect (type (z_vnot (z_vicmph_zero VR128:$x))),
                             VR128:$x, (z_vneg VR128:$x))),
              (lc (lp VR128:$x))>;
    def : Pat<(type (vselect (type (z_vicmpl_zero VR128:$x)),
                             VR128:$x, (z_vneg VR128:$x))),
              (lc (lp VR128:$x))>;
    def : Pat<(type (vselect (type (z_vnot (z_vicmpl_zero VR128:$x))),
                             (z_vneg VR128:$x), VR128:$x)),
              (lc (lp VR128:$x))>;
    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
                             (z_vneg VR128:$x)),
                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
                             VR128:$x))),
              (lp VR128:$x)>;
    def : Pat<(type (or (and (z_vsra_by_scalar VR128:$x, (i32 shift)),
                             VR128:$x),
                        (and (z_vnot (z_vsra_by_scalar VR128:$x, (i32 shift))),
                             (z_vneg VR128:$x)))),
              (lc (lp VR128:$x))>;
  }
}

defm : IntegerAbsoluteVectorOps<v16i8, VLCB, VLPB, 7>;
defm : IntegerAbsoluteVectorOps<v8i16, VLCH, VLPH, 15>;
defm : IntegerAbsoluteVectorOps<v4i32, VLCF, VLPF, 31>;
defm : IntegerAbsoluteVectorOps<v2i64, VLCG, VLPG, 63>;

// Instantiate packs/packu: recognize a saturating truncation and convert
// into the corresponding packs/packu instruction.
multiclass SignedSaturatingTruncate<ValueType input, ValueType output,
                                    Instruction packs> {
  def : Pat<
    (output (z_pack
      (smin (smax (input VR128:$a), ssat_trunc_min_vec), ssat_trunc_max_vec),
      (smin (smax (input VR128:$b), ssat_trunc_min_vec), ssat_trunc_max_vec)
    )),
    (packs VR128:$a, VR128:$b)
  >;

  def : Pat<
    (output (z_pack
      (smax (smin (input VR128:$a), ssat_trunc_max_vec), ssat_trunc_min_vec),
      (smax (smin (input VR128:$b), ssat_trunc_max_vec), ssat_trunc_min_vec)
    )),
    (packs VR128:$a, VR128:$b)
  >;
}

defm : SignedSaturatingTruncate<v8i16, v16i8, VPKSH>;
defm : SignedSaturatingTruncate<v4i32, v8i16, VPKSF>;
defm : SignedSaturatingTruncate<v2i64, v4i32, VPKSG>;

multiclass UnsignedSaturatingTruncate<ValueType input, ValueType output,
                                      Instruction packu> {
  def : Pat<
    (output (z_pack
      (umin (input VR128:$a), usat_trunc_max_vec),
      (umin (input VR128:$b), usat_trunc_max_vec)
    )),
    (packu VR128:$a, VR128:$b)
  >;
}

defm : UnsignedSaturatingTruncate<v8i16, v16i8, VPKLSH>;
defm : UnsignedSaturatingTruncate<v4i32, v8i16, VPKLSF>;
defm : UnsignedSaturatingTruncate<v2i64, v4i32, VPKLSG>;

// Instantiate comparison patterns to recognize VACC/VSCBI for TYPE.
multiclass IntegerComputeCarryOrBorrow<ValueType type,
                                       Instruction vacc, Instruction vscbi> {
  let Predicates = [FeatureVector] in {
    def : Pat<(z_vzext1 (type (z_vicmphl VR128:$x, (add VR128:$x, VR128:$y)))),
              (vacc VR128:$x, VR128:$y)>;
    def : Pat<(z_vzext1 (type (z_vicmphl VR128:$y, (add VR128:$x, VR128:$y)))),
              (vacc VR128:$x, VR128:$y)>;
    def : Pat<(z_vzext1 (z_vnot (type (z_vicmphl VR128:$y, VR128:$x)))),
              (vscbi VR128:$x, VR128:$y)>;
  }
}
defm : IntegerComputeCarryOrBorrow<v16i8, VACCB, VSCBIB>;
defm : IntegerComputeCarryOrBorrow<v8i16, VACCH, VSCBIH>;
defm : IntegerComputeCarryOrBorrow<v4i32, VACCF, VSCBIF>;
defm : IntegerComputeCarryOrBorrow<v2i64, VACCG, VSCBIG>;

// Instantiate full-vector shifts.
multiclass FullVectorShiftOps<SDPatternOperator shift,
                              Instruction sbit, Instruction sbyte> {
  let Predicates = [FeatureVector] in {
    def : Pat<(shift (i128 VR128:$x), imm32nobytes:$amt),
              (sbit VR128:$x, (VREPIB (UIMM8 imm:$amt)))>;
    def : Pat<(shift (i128 VR128:$x), imm32nobits:$amt),
              (sbyte VR128:$x, (VREPIB (UIMM8 imm:$amt)))>;
    def : Pat<(shift (i128 VR128:$x), imm32:$amt),
              (sbit (sbyte VR128:$x, (VREPIB (UIMM8 imm:$amt))),
                    (VREPIB (UIMM8 imm:$amt)))>;
    def : Pat<(shift (i128 VR128:$x), GR32:$amt),
              (sbit (sbyte VR128:$x, (VREPB (VLVGP32 GR32:$amt, GR32:$amt), 15)),
                    (VREPB (VLVGP32 GR32:$amt, GR32:$amt), 15))>;
  }
}
defm : FullVectorShiftOps<vshiftop<shl>, VSL, VSLB>;
defm : FullVectorShiftOps<vshiftop<srl>, VSRL, VSRLB>;
defm : FullVectorShiftOps<vshiftop<sra>, VSRA, VSRAB>;

//===----------------------------------------------------------------------===//
// Integer comparison
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Element compare.
  let Defs = [CC] in {
    def VEC  : CompareVRRaGeneric<"vec", 0xE7DB>;
    def VECB : CompareVRRa<"vecb", 0xE7DB, null_frag, v128b, 0>;
    def VECH : CompareVRRa<"vech", 0xE7DB, null_frag, v128h, 1>;
    def VECF : CompareVRRa<"vecf", 0xE7DB, null_frag, v128f, 2>;
    def VECG : CompareVRRa<"vecg", 0xE7DB, null_frag, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VECQ : CompareVRRa<"vecq", 0xE7DB, z_scmp, v128q, 4>;
  }

  // Element compare logical.
  let Defs = [CC] in {
    def VECL  : CompareVRRaGeneric<"vecl", 0xE7D9>;
    def VECLB : CompareVRRa<"veclb", 0xE7D9, null_frag, v128b, 0>;
    def VECLH : CompareVRRa<"veclh", 0xE7D9, null_frag, v128h, 1>;
    def VECLF : CompareVRRa<"veclf", 0xE7D9, null_frag, v128f, 2>;
    def VECLG : CompareVRRa<"veclg", 0xE7D9, null_frag, v128g, 3>;
    let Predicates = [FeatureVectorEnhancements3] in
      def VECLQ : CompareVRRa<"veclq", 0xE7D9, z_ucmp, v128q, 4>;
  }

  // Compare equal.
  def  VCEQ  : BinaryVRRbSPairGeneric<"vceq", 0xE7F8>;
  defm VCEQB : BinaryVRRbSPair<"vceqb", 0xE7F8, z_vicmpe, z_vicmpes,
                               v128b, v128b, 0>;
  defm VCEQH : BinaryVRRbSPair<"vceqh", 0xE7F8, z_vicmpe, z_vicmpes,
                               v128h, v128h, 1>;
  defm VCEQF : BinaryVRRbSPair<"vceqf", 0xE7F8, z_vicmpe, z_vicmpes,
                               v128f, v128f, 2>;
  defm VCEQG : BinaryVRRbSPair<"vceqg", 0xE7F8, z_vicmpe, z_vicmpes,
                               v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    defm VCEQQ : BinaryVRRbSPair<"vceqq", 0xE7F8, z_vicmpe, z_vicmpes,
                                 v128q, v128q, 4>;

  // Compare high.
  def  VCH  : BinaryVRRbSPairGeneric<"vch", 0xE7FB>;
  defm VCHB : BinaryVRRbSPair<"vchb", 0xE7FB, z_vicmph, z_vicmphs,
                              v128b, v128b, 0>;
  defm VCHH : BinaryVRRbSPair<"vchh", 0xE7FB, z_vicmph, z_vicmphs,
                              v128h, v128h, 1>;
  defm VCHF : BinaryVRRbSPair<"vchf", 0xE7FB, z_vicmph, z_vicmphs,
                              v128f, v128f, 2>;
  defm VCHG : BinaryVRRbSPair<"vchg", 0xE7FB, z_vicmph, z_vicmphs,
                              v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    defm VCHQ : BinaryVRRbSPair<"vchq", 0xE7FB, z_vicmph, z_vicmphs,
                                v128q, v128q, 4>;

  // Compare high logical.
  def  VCHL  : BinaryVRRbSPairGeneric<"vchl", 0xE7F9>;
  defm VCHLB : BinaryVRRbSPair<"vchlb", 0xE7F9, z_vicmphl, z_vicmphls,
                               v128b, v128b, 0>;
  defm VCHLH : BinaryVRRbSPair<"vchlh", 0xE7F9, z_vicmphl, z_vicmphls,
                               v128h, v128h, 1>;
  defm VCHLF : BinaryVRRbSPair<"vchlf", 0xE7F9, z_vicmphl, z_vicmphls,
                               v128f, v128f, 2>;
  defm VCHLG : BinaryVRRbSPair<"vchlg", 0xE7F9, z_vicmphl, z_vicmphls,
                               v128g, v128g, 3>;
  let Predicates = [FeatureVectorEnhancements3] in
    defm VCHLQ : BinaryVRRbSPair<"vchlq", 0xE7F9, z_vicmphl, z_vicmphls,
                                 v128q, v128q, 4>;

  // Test under mask.
  let Defs = [CC] in
    def VTM : CompareVRRa<"vtm", 0xE7D8, z_vtm, v128b, 0>;
}

//===----------------------------------------------------------------------===//
// Floating-point arithmetic
//===----------------------------------------------------------------------===//

// See comments in SystemZInstrFP.td for the suppression flags and
// rounding modes.
multiclass VectorRounding<Instruction insn, TypedReg tr> {
  def : FPConversion<insn, any_frint,      tr, tr, 0, 0>;
  def : FPConversion<insn, any_fnearbyint, tr, tr, 4, 0>;
  def : FPConversion<insn, any_ffloor,     tr, tr, 4, 7>;
  def : FPConversion<insn, any_fceil,      tr, tr, 4, 6>;
  def : FPConversion<insn, any_ftrunc,     tr, tr, 4, 5>;
  def : FPConversion<insn, any_froundeven, tr, tr, 4, 4>;
  def : FPConversion<insn, any_fround,     tr, tr, 4, 1>;
}

let Predicates = [FeatureVector] in {
  // Add.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
    def VFA   : BinaryVRRcFloatGeneric<"vfa", 0xE7E3>;
    def VFADB : BinaryVRRc<"vfadb", 0xE7E3, any_fadd, v128db, v128db, 3, 0>;
    def WFADB : BinaryVRRc<"wfadb", 0xE7E3, any_fadd, v64db, v64db, 3, 8, 0,
                           "adbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFASB : BinaryVRRc<"vfasb", 0xE7E3, any_fadd, v128sb, v128sb, 2, 0>;
      def WFASB : BinaryVRRc<"wfasb", 0xE7E3, any_fadd, v32sb, v32sb, 2, 8, 0,
                             "aebr">;
      def WFAXB : BinaryVRRc<"wfaxb", 0xE7E3, any_fadd, v128xb, v128xb, 4, 8>;
    }
  }

  // Convert from fixed.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VCDG  : TernaryVRRaFloatGeneric<"vcdg", 0xE7C3>;
    def VCDGB : TernaryVRRa<"vcdgb", 0xE7C3, null_frag, v128db, v128g, 3, 0>;
    def WCDGB : TernaryVRRa<"wcdgb", 0xE7C3, null_frag, v64db, v64g, 3, 8>;
  }
  def : FPConversion<VCDGB, any_sint_to_fp, v128db, v128g, 0, 0>;
  let Predicates = [FeatureVectorEnhancements2] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in
        def VCFPS  : TernaryVRRaFloatGeneric<"vcfps", 0xE7C3>;
      def VCEFB : TernaryVRRa<"vcefb", 0xE7C3, null_frag, v128sb, v128g, 2, 0>;
      def WCEFB : TernaryVRRa<"wcefb", 0xE7C3, null_frag, v32sb, v32f, 2, 8>;
    }
    def : FPConversion<VCEFB, any_sint_to_fp, v128sb, v128f, 0, 0>;
  }

  // Convert from logical.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VCDLG  : TernaryVRRaFloatGeneric<"vcdlg", 0xE7C1>;
    def VCDLGB : TernaryVRRa<"vcdlgb", 0xE7C1, null_frag, v128db, v128g, 3, 0>;
    def WCDLGB : TernaryVRRa<"wcdlgb", 0xE7C1, null_frag, v64db, v64g, 3, 8>;
  }
  def : FPConversion<VCDLGB, any_uint_to_fp, v128db, v128g, 0, 0>;
  let Predicates = [FeatureVectorEnhancements2] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in
        def VCFPL  : TernaryVRRaFloatGeneric<"vcfpl", 0xE7C1>;
      def VCELFB : TernaryVRRa<"vcelfb", 0xE7C1, null_frag, v128sb, v128g, 2, 0>;
      def WCELFB : TernaryVRRa<"wcelfb", 0xE7C1, null_frag, v32sb, v32f, 2, 8>;
    }
    def : FPConversion<VCELFB, any_uint_to_fp, v128sb, v128f, 0, 0>;
  }

  // Convert to fixed.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VCGD  : TernaryVRRaFloatGeneric<"vcgd", 0xE7C2>;
    def VCGDB : TernaryVRRa<"vcgdb", 0xE7C2, null_frag, v128g, v128db, 3, 0>;
    def WCGDB : TernaryVRRa<"wcgdb", 0xE7C2, null_frag, v64g, v64db, 3, 8>;
  }
  // Rounding mode should agree with SystemZInstrFP.td.
  def : FPConversion<VCGDB, any_fp_to_sint, v128g, v128db, 0, 5>;
  let Predicates = [FeatureVectorEnhancements2] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in
        def VCSFP  : TernaryVRRaFloatGeneric<"vcsfp", 0xE7C2>;
      def VCFEB : TernaryVRRa<"vcfeb", 0xE7C2, null_frag, v128sb, v128g, 2, 0>;
      def WCFEB : TernaryVRRa<"wcfeb", 0xE7C2, null_frag, v32sb, v32f, 2, 8>;
    }
    // Rounding mode should agree with SystemZInstrFP.td.
    def : FPConversion<VCFEB, any_fp_to_sint, v128f, v128sb, 0, 5>;
  }

  // Convert to logical.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VCLGD  : TernaryVRRaFloatGeneric<"vclgd", 0xE7C0>;
    def VCLGDB : TernaryVRRa<"vclgdb", 0xE7C0, null_frag, v128g, v128db, 3, 0>;
    def WCLGDB : TernaryVRRa<"wclgdb", 0xE7C0, null_frag, v64g, v64db, 3, 8>;
  }
  // Rounding mode should agree with SystemZInstrFP.td.
  def : FPConversion<VCLGDB, any_fp_to_uint, v128g, v128db, 0, 5>;
  let Predicates = [FeatureVectorEnhancements2] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in
        def VCLFP  : TernaryVRRaFloatGeneric<"vclfp", 0xE7C0>;
      def VCLFEB : TernaryVRRa<"vclfeb", 0xE7C0, null_frag, v128sb, v128g, 2, 0>;
      def WCLFEB : TernaryVRRa<"wclfeb", 0xE7C0, null_frag, v32sb, v32f, 2, 8>;
    }
    // Rounding mode should agree with SystemZInstrFP.td.
    def : FPConversion<VCLFEB, any_fp_to_uint, v128f, v128sb, 0, 5>;
  }

  // Divide.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VFD   : BinaryVRRcFloatGeneric<"vfd", 0xE7E5>;
    def VFDDB : BinaryVRRc<"vfddb", 0xE7E5, any_fdiv, v128db, v128db, 3, 0>;
    def WFDDB : BinaryVRRc<"wfddb", 0xE7E5, any_fdiv, v64db, v64db, 3, 8, 0,
                           "ddbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFDSB : BinaryVRRc<"vfdsb", 0xE7E5, any_fdiv, v128sb, v128sb, 2, 0>;
      def WFDSB : BinaryVRRc<"wfdsb", 0xE7E5, any_fdiv, v32sb, v32sb, 2, 8, 0,
                             "debr">;
      def WFDXB : BinaryVRRc<"wfdxb", 0xE7E5, any_fdiv, v128xb, v128xb, 4, 8>;
    }
  }

  // Load FP integer.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VFI   : TernaryVRRaFloatGeneric<"vfi", 0xE7C7>;
    def VFIDB : TernaryVRRa<"vfidb", 0xE7C7, int_s390_vfidb, v128db, v128db, 3, 0>;
    def WFIDB : TernaryVRRa<"wfidb", 0xE7C7, null_frag, v64db, v64db, 3, 8>;
  }
  defm : VectorRounding<VFIDB, v128db>;
  defm : VectorRounding<WFIDB, v64db>;
  let Predicates = [FeatureVectorEnhancements1] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      def VFISB : TernaryVRRa<"vfisb", 0xE7C7, int_s390_vfisb, v128sb, v128sb, 2, 0>;
      def WFISB : TernaryVRRa<"wfisb", 0xE7C7, null_frag, v32sb, v32sb, 2, 8>;
      def WFIXB : TernaryVRRa<"wfixb", 0xE7C7, null_frag, v128xb, v128xb, 4, 8>;
    }
    defm : VectorRounding<VFISB, v128sb>;
    defm : VectorRounding<WFISB, v32sb>;
    defm : VectorRounding<WFIXB, v128xb>;
  }

  // Load lengthened.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VLDE  : UnaryVRRaFloatGeneric<"vlde", 0xE7C4>;
    def VLDEB : UnaryVRRa<"vldeb", 0xE7C4, z_any_vextend, v128db, v128sb, 2, 0>;
    def WLDEB : UnaryVRRa<"wldeb", 0xE7C4, any_fpextend, v64db, v32sb, 2, 8, 0,
                          "ldebr">;
  }
  let Predicates = [FeatureVectorEnhancements1] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in {
        def VFLL  : UnaryVRRaFloatGeneric<"vfll", 0xE7C4>;
        def VFLLS : UnaryVRRa<"vflls", 0xE7C4, null_frag, v128db, v128sb, 2, 0>;
        def WFLLS : UnaryVRRa<"wflls", 0xE7C4, null_frag, v64db, v32sb, 2, 8>;
      }
      def WFLLD : UnaryVRRa<"wflld", 0xE7C4, any_fpextend, v128xb, v64db, 3, 8>;
    }
    def : Pat<(f128 (any_fpextend (f32 VR32:$src))),
              (WFLLD (WLDEB VR32:$src))>;
  }

  // Load rounded.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VLED  : TernaryVRRaFloatGeneric<"vled", 0xE7C5>;
    def VLEDB : TernaryVRRa<"vledb", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
    def WLEDB : TernaryVRRa<"wledb", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
  }
  def : Pat<(v4f32 (z_any_vround (v2f64 VR128:$src))), (VLEDB VR128:$src, 0, 0)>;
  def : FPConversion<WLEDB, any_fpround, v32sb, v64db, 0, 0>;
  let Predicates = [FeatureVectorEnhancements1] in {
    let Uses = [FPC], mayRaiseFPException = 1 in {
      let isAsmParserOnly = 1 in {
        def VFLR  : TernaryVRRaFloatGeneric<"vflr", 0xE7C5>;
        def VFLRD : TernaryVRRa<"vflrd", 0xE7C5, null_frag, v128sb, v128db, 3, 0>;
        def WFLRD : TernaryVRRa<"wflrd", 0xE7C5, null_frag, v32sb, v64db, 3, 8>;
      }
      def WFLRX : TernaryVRRa<"wflrx", 0xE7C5, null_frag, v64db, v128xb, 4, 8>;
    }
    def : FPConversion<WFLRX, any_fpround, v64db, v128xb, 0, 0>;
    def : Pat<(f32 (any_fpround (f128 VR128:$src))),
              (WLEDB (WFLRX VR128:$src, 0, 3), 0, 0)>;
  }

  // Maximum.
  multiclass VectorMax<Instruction insn, TypedReg tr> {
    def : FPMinMax<insn, any_fmaxnum, tr, 4>;
    def : FPMinMax<insn, any_fmaximum, tr, 1>;
  }
  let Predicates = [FeatureVectorEnhancements1] in {
    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
      def VFMAX   : TernaryVRRcFloatGeneric<"vfmax", 0xE7EF>;
      def VFMAXDB : TernaryVRRcFloat<"vfmaxdb", 0xE7EF, int_s390_vfmaxdb,
                                     v128db, v128db, 3, 0>;
      def WFMAXDB : TernaryVRRcFloat<"wfmaxdb", 0xE7EF, null_frag,
                                     v64db, v64db, 3, 8>;
      def VFMAXSB : TernaryVRRcFloat<"vfmaxsb", 0xE7EF, int_s390_vfmaxsb,
                                     v128sb, v128sb, 2, 0>;
      def WFMAXSB : TernaryVRRcFloat<"wfmaxsb", 0xE7EF, null_frag,
                                     v32sb, v32sb, 2, 8>;
      def WFMAXXB : TernaryVRRcFloat<"wfmaxxb", 0xE7EF, null_frag,
                                     v128xb, v128xb, 4, 8>;
    }
    defm : VectorMax<VFMAXDB, v128db>;
    defm : VectorMax<WFMAXDB, v64db>;
    defm : VectorMax<VFMAXSB, v128sb>;
    defm : VectorMax<WFMAXSB, v32sb>;
    defm : VectorMax<WFMAXXB, v128xb>;
  }

  // Minimum.
  multiclass VectorMin<Instruction insn, TypedReg tr> {
    def : FPMinMax<insn, any_fminnum, tr, 4>;
    def : FPMinMax<insn, any_fminimum, tr, 1>;
  }
  let Predicates = [FeatureVectorEnhancements1] in {
    let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
      def VFMIN   : TernaryVRRcFloatGeneric<"vfmin", 0xE7EE>;
      def VFMINDB : TernaryVRRcFloat<"vfmindb", 0xE7EE, int_s390_vfmindb,
                                     v128db, v128db, 3, 0>;
      def WFMINDB : TernaryVRRcFloat<"wfmindb", 0xE7EE, null_frag,
                                     v64db, v64db, 3, 8>;
      def VFMINSB : TernaryVRRcFloat<"vfminsb", 0xE7EE, int_s390_vfminsb,
                                     v128sb, v128sb, 2, 0>;
      def WFMINSB : TernaryVRRcFloat<"wfminsb", 0xE7EE, null_frag,
                                     v32sb, v32sb, 2, 8>;
      def WFMINXB : TernaryVRRcFloat<"wfminxb", 0xE7EE, null_frag,
                                     v128xb, v128xb, 4, 8>;
    }
    defm : VectorMin<VFMINDB, v128db>;
    defm : VectorMin<WFMINDB, v64db>;
    defm : VectorMin<VFMINSB, v128sb>;
    defm : VectorMin<WFMINSB, v32sb>;
    defm : VectorMin<WFMINXB, v128xb>;
  }

  // Multiply.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
    def VFM   : BinaryVRRcFloatGeneric<"vfm", 0xE7E7>;
    def VFMDB : BinaryVRRc<"vfmdb", 0xE7E7, any_fmul, v128db, v128db, 3, 0>;
    def WFMDB : BinaryVRRc<"wfmdb", 0xE7E7, any_fmul, v64db, v64db, 3, 8, 0,
                           "mdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFMSB : BinaryVRRc<"vfmsb", 0xE7E7, any_fmul, v128sb, v128sb, 2, 0>;
      def WFMSB : BinaryVRRc<"wfmsb", 0xE7E7, any_fmul, v32sb, v32sb, 2, 8, 0,
                             "meebr">;
      def WFMXB : BinaryVRRc<"wfmxb", 0xE7E7, any_fmul, v128xb, v128xb, 4, 8>;
    }
  }

  // Multiply and add.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
    def VFMA   : TernaryVRReFloatGeneric<"vfma", 0xE78F>;
    def VFMADB : TernaryVRRe<"vfmadb", 0xE78F, any_fma, v128db, v128db, 0, 3>;
    def WFMADB : TernaryVRRe<"wfmadb", 0xE78F, any_fma, v64db, v64db, 8, 3,
                             "madbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFMASB : TernaryVRRe<"vfmasb", 0xE78F, any_fma, v128sb, v128sb, 0, 2>;
      def WFMASB : TernaryVRRe<"wfmasb", 0xE78F, any_fma, v32sb, v32sb, 8, 2,
                               "maebr">;
      def WFMAXB : TernaryVRRe<"wfmaxb", 0xE78F, any_fma, v128xb, v128xb, 8, 4>;
    }
  }

  // Multiply and subtract.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1 in {
    def VFMS   : TernaryVRReFloatGeneric<"vfms", 0xE78E>;
    def VFMSDB : TernaryVRRe<"vfmsdb", 0xE78E, any_fms, v128db, v128db, 0, 3>;
    def WFMSDB : TernaryVRRe<"wfmsdb", 0xE78E, any_fms, v64db, v64db, 8, 3,
                             "msdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFMSSB : TernaryVRRe<"vfmssb", 0xE78E, any_fms, v128sb, v128sb, 0, 2>;
      def WFMSSB : TernaryVRRe<"wfmssb", 0xE78E, any_fms, v32sb, v32sb, 8, 2,
                               "msebr">;
      def WFMSXB : TernaryVRRe<"wfmsxb", 0xE78E, any_fms, v128xb, v128xb, 8, 4>;
    }
  }

  // Negative multiply and add.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
      Predicates = [FeatureVectorEnhancements1] in {
    def VFNMA   : TernaryVRReFloatGeneric<"vfnma", 0xE79F>;
    def VFNMADB : TernaryVRRe<"vfnmadb", 0xE79F, any_fnma, v128db, v128db, 0, 3>;
    def WFNMADB : TernaryVRRe<"wfnmadb", 0xE79F, any_fnma, v64db, v64db, 8, 3>;
    def VFNMASB : TernaryVRRe<"vfnmasb", 0xE79F, any_fnma, v128sb, v128sb, 0, 2>;
    def WFNMASB : TernaryVRRe<"wfnmasb", 0xE79F, any_fnma, v32sb, v32sb, 8, 2>;
    def WFNMAXB : TernaryVRRe<"wfnmaxb", 0xE79F, any_fnma, v128xb, v128xb, 8, 4>;
  }

  // Negative multiply and subtract.
  let Uses = [FPC], mayRaiseFPException = 1, isCommutable = 1,
      Predicates = [FeatureVectorEnhancements1] in {
    def VFNMS   : TernaryVRReFloatGeneric<"vfnms", 0xE79E>;
    def VFNMSDB : TernaryVRRe<"vfnmsdb", 0xE79E, any_fnms, v128db, v128db, 0, 3>;
    def WFNMSDB : TernaryVRRe<"wfnmsdb", 0xE79E, any_fnms, v64db, v64db, 8, 3>;
    def VFNMSSB : TernaryVRRe<"vfnmssb", 0xE79E, any_fnms, v128sb, v128sb, 0, 2>;
    def WFNMSSB : TernaryVRRe<"wfnmssb", 0xE79E, any_fnms, v32sb, v32sb, 8, 2>;
    def WFNMSXB : TernaryVRRe<"wfnmsxb", 0xE79E, any_fnms, v128xb, v128xb, 8, 4>;
  }

  // Perform sign operation.
  def VFPSO   : BinaryVRRaFloatGeneric<"vfpso", 0xE7CC>;
  def VFPSODB : BinaryVRRa<"vfpsodb", 0xE7CC, null_frag, v128db, v128db, 3, 0>;
  def WFPSODB : BinaryVRRa<"wfpsodb", 0xE7CC, null_frag, v64db, v64db, 3, 8>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VFPSOSB : BinaryVRRa<"vfpsosb", 0xE7CC, null_frag, v128sb, v128sb, 2, 0>;
    def WFPSOSB : BinaryVRRa<"wfpsosb", 0xE7CC, null_frag, v32sb, v32sb, 2, 8>;
    def WFPSOXB : BinaryVRRa<"wfpsoxb", 0xE7CC, null_frag, v128xb, v128xb, 4, 8>;
  }

  // Load complement.
  def VFLCDB : UnaryVRRa<"vflcdb", 0xE7CC, fneg, v128db, v128db, 3, 0, 0>;
  def WFLCDB : UnaryVRRa<"wflcdb", 0xE7CC, fneg, v64db, v64db, 3, 8, 0>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VFLCSB : UnaryVRRa<"vflcsb", 0xE7CC, fneg, v128sb, v128sb, 2, 0, 0>;
    def WFLCSB : UnaryVRRa<"wflcsb", 0xE7CC, fneg, v32sb, v32sb, 2, 8, 0>;
    def WFLCXB : UnaryVRRa<"wflcxb", 0xE7CC, fneg, v128xb, v128xb, 4, 8, 0>;
  }

  // Load negative.
  def VFLNDB : UnaryVRRa<"vflndb", 0xE7CC, fnabs, v128db, v128db, 3, 0, 1>;
  def WFLNDB : UnaryVRRa<"wflndb", 0xE7CC, fnabs, v64db, v64db, 3, 8, 1>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VFLNSB : UnaryVRRa<"vflnsb", 0xE7CC, fnabs, v128sb, v128sb, 2, 0, 1>;
    def WFLNSB : UnaryVRRa<"wflnsb", 0xE7CC, fnabs, v32sb, v32sb, 2, 8, 1>;
    def WFLNXB : UnaryVRRa<"wflnxb", 0xE7CC, fnabs, v128xb, v128xb, 4, 8, 1>;
  }

  // Load positive.
  def VFLPDB : UnaryVRRa<"vflpdb", 0xE7CC, fabs, v128db, v128db, 3, 0, 2>;
  def WFLPDB : UnaryVRRa<"wflpdb", 0xE7CC, fabs, v64db, v64db, 3, 8, 2>;
  let Predicates = [FeatureVectorEnhancements1] in {
    def VFLPSB : UnaryVRRa<"vflpsb", 0xE7CC, fabs, v128sb, v128sb, 2, 0, 2>;
    def WFLPSB : UnaryVRRa<"wflpsb", 0xE7CC, fabs, v32sb, v32sb, 2, 8, 2>;
    def WFLPXB : UnaryVRRa<"wflpxb", 0xE7CC, fabs, v128xb, v128xb, 4, 8, 2>;
  }

  // Square root.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VFSQ   : UnaryVRRaFloatGeneric<"vfsq", 0xE7CE>;
    def VFSQDB : UnaryVRRa<"vfsqdb", 0xE7CE, any_fsqrt, v128db, v128db, 3, 0>;
    def WFSQDB : UnaryVRRa<"wfsqdb", 0xE7CE, any_fsqrt, v64db, v64db, 3, 8, 0,
                           "sqdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFSQSB : UnaryVRRa<"vfsqsb", 0xE7CE, any_fsqrt, v128sb, v128sb, 2, 0>;
      def WFSQSB : UnaryVRRa<"wfsqsb", 0xE7CE, any_fsqrt, v32sb, v32sb, 2, 8, 0,
                             "sqebr">;
      def WFSQXB : UnaryVRRa<"wfsqxb", 0xE7CE, any_fsqrt, v128xb, v128xb, 4, 8>;
    }
  }

  // Subtract.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def VFS   : BinaryVRRcFloatGeneric<"vfs", 0xE7E2>;
    def VFSDB : BinaryVRRc<"vfsdb", 0xE7E2, any_fsub, v128db, v128db, 3, 0>;
    def WFSDB : BinaryVRRc<"wfsdb", 0xE7E2, any_fsub, v64db, v64db, 3, 8, 0,
                           "sdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFSSB : BinaryVRRc<"vfssb", 0xE7E2, any_fsub, v128sb, v128sb, 2, 0>;
      def WFSSB : BinaryVRRc<"wfssb", 0xE7E2, any_fsub, v32sb, v32sb, 2, 8, 0,
                             "sebr">;
      def WFSXB : BinaryVRRc<"wfsxb", 0xE7E2, any_fsub, v128xb, v128xb, 4, 8>;
    }
  }

  // Test data class immediate.
  let Defs = [CC] in {
    def VFTCI   : BinaryVRIeFloatGeneric<"vftci", 0xE74A>;
    def VFTCIDB : BinaryVRIe<"vftcidb", 0xE74A, z_vftci, v128g, v128db, 3, 0>;
    def WFTCIDB : BinaryVRIe<"wftcidb", 0xE74A, null_frag, v64g, v64db, 3, 8>;
    let Predicates = [FeatureVectorEnhancements1] in {
      def VFTCISB : BinaryVRIe<"vftcisb", 0xE74A, z_vftci, v128f, v128sb, 2, 0>;
      def WFTCISB : BinaryVRIe<"wftcisb", 0xE74A, null_frag, v32f, v32sb, 2, 8>;
      def WFTCIXB : BinaryVRIe<"wftcixb", 0xE74A, null_frag, v128q, v128xb, 4, 8>;
    }
  }
}

//===----------------------------------------------------------------------===//
// Floating-point comparison
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  // Compare scalar.
  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
    def WFC   : CompareVRRaFloatGeneric<"wfc", 0xE7CB>;
    def WFCDB : CompareVRRa<"wfcdb", 0xE7CB, z_any_fcmp, v64db, 3, "cdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def WFCSB : CompareVRRa<"wfcsb", 0xE7CB, z_any_fcmp, v32sb, 2, "cebr">;
      def WFCXB : CompareVRRa<"wfcxb", 0xE7CB, z_any_fcmp, v128xb, 4>;
    }
  }

  // Compare and signal scalar.
  let Uses = [FPC], mayRaiseFPException = 1, Defs = [CC] in {
    def WFK   : CompareVRRaFloatGeneric<"wfk", 0xE7CA>;
    def WFKDB : CompareVRRa<"wfkdb", 0xE7CA, z_strict_fcmps, v64db, 3, "kdbr">;
    let Predicates = [FeatureVectorEnhancements1] in {
      def WFKSB : CompareVRRa<"wfksb", 0xE7CA, z_strict_fcmps, v32sb, 2, "kebr">;
      def WFKXB : CompareVRRa<"wfkxb", 0xE7CA, z_strict_fcmps, v128xb, 4>;
    }
  }

  // Compare equal.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def  VFCE   : BinaryVRRcSPairFloatGeneric<"vfce", 0xE7E8>;
    defm VFCEDB : BinaryVRRcSPair<"vfcedb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
                                  v128g, v128db, 3, 0>;
    defm WFCEDB : BinaryVRRcSPair<"wfcedb", 0xE7E8, null_frag, null_frag,
                                  v64g, v64db, 3, 8>;
    let Predicates = [FeatureVectorEnhancements1] in {
      defm VFCESB : BinaryVRRcSPair<"vfcesb", 0xE7E8, z_any_vfcmpe, z_vfcmpes,
                                    v128f, v128sb, 2, 0>;
      defm WFCESB : BinaryVRRcSPair<"wfcesb", 0xE7E8, null_frag, null_frag,
                                    v32f, v32sb, 2, 8>;
      defm WFCEXB : BinaryVRRcSPair<"wfcexb", 0xE7E8, null_frag, null_frag,
                                    v128q, v128xb, 4, 8>;
    }
  }

  // Compare and signal equal.
  let Uses = [FPC], mayRaiseFPException = 1,
      Predicates = [FeatureVectorEnhancements1] in {
    defm VFKEDB : BinaryVRRcSPair<"vfkedb", 0xE7E8, z_strict_vfcmpes, null_frag,
                                  v128g, v128db, 3, 4>;
    defm WFKEDB : BinaryVRRcSPair<"wfkedb", 0xE7E8, null_frag, null_frag,
                                  v64g, v64db, 3, 12>;
    defm VFKESB : BinaryVRRcSPair<"vfkesb", 0xE7E8, z_strict_vfcmpes, null_frag,
                                  v128f, v128sb, 2, 4>;
    defm WFKESB : BinaryVRRcSPair<"wfkesb", 0xE7E8, null_frag, null_frag,
                                  v32f, v32sb, 2, 12>;
    defm WFKEXB : BinaryVRRcSPair<"wfkexb", 0xE7E8, null_frag, null_frag,
                                  v128q, v128xb, 4, 12>;
  }

  // Compare high.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def  VFCH   : BinaryVRRcSPairFloatGeneric<"vfch", 0xE7EB>;
    defm VFCHDB : BinaryVRRcSPair<"vfchdb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
                                  v128g, v128db, 3, 0>;
    defm WFCHDB : BinaryVRRcSPair<"wfchdb", 0xE7EB, null_frag, null_frag,
                                  v64g, v64db, 3, 8>;
    let Predicates = [FeatureVectorEnhancements1] in {
      defm VFCHSB : BinaryVRRcSPair<"vfchsb", 0xE7EB, z_any_vfcmph, z_vfcmphs,
                                    v128f, v128sb, 2, 0>;
      defm WFCHSB : BinaryVRRcSPair<"wfchsb", 0xE7EB, null_frag, null_frag,
                                    v32f, v32sb, 2, 8>;
      defm WFCHXB : BinaryVRRcSPair<"wfchxb", 0xE7EB, null_frag, null_frag,
                                    v128q, v128xb, 4, 8>;
    }
  }

  // Compare and signal high.
  let Uses = [FPC], mayRaiseFPException = 1,
      Predicates = [FeatureVectorEnhancements1] in {
    defm VFKHDB : BinaryVRRcSPair<"vfkhdb", 0xE7EB, z_strict_vfcmphs, null_frag,
                                  v128g, v128db, 3, 4>;
    defm WFKHDB : BinaryVRRcSPair<"wfkhdb", 0xE7EB, null_frag, null_frag,
                                  v64g, v64db, 3, 12>;
    defm VFKHSB : BinaryVRRcSPair<"vfkhsb", 0xE7EB, z_strict_vfcmphs, null_frag,
                                  v128f, v128sb, 2, 4>;
    defm WFKHSB : BinaryVRRcSPair<"wfkhsb", 0xE7EB, null_frag, null_frag,
                                  v32f, v32sb, 2, 12>;
    defm WFKHXB : BinaryVRRcSPair<"wfkhxb", 0xE7EB, null_frag, null_frag,
                                  v128q, v128xb, 4, 12>;
  }

  // Compare high or equal.
  let Uses = [FPC], mayRaiseFPException = 1 in {
    def  VFCHE   : BinaryVRRcSPairFloatGeneric<"vfche", 0xE7EA>;
    defm VFCHEDB : BinaryVRRcSPair<"vfchedb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
                                   v128g, v128db, 3, 0>;
    defm WFCHEDB : BinaryVRRcSPair<"wfchedb", 0xE7EA, null_frag, null_frag,
                                   v64g, v64db, 3, 8>;
    let Predicates = [FeatureVectorEnhancements1] in {
      defm VFCHESB : BinaryVRRcSPair<"vfchesb", 0xE7EA, z_any_vfcmphe, z_vfcmphes,
                                     v128f, v128sb, 2, 0>;
      defm WFCHESB : BinaryVRRcSPair<"wfchesb", 0xE7EA, null_frag, null_frag,
                                     v32f, v32sb, 2, 8>;
      defm WFCHEXB : BinaryVRRcSPair<"wfchexb", 0xE7EA, null_frag, null_frag,
                                     v128q, v128xb, 4, 8>;
    }
  }

  // Compare and signal high or equal.
  let Uses = [FPC], mayRaiseFPException = 1,
      Predicates = [FeatureVectorEnhancements1] in {
    defm VFKHEDB : BinaryVRRcSPair<"vfkhedb", 0xE7EA, z_strict_vfcmphes, null_frag,
                                   v128g, v128db, 3, 4>;
    defm WFKHEDB : BinaryVRRcSPair<"wfkhedb", 0xE7EA, null_frag, null_frag,
                                   v64g, v64db, 3, 12>;
    defm VFKHESB : BinaryVRRcSPair<"vfkhesb", 0xE7EA, z_strict_vfcmphes, null_frag,
                                   v128f, v128sb, 2, 4>;
    defm WFKHESB : BinaryVRRcSPair<"wfkhesb", 0xE7EA, null_frag, null_frag,
                                   v32f, v32sb, 2, 12>;
    defm WFKHEXB : BinaryVRRcSPair<"wfkhexb", 0xE7EA, null_frag, null_frag,
                                   v128q, v128xb, 4, 12>;
  }
}

//===----------------------------------------------------------------------===//
// Support for 128-bit integer values in vector registers
//===----------------------------------------------------------------------===//

// Loads and stores.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (load bdxaddr12only:$addr)),
            (VL bdxaddr12only:$addr)>;
  def : Pat<(store (i128 VR128:$src), bdxaddr12only:$addr),
            (VST VR128:$src, bdxaddr12only:$addr)>;
}

// Full i128 move from GPR pair.
let Predicates = [FeatureVector] in
  def : Pat<(i128 (or (zext GR64:$x), (shl (anyext GR64:$y), (i32 64)))),
            (VLVGP GR64:$y, GR64:$x)>;

// Any-extensions from GPR to i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (anyext GR32:$x)), (VLVGP32 GR32:$x, GR32:$x)>;
  def : Pat<(i128 (anyext GR64:$x)), (VLVGP GR64:$x, GR64:$x)>;
}

// Any-extending loads into i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (z_extloadi8 bdxaddr12only:$addr)),
            (VLREPB bdxaddr12only:$addr)>;
  def : Pat<(i128 (z_extloadi16 bdxaddr12only:$addr)),
            (VLREPH bdxaddr12only:$addr)>;
  def : Pat<(i128 (z_extloadi32 bdxaddr12only:$addr)),
            (VLREPF bdxaddr12only:$addr)>;
  def : Pat<(i128 (z_extloadi64 bdxaddr12only:$addr)),
            (VLREPG bdxaddr12only:$addr)>;
}

// Truncations from i128 to GPR.
let Predicates = [FeatureVector] in {
  def : Pat<(i32 (trunc (i128 VR128:$vec))),
            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 3), subreg_l32)>;
  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 32)))),
            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 2), subreg_l32)>;
  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 64)))),
            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 1), subreg_l32)>;
  def : Pat<(i32 (trunc (srl (i128 VR128:$vec), (i32 96)))),
            (EXTRACT_SUBREG (VLGVF VR128:$vec, zero_reg, 0), subreg_l32)>;
  def : Pat<(i64 (trunc (i128 VR128:$vec))),
            (VLGVG VR128:$vec, zero_reg, 1)>;
  def : Pat<(i64 (trunc (srl (i128 VR128:$vec), (i32 64)))),
            (VLGVG VR128:$vec, zero_reg, 0)>;
}

// Truncating stores from i128.
let Predicates = [FeatureVector] in {
  def : Pat<(truncstorei8 (i128 VR128:$x), bdxaddr12only:$addr),
            (VSTEB VR128:$x, bdxaddr12only:$addr, 15)>;
  def : Pat<(truncstorei16 (i128 VR128:$x), bdxaddr12only:$addr),
            (VSTEH VR128:$x, bdxaddr12only:$addr, 7)>;
  def : Pat<(truncstorei32 (i128 VR128:$x), bdxaddr12only:$addr),
            (VSTEF VR128:$x, bdxaddr12only:$addr, 3)>;
  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 32)), bdxaddr12only:$addr),
            (VSTEF VR128:$x, bdxaddr12only:$addr, 2)>;
  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 64)), bdxaddr12only:$addr),
            (VSTEF VR128:$x, bdxaddr12only:$addr, 1)>;
  def : Pat<(truncstorei32 (srl (i128 VR128:$x), (i32 96)), bdxaddr12only:$addr),
            (VSTEF VR128:$x, bdxaddr12only:$addr, 0)>;
  def : Pat<(truncstorei64 (i128 VR128:$x), bdxaddr12only:$addr),
            (VSTEG VR128:$x, bdxaddr12only:$addr, 1)>;
  def : Pat<(truncstorei64 (srl (i128 VR128:$x), (i32 64)), bdxaddr12only:$addr),
            (VSTEG VR128:$x, bdxaddr12only:$addr, 0)>;
}

// Zero-extensions from GPR to i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (zext8 (anyext GR32:$x))),
            (VLVGB (VGBM 0), GR32:$x, zero_reg, 15)>;
  def : Pat<(i128 (zext16 (anyext GR32:$x))),
            (VLVGH (VGBM 0), GR32:$x, zero_reg, 7)>;
  def : Pat<(i128 (zext GR32:$x)),
            (VLVGF (VGBM 0), GR32:$x, zero_reg, 3)>;
  def : Pat<(i128 (zext GR64:$x)),
            (VLVGG (VGBM 0), GR64:$x, zero_reg, 1)>;
}

// Zero-extending loads into i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (z_zextloadi8 bdxaddr12only:$addr)),
            (VLEB (VGBM 0), bdxaddr12only:$addr, 15)>;
  def : Pat<(i128 (z_zextloadi16 bdxaddr12only:$addr)),
            (VLEH (VGBM 0), bdxaddr12only:$addr, 7)>;
  def : Pat<(i128 (z_zextloadi32 bdxaddr12only:$addr)),
            (VLEF (VGBM 0), bdxaddr12only:$addr, 3)>;
  def : Pat<(i128 (z_zextloadi64 bdxaddr12only:$addr)),
            (VLEG (VGBM 0), bdxaddr12only:$addr, 1)>;
}

// Zero-extensions from VR element to i128 on z17.
let Predicates = [FeatureVectorEnhancements3] in {
  def : Pat<(i128 (zext (i64 (z_vector_extract (v2i64 VR128:$src), 0)))),
            (VUPLHG VR128:$src)>;
  def : Pat<(i128 (zext (i64 (z_vector_extract (v2i64 VR128:$src), 1)))),
            (VUPLLG VR128:$src)>;
  def : Pat<(i128 (zext (i32 (z_vector_extract (v4i32 VR128:$src), 0)))),
            (VUPLHG (VUPLHF VR128:$src))>;
  def : Pat<(i128 (zext (i32 (z_vector_extract (v4i32 VR128:$src), 1)))),
            (VUPLHG (VUPLLF VR128:$src))>;
  def : Pat<(i128 (zext (i32 (z_vector_extract (v4i32 VR128:$src), 2)))),
            (VUPLLG (VUPLHF VR128:$src))>;
  def : Pat<(i128 (zext (i32 (z_vector_extract (v4i32 VR128:$src), 3)))),
            (VUPLLG (VUPLLF VR128:$src))>;
}

// In-register i128 sign-extensions on z17.
let Predicates = [FeatureVectorEnhancements3] in {
  def : Pat<(i128 (sext_inreg VR128:$x, i8)), (VUPLG (VSEGB VR128:$x))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i16)), (VUPLG (VSEGH VR128:$x))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i32)), (VUPLG (VSEGF VR128:$x))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i64)), (VUPLG VR128:$x)>;
}

// In-register i128 sign-extensions.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (sext_inreg VR128:$x, i8)),
            (VSRAB (VREPB VR128:$x, 15), (VREPIB 120))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i16)),
            (VSRAB (VREPH VR128:$x, 7), (VREPIB 112))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i32)),
            (VSRAB (VREPF VR128:$x, 3), (VREPIB 96))>;
  def : Pat<(i128 (sext_inreg VR128:$x, i64)),
            (VSRAB (VREPG VR128:$x, 1), (VREPIB 64))>;
}

// Sign-extensions from GPR to i128 on z17.
let Predicates = [FeatureVectorEnhancements3] in {
  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i8)),
            (VUPLG (VLVGP (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$x, subreg_l32)),
                          (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$x, subreg_l32))))>;
  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i16)),
            (VUPLG (VLVGP (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$x, subreg_l32)),
                          (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$x, subreg_l32))))>;
  def : Pat<(i128 (sext GR32:$x)),
            (VUPLG (VLVGP (LGFR GR32:$x), (LGFR GR32:$x)))>;
  def : Pat<(i128 (sext GR64:$x)),
            (VUPLG (VLVGP GR64:$x, GR64:$x))>;
}

// Sign-extensions from GPR to i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i8)),
            (VLVGP (SRAG (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                                 GR32:$x, subreg_l32)), zero_reg, 63),
                   (LGBR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                            GR32:$x, subreg_l32)))>;
  def : Pat<(i128 (sext_inreg (anyext GR32:$x), i16)),
            (VLVGP (SRAG (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                                  GR32:$x, subreg_l32)), zero_reg, 63),
                   (LGHR (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                            GR32:$x, subreg_l32)))>;
  def : Pat<(i128 (sext GR32:$x)),
            (VLVGP (SRAG (LGFR GR32:$x), zero_reg, 63), (LGFR GR32:$x))>;
  def : Pat<(i128 (sext GR64:$x)),
            (VLVGP (SRAG GR64:$x, zero_reg, 63), GR64:$x)>;
}

// Sign-extending loads into i128.
let Predicates = [FeatureVector] in {
  def : Pat<(i128 (z_sextloadi8 bdxaddr12only:$addr)),
            (VSRAB (VLREPB bdxaddr12only:$addr), (VREPIB 120))>;
  def : Pat<(i128 (z_sextloadi16 bdxaddr12only:$addr)),
            (VSRAB (VLREPH bdxaddr12only:$addr), (VREPIB 112))>;
  def : Pat<(i128 (z_sextloadi32 bdxaddr12only:$addr)),
            (VSRAB (VLREPF bdxaddr12only:$addr), (VREPIB 96))>;
  def : Pat<(i128 (z_sextloadi64 bdxaddr12only:$addr)),
            (VSRAB (VLREPG bdxaddr12only:$addr), (VREPIB 64))>;
}

// Sign-extensions from VR element to i128 on z17.
let Predicates = [FeatureVectorEnhancements3] in {
  def : Pat<(i128 (sext (i64 (z_vector_extract (v2i64 VR128:$src), 0)))),
            (VUPHG VR128:$src)>;
  def : Pat<(i128 (sext (i64 (z_vector_extract (v2i64 VR128:$src), 1)))),
            (VUPLG VR128:$src)>;
  def : Pat<(i128 (sext (i32 (z_vector_extract (v4i32 VR128:$src), 0)))),
            (VUPHG (VUPHF VR128:$src))>;
  def : Pat<(i128 (sext (i32 (z_vector_extract (v4i32 VR128:$src), 1)))),
            (VUPHG (VUPLF VR128:$src))>;
  def : Pat<(i128 (sext (i32 (z_vector_extract (v4i32 VR128:$src), 2)))),
            (VUPLG (VUPHF VR128:$src))>;
  def : Pat<(i128 (sext (i32 (z_vector_extract (v4i32 VR128:$src), 3)))),
            (VUPLG (VUPLF VR128:$src))>;
}

// i128 comparison pseudo-instructions.
let Predicates = [FeatureVector], Defs = [CC],
    usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
  def SCmp128Hi : Pseudo<(outs), (ins VR128:$src1, VR128:$src2),
                         [(set CC, (z_scmp128hi (i128 VR128:$src1),
                                                (i128 VR128:$src2)))]>;
  def UCmp128Hi : Pseudo<(outs), (ins VR128:$src1, VR128:$src2),
                         [(set CC, (z_ucmp128hi (i128 VR128:$src1),
                                                (i128 VR128:$src2)))]>;
}

// i128 select pseudo-instructions.
let Predicates = [FeatureVector] in
  def Select128 : SelectWrapper<i128, VR128>;

//===----------------------------------------------------------------------===//
// Conversions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (i128  VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (v8f16 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>;
def : Pat<(v16i8 (bitconvert (f128  VR128:$src))), (v16i8 VR128:$src)>;

def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (i128  VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (v8f16 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>;
def : Pat<(v8i16 (bitconvert (f128  VR128:$src))), (v8i16 VR128:$src)>;

def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (i128  VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (v8f16 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>;
def : Pat<(v4i32 (bitconvert (f128  VR128:$src))), (v4i32 VR128:$src)>;

def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (i128  VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (v8f16 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>;
def : Pat<(v2i64 (bitconvert (f128  VR128:$src))), (v2i64 VR128:$src)>;

def : Pat<(v8f16 (bitconvert (v16i8 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (v8i16 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (v4i32 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (v2i64 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (i128  VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (v4f32 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (v2f64 VR128:$src))), (v8f16 VR128:$src)>;
def : Pat<(v8f16 (bitconvert (f128  VR128:$src))), (v8f16 VR128:$src)>;

def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (i128  VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (v8f16 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>;
def : Pat<(v4f32 (bitconvert (f128  VR128:$src))), (v4f32 VR128:$src)>;

def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (i128  VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (v8f16 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>;
def : Pat<(v2f64 (bitconvert (f128  VR128:$src))), (v2f64 VR128:$src)>;

def : Pat<(f128  (bitconvert (v16i8 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v8i16 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v4i32 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v2i64 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (i128  VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v8f16 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v4f32 VR128:$src))), (f128  VR128:$src)>;
def : Pat<(f128  (bitconvert (v2f64 VR128:$src))), (f128  VR128:$src)>;

def : Pat<(i128  (bitconvert (v16i8 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v8i16 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v4i32 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v2i64 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v8f16 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v4f32 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (v2f64 VR128:$src))), (i128  VR128:$src)>;
def : Pat<(i128  (bitconvert (f128  VR128:$src))), (i128  VR128:$src)>;
} // End Predicates = [FeatureVector]

//===----------------------------------------------------------------------===//
// Replicating scalars
//===----------------------------------------------------------------------===//

// Define patterns for replicating a scalar GR32 into a vector of type TYPE.
// INDEX is 8 minus the element size in bytes.
class VectorReplicateScalar<ValueType type, Instruction insn, bits<16> index>
  : Pat<(type (z_replicate GR32:$scalar)),
        (insn (VLVGP32 GR32:$scalar, GR32:$scalar), index)>;

def : VectorReplicateScalar<v16i8, VREPB, 7>;
def : VectorReplicateScalar<v8i16, VREPH, 3>;
def : VectorReplicateScalar<v4i32, VREPF, 1>;

// i64 replications are just a single instruction.
def : Pat<(v2i64 (z_replicate GR64:$scalar)),
          (VLVGP GR64:$scalar, GR64:$scalar)>;

//===----------------------------------------------------------------------===//
// Floating-point insertion and extraction
//===----------------------------------------------------------------------===//

// Moving 32-bit values between GPRs and FPRs can be done using VLVGF
// and VLGVF.
let Predicates = [FeatureVector] in {
  def LEFR : UnaryAliasVRS<VR32, GR32>;
  def LFER : UnaryAliasVRS<GR64, VR32>;
  def : Pat<(f32 (bitconvert (i32 GR32:$src))), (LEFR GR32:$src)>;
  def : Pat<(i32 (bitconvert (f32 VR32:$src))),
            (EXTRACT_SUBREG (LFER VR32:$src), subreg_l32)>;
  def LEFR_16 : UnaryAliasVRS<VR16, GR32>;
  def LFER_16 : UnaryAliasVRS<GR32, VR16>;
}

// Floating-point values are stored in element 0 of the corresponding
// vector register.  Scalar to vector conversion is just a subreg and
// scalar replication can just replicate element 0 of the vector register.
multiclass ScalarToVectorFP<Instruction vrep, ValueType vt, RegisterOperand cls,
                            SubRegIndex subreg> {
  def : Pat<(vt (scalar_to_vector cls:$scalar)),
            (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar, subreg)>;
  def : Pat<(vt (z_replicate cls:$scalar)),
            (vrep (INSERT_SUBREG (vt (IMPLICIT_DEF)), cls:$scalar,
                                 subreg), 0)>;
}
defm : ScalarToVectorFP<VREPH, v8f16, FP16, subreg_h16>;
defm : ScalarToVectorFP<VREPF, v4f32, FP32, subreg_h32>;
defm : ScalarToVectorFP<VREPG, v2f64, FP64, subreg_h64>;

// Match v2f64 insertions.  The AddedComplexity counters the 3 added by
// TableGen for the base register operand in VLVG-based integer insertions
// and ensures that this version is strictly better.
let AddedComplexity = 4 in {
  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 0),
            (VPDI (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
                                 subreg_h64), VR128:$vec, 1)>;
  def : Pat<(z_vector_insert (v2f64 VR128:$vec), FP64:$elt, 1),
            (VPDI VR128:$vec, (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), FP64:$elt,
                                             subreg_h64), 0)>;
}

// We extract floating-point element X by replicating (for elements other
// than 0) and then taking a high subreg.  The AddedComplexity counters the
// 3 added by TableGen for the base register operand in VLGV-based integer
// extractions and ensures that this version is strictly better.
let AddedComplexity = 4 in {
  def : Pat<(f16 (z_vector_extract (v8f16 VR128:$vec), 0)),
            (EXTRACT_SUBREG VR128:$vec, subreg_h16)>;
  def : Pat<(f16 (z_vector_extract (v8f16 VR128:$vec), imm32zx3:$index)),
            (EXTRACT_SUBREG (VREPH VR128:$vec, imm32zx2:$index), subreg_h16)>;

  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), 0)),
            (EXTRACT_SUBREG VR128:$vec, subreg_h32)>;
  def : Pat<(f32 (z_vector_extract (v4f32 VR128:$vec), imm32zx2:$index)),
            (EXTRACT_SUBREG (VREPF VR128:$vec, imm32zx2:$index), subreg_h32)>;

  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), 0)),
            (EXTRACT_SUBREG VR128:$vec, subreg_h64)>;
  def : Pat<(f64 (z_vector_extract (v2f64 VR128:$vec), imm32zx1:$index)),
            (EXTRACT_SUBREG (VREPG VR128:$vec, imm32zx1:$index), subreg_h64)>;
}

//===----------------------------------------------------------------------===//
// Support for 128-bit floating-point values in vector registers
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVectorEnhancements1] in {
  def : Pat<(f128 (load bdxaddr12only:$addr)),
            (VL bdxaddr12only:$addr)>;
  def : Pat<(store (f128 VR128:$src), bdxaddr12only:$addr),
            (VST VR128:$src, bdxaddr12only:$addr)>;

  def : Pat<(f128 fpimm0), (VZERO)>;
  def : Pat<(f128 fpimmneg0), (WFLNXB (VZERO))>;
}

//===----------------------------------------------------------------------===//
// String instructions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector] in {
  defm VFAE  : TernaryOptVRRbSPairGeneric<"vfae", 0xE782>;
  defm VFAEB : TernaryOptVRRbSPair<"vfaeb", 0xE782, int_s390_vfaeb,
                                   z_vfae_cc, v128b, v128b, 0>;
  defm VFAEH : TernaryOptVRRbSPair<"vfaeh", 0xE782, int_s390_vfaeh,
                                   z_vfae_cc, v128h, v128h, 1>;
  defm VFAEF : TernaryOptVRRbSPair<"vfaef", 0xE782, int_s390_vfaef,
                                   z_vfae_cc, v128f, v128f, 2>;
  defm VFAEZB : TernaryOptVRRbSPair<"vfaezb", 0xE782, int_s390_vfaezb,
                                    z_vfaez_cc, v128b, v128b, 0, 2>;
  defm VFAEZH : TernaryOptVRRbSPair<"vfaezh", 0xE782, int_s390_vfaezh,
                                    z_vfaez_cc, v128h, v128h, 1, 2>;
  defm VFAEZF : TernaryOptVRRbSPair<"vfaezf", 0xE782, int_s390_vfaezf,
                                    z_vfaez_cc, v128f, v128f, 2, 2>;

  defm VFEE  : BinaryExtraVRRbSPairGeneric<"vfee", 0xE780>;
  defm VFEEB : BinaryExtraVRRbSPair<"vfeeb", 0xE780, int_s390_vfeeb,
                                    z_vfee_cc, v128b, v128b, 0>;
  defm VFEEH : BinaryExtraVRRbSPair<"vfeeh", 0xE780, int_s390_vfeeh,
                                    z_vfee_cc, v128h, v128h, 1>;
  defm VFEEF : BinaryExtraVRRbSPair<"vfeef", 0xE780, int_s390_vfeef,
                                    z_vfee_cc, v128f, v128f, 2>;
  defm VFEEZB : BinaryVRRbSPair<"vfeezb", 0xE780, int_s390_vfeezb,
                                z_vfeez_cc, v128b, v128b, 0, 2>;
  defm VFEEZH : BinaryVRRbSPair<"vfeezh", 0xE780, int_s390_vfeezh,
                                z_vfeez_cc, v128h, v128h, 1, 2>;
  defm VFEEZF : BinaryVRRbSPair<"vfeezf", 0xE780, int_s390_vfeezf,
                                z_vfeez_cc, v128f, v128f, 2, 2>;

  defm VFENE  : BinaryExtraVRRbSPairGeneric<"vfene", 0xE781>;
  defm VFENEB : BinaryExtraVRRbSPair<"vfeneb", 0xE781, int_s390_vfeneb,
                                     z_vfene_cc, v128b, v128b, 0>;
  defm VFENEH : BinaryExtraVRRbSPair<"vfeneh", 0xE781, int_s390_vfeneh,
                                     z_vfene_cc, v128h, v128h, 1>;
  defm VFENEF : BinaryExtraVRRbSPair<"vfenef", 0xE781, int_s390_vfenef,
                                     z_vfene_cc, v128f, v128f, 2>;
  defm VFENEZB : BinaryVRRbSPair<"vfenezb", 0xE781, int_s390_vfenezb,
                                 z_vfenez_cc, v128b, v128b, 0, 2>;
  defm VFENEZH : BinaryVRRbSPair<"vfenezh", 0xE781, int_s390_vfenezh,
                                 z_vfenez_cc, v128h, v128h, 1, 2>;
  defm VFENEZF : BinaryVRRbSPair<"vfenezf", 0xE781, int_s390_vfenezf,
                                 z_vfenez_cc, v128f, v128f, 2, 2>;

  defm VISTR  : UnaryExtraVRRaSPairGeneric<"vistr", 0xE75C>;
  defm VISTRB : UnaryExtraVRRaSPair<"vistrb", 0xE75C, int_s390_vistrb,
                                    z_vistr_cc, v128b, v128b, 0>;
  defm VISTRH : UnaryExtraVRRaSPair<"vistrh", 0xE75C, int_s390_vistrh,
                                    z_vistr_cc, v128h, v128h, 1>;
  defm VISTRF : UnaryExtraVRRaSPair<"vistrf", 0xE75C, int_s390_vistrf,
                                    z_vistr_cc, v128f, v128f, 2>;

  defm VSTRC  : QuaternaryOptVRRdSPairGeneric<"vstrc", 0xE78A>;
  defm VSTRCB : QuaternaryOptVRRdSPair<"vstrcb", 0xE78A, int_s390_vstrcb,
                                       z_vstrc_cc, v128b, v128b, 0>;
  defm VSTRCH : QuaternaryOptVRRdSPair<"vstrch", 0xE78A, int_s390_vstrch,
                                       z_vstrc_cc, v128h, v128h, 1>;
  defm VSTRCF : QuaternaryOptVRRdSPair<"vstrcf", 0xE78A, int_s390_vstrcf,
                                       z_vstrc_cc, v128f, v128f, 2>;
  defm VSTRCZB : QuaternaryOptVRRdSPair<"vstrczb", 0xE78A, int_s390_vstrczb,
                                        z_vstrcz_cc, v128b, v128b, 0, 2>;
  defm VSTRCZH : QuaternaryOptVRRdSPair<"vstrczh", 0xE78A, int_s390_vstrczh,
                                        z_vstrcz_cc, v128h, v128h, 1, 2>;
  defm VSTRCZF : QuaternaryOptVRRdSPair<"vstrczf", 0xE78A, int_s390_vstrczf,
                                        z_vstrcz_cc, v128f, v128f, 2, 2>;
}

let Predicates = [FeatureVectorEnhancements2] in {
  defm VSTRS  : TernaryExtraVRRdGeneric<"vstrs", 0xE78B>;
  defm VSTRSB : TernaryExtraVRRd<"vstrsb", 0xE78B,
                                 z_vstrs_cc, v128b, v128b, 0>;
  defm VSTRSH : TernaryExtraVRRd<"vstrsh", 0xE78B,
                                 z_vstrs_cc, v128b, v128h, 1>;
  defm VSTRSF : TernaryExtraVRRd<"vstrsf", 0xE78B,
                                 z_vstrs_cc, v128b, v128f, 2>;
  let Defs = [CC] in {
    def VSTRSZB : TernaryVRRd<"vstrszb", 0xE78B,
                              z_vstrsz_cc, v128b, v128b, 0, 2>;
    def VSTRSZH : TernaryVRRd<"vstrszh", 0xE78B,
                              z_vstrsz_cc, v128b, v128h, 1, 2>;
    def VSTRSZF : TernaryVRRd<"vstrszf", 0xE78B,
                              z_vstrsz_cc, v128b, v128f, 2, 2>;
  }
}

//===----------------------------------------------------------------------===//
// NNP assist instructions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVector, FeatureNNPAssist] in {
  let Uses = [FPC], mayRaiseFPException = 1 in
    def VCFN : UnaryVRRaFloatGeneric<"vcfn", 0xE65D>;
  def : Pat<(int_s390_vcfn VR128:$x, imm32zx4_timm:$m),
            (VCFN VR128:$x, 1, imm32zx4:$m)>;

  let Uses = [FPC], mayRaiseFPException = 1 in
    def VCLFNL : UnaryVRRaFloatGeneric<"vclfnl", 0xE65E>;
  def : Pat<(int_s390_vclfnls VR128:$x, imm32zx4_timm:$m),
            (VCLFNL VR128:$x, 2, imm32zx4:$m)>;

  let Uses = [FPC], mayRaiseFPException = 1 in
    def VCLFNH : UnaryVRRaFloatGeneric<"vclfnh", 0xE656>;
  def : Pat<(int_s390_vclfnhs VR128:$x, imm32zx4_timm:$m),
            (VCLFNH VR128:$x, 2, imm32zx4:$m)>;

  let Uses = [FPC], mayRaiseFPException = 1 in
    def VCNF : UnaryVRRaFloatGeneric<"vcnf", 0xE655>;
  def : Pat<(int_s390_vcnf VR128:$x, imm32zx4_timm:$m),
            (VCNF VR128:$x, imm32zx4:$m, 1)>;

  let Uses = [FPC], mayRaiseFPException = 1 in
    def VCRNF : BinaryVRRcFloatGeneric<"vcrnf", 0xE675>;
  def : Pat<(int_s390_vcrnfs VR128:$x, VR128:$y, imm32zx4_timm:$m),
            (VCRNF VR128:$x, VR128:$y, imm32zx4:$m, 2)>;
}

//===----------------------------------------------------------------------===//
// Packed-decimal instructions
//===----------------------------------------------------------------------===//

let Predicates = [FeatureVectorPackedDecimal] in {
  def VLIP : BinaryVRIh<"vlip", 0xE649>;

  def VPKZ : BinaryVSI<"vpkz", 0xE634, null_frag, 0>;
  def VUPKZ : StoreLengthVSI<"vupkz", 0xE63C, null_frag, 0>;

  let Defs = [CC] in {
    let Predicates = [FeatureVectorPackedDecimalEnhancement] in {
      def VCVBOpt : TernaryVRRi<"vcvb", 0xE650, GR32>;
      def VCVBGOpt : TernaryVRRi<"vcvbg", 0xE652, GR64>;
    }
    def VCVB : BinaryVRRi<"vcvb", 0xE650, GR32>;
    def VCVBG : BinaryVRRi<"vcvbg", 0xE652, GR64>;
    def VCVD : TernaryVRIi<"vcvd", 0xE658, GR32>;
    def VCVDG : TernaryVRIi<"vcvdg", 0xE65A, GR64>;

    def VAP : QuaternaryVRIf<"vap", 0xE671>;
    def VSP : QuaternaryVRIf<"vsp", 0xE673>;

    def VMP : QuaternaryVRIf<"vmp", 0xE678>;
    def VMSP : QuaternaryVRIf<"vmsp", 0xE679>;

    def VDP : QuaternaryVRIf<"vdp", 0xE67A>;
    def VRP : QuaternaryVRIf<"vrp", 0xE67B>;
    def VSDP : QuaternaryVRIf<"vsdp", 0xE67E>;

    def VSRP : QuaternaryVRIg<"vsrp", 0xE659>;
    def VPSOP : QuaternaryVRIg<"vpsop", 0xE65B>;

    def VTP : TestVRRg<"vtp", 0xE65F>;
    def VCP : CompareVRRh<"vcp", 0xE677>;
  }
}

let Predicates = [FeatureVectorPackedDecimalEnhancement2] in {
  def VSCHP : BinaryExtraVRRbGeneric<"vschp", 0xE674>;
  def VSCHSP : BinaryExtraVRRb<"vschsp", 0xE674, 2>;
  def VSCHDP : BinaryExtraVRRb<"vschdp", 0xE674, 3>;
  def VSCHXP : BinaryExtraVRRb<"vschxp", 0xE674, 4>;

  def VSCSHP : BinaryVRRb<"vscshp", 0xE67C, null_frag, v128b, v128b>;

  def VCSPH : TernaryVRRj<"vcsph", 0xE67D>;

  let Defs = [CC] in
    def VCLZDP : BinaryVRRk<"vclzdp", 0xE651>;

  let Defs = [CC] in
    def VSRPR : QuaternaryVRIf<"vsrpr", 0xE672>;

  let Defs = [CC] in {
    def VPKZR : QuaternaryVRIf<"vpkzr", 0xE670>;
    def VUPKZH : BinaryVRRk<"vupkzh", 0xE654>;
    def VUPKZL : BinaryVRRk<"vupkzl", 0xE65C>;
  }
}

let Predicates = [FeatureVectorPackedDecimalEnhancement3] in {
  def VCVBQ : BinaryVRRk<"vcvbq", 0xE64E>;
  let Defs = [CC] in
    def VCVDQ : TernaryVRIj<"vcvdq", 0xE64A>;

  let Defs = [CC] in {
    def VTPOpt : TestExtraVRRg<"vtp", 0xE65F>;
    def VTZ : TestExtraVRIl<"vtz", 0xE67F>;
  }
}