blob: 3753bee76777e536561cc72e88c72768d98039a4 [file] [log] [blame]
//===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "HexagonISelLowering.h"
#include "HexagonRegisterInfo.h"
#include "HexagonSubtarget.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/IR/IntrinsicsHexagon.h"
#include "llvm/Support/CommandLine.h"
#include <algorithm>
#include <string>
#include <utility>
using namespace llvm;
static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
cl::Hidden, cl::init(16),
cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
// For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
MVT ElemTy = Ty.getScalarType();
switch (ElemTy.SimpleTy) {
case MVT::f16:
return std::make_tuple(5, 15, 10);
case MVT::f32:
return std::make_tuple(8, 127, 23);
case MVT::f64:
return std::make_tuple(11, 1023, 52);
default:
break;
}
llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str());
}
void
HexagonTargetLowering::initializeHVXLowering() {
if (Subtarget.useHVX64BOps()) {
addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
// These "short" boolean vector types should be legal because
// they will appear as results of vector compares. If they were
// not legal, type legalization would try to make them legal
// and that would require using operations that do not use or
// produce such types. That, in turn, would imply using custom
// nodes, which would be unoptimizable by the DAG combiner.
// The idea is to rely on target-independent operations as much
// as possible.
addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
} else if (Subtarget.useHVX128BOps()) {
addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
addRegisterClass(MVT::v32f32, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v64f16, &Hexagon::HvxVRRegClass);
addRegisterClass(MVT::v64f32, &Hexagon::HvxWRRegClass);
addRegisterClass(MVT::v128f16, &Hexagon::HvxWRRegClass);
}
}
// Set up operation actions.
bool Use64b = Subtarget.useHVX64BOps();
ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
setOperationAction(Opc, FromTy, Promote);
AddPromotedToType(Opc, FromTy, ToTy);
};
// Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
// Note: v16i1 -> i16 is handled in type legalization instead of op
// legalization.
setOperationAction(ISD::BITCAST, MVT::i16, Custom);
setOperationAction(ISD::BITCAST, MVT::i32, Custom);
setOperationAction(ISD::BITCAST, MVT::i64, Custom);
setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
setOperationAction(ISD::BITCAST, MVT::v128i1, Custom);
setOperationAction(ISD::BITCAST, MVT::i128, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal);
setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal);
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
Subtarget.useHVXFloatingPoint()) {
static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
for (MVT T : FloatV) {
setOperationAction(ISD::FADD, T, Legal);
setOperationAction(ISD::FSUB, T, Legal);
setOperationAction(ISD::FMUL, T, Legal);
setOperationAction(ISD::FMINNUM, T, Legal);
setOperationAction(ISD::FMAXNUM, T, Legal);
setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom);
setOperationAction(ISD::SPLAT_VECTOR, T, Legal);
setOperationAction(ISD::SPLAT_VECTOR, T, Legal);
setOperationAction(ISD::MLOAD, T, Custom);
setOperationAction(ISD::MSTORE, T, Custom);
// Custom-lower BUILD_VECTOR. The standard (target-independent)
// handling of it would convert it to a load, which is not always
// the optimal choice.
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
}
// BUILD_VECTOR with f16 operands cannot be promoted without
// promoting the result, so lower the node to vsplat or constant pool
setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::f16, Custom);
setOperationAction(ISD::SPLAT_VECTOR, MVT::f16, Custom);
// Vector shuffle is always promoted to ByteV and a bitcast to f16 is
// generated.
setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW);
setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV);
setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW);
setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV);
for (MVT P : FloatW) {
setOperationAction(ISD::LOAD, P, Custom);
setOperationAction(ISD::STORE, P, Custom);
setOperationAction(ISD::FADD, P, Custom);
setOperationAction(ISD::FSUB, P, Custom);
setOperationAction(ISD::FMUL, P, Custom);
setOperationAction(ISD::FMINNUM, P, Custom);
setOperationAction(ISD::FMAXNUM, P, Custom);
setOperationAction(ISD::SETCC, P, Custom);
setOperationAction(ISD::VSELECT, P, Custom);
// Custom-lower BUILD_VECTOR. The standard (target-independent)
// handling of it would convert it to a load, which is not always
// the optimal choice.
setOperationAction(ISD::BUILD_VECTOR, P, Custom);
// Make concat-vectors custom to handle concats of more than 2 vectors.
setOperationAction(ISD::CONCAT_VECTORS, P, Custom);
setOperationAction(ISD::MLOAD, P, Custom);
setOperationAction(ISD::MSTORE, P, Custom);
}
if (Subtarget.useHVXQFloatOps()) {
setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Custom);
setOperationAction(ISD::FP_ROUND, MVT::v64f16, Legal);
} else if (Subtarget.useHVXIEEEFPOps()) {
setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Legal);
setOperationAction(ISD::FP_ROUND, MVT::v64f16, Legal);
}
}
for (MVT T : LegalV) {
setIndexedLoadAction(ISD::POST_INC, T, Legal);
setIndexedStoreAction(ISD::POST_INC, T, Legal);
setOperationAction(ISD::ABS, T, Legal);
setOperationAction(ISD::AND, T, Legal);
setOperationAction(ISD::OR, T, Legal);
setOperationAction(ISD::XOR, T, Legal);
setOperationAction(ISD::ADD, T, Legal);
setOperationAction(ISD::SUB, T, Legal);
setOperationAction(ISD::MUL, T, Legal);
setOperationAction(ISD::CTPOP, T, Legal);
setOperationAction(ISD::CTLZ, T, Legal);
setOperationAction(ISD::SELECT, T, Legal);
setOperationAction(ISD::SPLAT_VECTOR, T, Legal);
if (T != ByteV) {
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
setOperationAction(ISD::BSWAP, T, Legal);
}
setOperationAction(ISD::SMIN, T, Legal);
setOperationAction(ISD::SMAX, T, Legal);
if (T.getScalarType() != MVT::i32) {
setOperationAction(ISD::UMIN, T, Legal);
setOperationAction(ISD::UMAX, T, Legal);
}
setOperationAction(ISD::CTTZ, T, Custom);
setOperationAction(ISD::LOAD, T, Custom);
setOperationAction(ISD::MLOAD, T, Custom);
setOperationAction(ISD::MSTORE, T, Custom);
if (T.getScalarType() != MVT::i32) {
setOperationAction(ISD::MULHS, T, Legal);
setOperationAction(ISD::MULHU, T, Legal);
}
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
// Make concat-vectors custom to handle concats of more than 2 vectors.
setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
setOperationAction(ISD::ANY_EXTEND, T, Custom);
setOperationAction(ISD::SIGN_EXTEND, T, Custom);
setOperationAction(ISD::ZERO_EXTEND, T, Custom);
setOperationAction(ISD::FSHL, T, Custom);
setOperationAction(ISD::FSHR, T, Custom);
if (T != ByteV) {
setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
// HVX only has shifts of words and halfwords.
setOperationAction(ISD::SRA, T, Custom);
setOperationAction(ISD::SHL, T, Custom);
setOperationAction(ISD::SRL, T, Custom);
// Promote all shuffles to operate on vectors of bytes.
setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
}
if (Subtarget.useHVXFloatingPoint()) {
// Same action for both QFloat and IEEE.
setOperationAction(ISD::SINT_TO_FP, T, Custom);
setOperationAction(ISD::UINT_TO_FP, T, Custom);
setOperationAction(ISD::FP_TO_SINT, T, Custom);
setOperationAction(ISD::FP_TO_UINT, T, Custom);
}
setCondCodeAction(ISD::SETNE, T, Expand);
setCondCodeAction(ISD::SETLE, T, Expand);
setCondCodeAction(ISD::SETGE, T, Expand);
setCondCodeAction(ISD::SETLT, T, Expand);
setCondCodeAction(ISD::SETULE, T, Expand);
setCondCodeAction(ISD::SETUGE, T, Expand);
setCondCodeAction(ISD::SETULT, T, Expand);
}
for (MVT T : LegalW) {
// Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
// independent) handling of it would convert it to a load, which is
// not always the optimal choice.
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
// Make concat-vectors custom to handle concats of more than 2 vectors.
setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
// Custom-lower these operations for pairs. Expand them into a concat
// of the corresponding operations on individual vectors.
setOperationAction(ISD::ANY_EXTEND, T, Custom);
setOperationAction(ISD::SIGN_EXTEND, T, Custom);
setOperationAction(ISD::ZERO_EXTEND, T, Custom);
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom);
setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
setOperationAction(ISD::SPLAT_VECTOR, T, Custom);
setOperationAction(ISD::LOAD, T, Custom);
setOperationAction(ISD::STORE, T, Custom);
setOperationAction(ISD::MLOAD, T, Custom);
setOperationAction(ISD::MSTORE, T, Custom);
setOperationAction(ISD::ABS, T, Custom);
setOperationAction(ISD::CTLZ, T, Custom);
setOperationAction(ISD::CTTZ, T, Custom);
setOperationAction(ISD::CTPOP, T, Custom);
setOperationAction(ISD::ADD, T, Legal);
setOperationAction(ISD::SUB, T, Legal);
setOperationAction(ISD::MUL, T, Custom);
setOperationAction(ISD::MULHS, T, Custom);
setOperationAction(ISD::MULHU, T, Custom);
setOperationAction(ISD::AND, T, Custom);
setOperationAction(ISD::OR, T, Custom);
setOperationAction(ISD::XOR, T, Custom);
setOperationAction(ISD::SETCC, T, Custom);
setOperationAction(ISD::VSELECT, T, Custom);
if (T != ByteW) {
setOperationAction(ISD::SRA, T, Custom);
setOperationAction(ISD::SHL, T, Custom);
setOperationAction(ISD::SRL, T, Custom);
// Promote all shuffles to operate on vectors of bytes.
setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
}
setOperationAction(ISD::FSHL, T, Custom);
setOperationAction(ISD::FSHR, T, Custom);
setOperationAction(ISD::SMIN, T, Custom);
setOperationAction(ISD::SMAX, T, Custom);
if (T.getScalarType() != MVT::i32) {
setOperationAction(ISD::UMIN, T, Custom);
setOperationAction(ISD::UMAX, T, Custom);
}
if (Subtarget.useHVXFloatingPoint()) {
// Same action for both QFloat and IEEE.
setOperationAction(ISD::SINT_TO_FP, T, Custom);
setOperationAction(ISD::UINT_TO_FP, T, Custom);
setOperationAction(ISD::FP_TO_SINT, T, Custom);
setOperationAction(ISD::FP_TO_UINT, T, Custom);
}
}
// Legalize all of these to HexagonISD::[SU]MUL_LOHI.
setOperationAction(ISD::MULHS, WordV, Custom); // -> _LOHI
setOperationAction(ISD::MULHU, WordV, Custom); // -> _LOHI
setOperationAction(ISD::SMUL_LOHI, WordV, Custom);
setOperationAction(ISD::UMUL_LOHI, WordV, Custom);
setCondCodeAction(ISD::SETNE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETLE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETGE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETLT, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETONE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETOLE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETOGE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETOLT, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETUNE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETULE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETUGE, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETULT, MVT::v64f16, Expand);
setCondCodeAction(ISD::SETNE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETLE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETGE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETLT, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETONE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETOLE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETOGE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETOLT, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETUNE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETULE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETUGE, MVT::v32f32, Expand);
setCondCodeAction(ISD::SETULT, MVT::v32f32, Expand);
// Boolean vectors.
for (MVT T : LegalW) {
// Boolean types for vector pairs will overlap with the boolean
// types for single vectors, e.g.
// v64i8 -> v64i1 (single)
// v64i16 -> v64i1 (pair)
// Set these actions first, and allow the single actions to overwrite
// any duplicates.
MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
setOperationAction(ISD::SETCC, BoolW, Custom);
setOperationAction(ISD::AND, BoolW, Custom);
setOperationAction(ISD::OR, BoolW, Custom);
setOperationAction(ISD::XOR, BoolW, Custom);
// Masked load/store takes a mask that may need splitting.
setOperationAction(ISD::MLOAD, BoolW, Custom);
setOperationAction(ISD::MSTORE, BoolW, Custom);
}
for (MVT T : LegalV) {
MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom);
setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom);
setOperationAction(ISD::SELECT, BoolV, Custom);
setOperationAction(ISD::AND, BoolV, Legal);
setOperationAction(ISD::OR, BoolV, Legal);
setOperationAction(ISD::XOR, BoolV, Legal);
}
if (Use64b) {
for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal);
} else {
for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal);
}
// Handle store widening for short vectors.
unsigned HwLen = Subtarget.getVectorLength();
for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
if (ElemTy == MVT::i1)
continue;
int ElemWidth = ElemTy.getFixedSizeInBits();
int MaxElems = (8*HwLen) / ElemWidth;
for (int N = 2; N < MaxElems; N *= 2) {
MVT VecTy = MVT::getVectorVT(ElemTy, N);
auto Action = getPreferredVectorAction(VecTy);
if (Action == TargetLoweringBase::TypeWidenVector) {
setOperationAction(ISD::LOAD, VecTy, Custom);
setOperationAction(ISD::STORE, VecTy, Custom);
setOperationAction(ISD::SETCC, VecTy, Custom);
setOperationAction(ISD::TRUNCATE, VecTy, Custom);
setOperationAction(ISD::ANY_EXTEND, VecTy, Custom);
setOperationAction(ISD::SIGN_EXTEND, VecTy, Custom);
setOperationAction(ISD::ZERO_EXTEND, VecTy, Custom);
if (Subtarget.useHVXFloatingPoint()) {
setOperationAction(ISD::FP_TO_SINT, VecTy, Custom);
setOperationAction(ISD::FP_TO_UINT, VecTy, Custom);
setOperationAction(ISD::SINT_TO_FP, VecTy, Custom);
setOperationAction(ISD::UINT_TO_FP, VecTy, Custom);
}
MVT BoolTy = MVT::getVectorVT(MVT::i1, N);
if (!isTypeLegal(BoolTy))
setOperationAction(ISD::SETCC, BoolTy, Custom);
}
}
}
setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
}
unsigned
HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
MVT ElemTy = VecTy.getVectorElementType();
unsigned VecLen = VecTy.getVectorNumElements();
unsigned HwLen = Subtarget.getVectorLength();
// Split vectors of i1 that exceed byte vector length.
if (ElemTy == MVT::i1 && VecLen > HwLen)
return TargetLoweringBase::TypeSplitVector;
ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
// For shorter vectors of i1, widen them if any of the corresponding
// vectors of integers needs to be widened.
if (ElemTy == MVT::i1) {
for (MVT T : Tys) {
assert(T != MVT::i1);
auto A = getPreferredHvxVectorAction(MVT::getVectorVT(T, VecLen));
if (A != ~0u)
return A;
}
return ~0u;
}
// If the size of VecTy is at least half of the vector length,
// widen the vector. Note: the threshold was not selected in
// any scientific way.
if (llvm::is_contained(Tys, ElemTy)) {
unsigned VecWidth = VecTy.getSizeInBits();
unsigned HwWidth = 8*HwLen;
if (VecWidth > 2*HwWidth)
return TargetLoweringBase::TypeSplitVector;
bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
return TargetLoweringBase::TypeWidenVector;
if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
return TargetLoweringBase::TypeWidenVector;
}
// Defer to default.
return ~0u;
}
unsigned
HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
unsigned Opc = Op.getOpcode();
switch (Opc) {
case HexagonISD::SMUL_LOHI:
case HexagonISD::UMUL_LOHI:
case HexagonISD::USMUL_LOHI:
return TargetLoweringBase::Custom;
}
return TargetLoweringBase::Legal;
}
SDValue
HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
const SDLoc &dl, SelectionDAG &DAG) const {
SmallVector<SDValue,4> IntOps;
IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
append_range(IntOps, Ops);
return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
}
MVT
HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
MVT ElemTy = Tys.first.getVectorElementType();
return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
Tys.second.getVectorNumElements());
}
HexagonTargetLowering::TypePair
HexagonTargetLowering::typeSplit(MVT VecTy) const {
assert(VecTy.isVector());
unsigned NumElem = VecTy.getVectorNumElements();
assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
return { HalfTy, HalfTy };
}
MVT
HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
MVT ElemTy = VecTy.getVectorElementType();
MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
}
MVT
HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
MVT ElemTy = VecTy.getVectorElementType();
MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
}
SDValue
HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
SelectionDAG &DAG) const {
if (ty(Vec).getVectorElementType() == ElemTy)
return Vec;
MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
return DAG.getBitcast(CastTy, Vec);
}
SDValue
HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
SelectionDAG &DAG) const {
return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
Ops.first, Ops.second);
}
HexagonTargetLowering::VectorPair
HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
SelectionDAG &DAG) const {
TypePair Tys = typeSplit(ty(Vec));
if (Vec.getOpcode() == HexagonISD::QCAT)
return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
}
bool
HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
return Subtarget.isHVXVectorType(Ty) &&
Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
}
bool
HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
return Subtarget.isHVXVectorType(Ty) &&
Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
}
bool
HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
return Subtarget.isHVXVectorType(Ty, true) &&
Ty.getVectorElementType() == MVT::i1;
}
bool HexagonTargetLowering::allowsHvxMemoryAccess(
MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
// Bool vectors are excluded by default, but make it explicit to
// emphasize that bool vectors cannot be loaded or stored.
// Also, disallow double vector stores (to prevent unnecessary
// store widening in DAG combiner).
if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
return false;
if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
return false;
if (Fast)
*Fast = 1;
return true;
}
bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const {
if (!Subtarget.isHVXVectorType(VecTy))
return false;
// XXX Should this be false? vmemu are a bit slower than vmem.
if (Fast)
*Fast = 1;
return true;
}
void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
MachineInstr &MI, SDNode *Node) const {
unsigned Opc = MI.getOpcode();
const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
MachineBasicBlock &MB = *MI.getParent();
MachineFunction &MF = *MB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
DebugLoc DL = MI.getDebugLoc();
auto At = MI.getIterator();
switch (Opc) {
case Hexagon::PS_vsplatib:
if (Subtarget.useHVXV62Ops()) {
// SplatV = A2_tfrsi #imm
// OutV = V6_lvsplatb SplatV
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
.add(MI.getOperand(1));
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV)
.addReg(SplatV);
} else {
// SplatV = A2_tfrsi #imm:#imm:#imm:#imm
// OutV = V6_lvsplatw SplatV
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
const MachineOperand &InpOp = MI.getOperand(1);
assert(InpOp.isImm());
uint32_t V = InpOp.getImm() & 0xFF;
BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
.addImm(V << 24 | V << 16 | V << 8 | V);
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
}
MB.erase(At);
break;
case Hexagon::PS_vsplatrb:
if (Subtarget.useHVXV62Ops()) {
// OutV = V6_lvsplatb Inp
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV)
.add(MI.getOperand(1));
} else {
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
const MachineOperand &InpOp = MI.getOperand(1);
BuildMI(MB, At, DL, TII.get(Hexagon::S2_vsplatrb), SplatV)
.addReg(InpOp.getReg(), 0, InpOp.getSubReg());
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV)
.addReg(SplatV);
}
MB.erase(At);
break;
case Hexagon::PS_vsplatih:
if (Subtarget.useHVXV62Ops()) {
// SplatV = A2_tfrsi #imm
// OutV = V6_lvsplath SplatV
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
.add(MI.getOperand(1));
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV)
.addReg(SplatV);
} else {
// SplatV = A2_tfrsi #imm:#imm
// OutV = V6_lvsplatw SplatV
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
const MachineOperand &InpOp = MI.getOperand(1);
assert(InpOp.isImm());
uint32_t V = InpOp.getImm() & 0xFFFF;
BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
.addImm(V << 16 | V);
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
}
MB.erase(At);
break;
case Hexagon::PS_vsplatrh:
if (Subtarget.useHVXV62Ops()) {
// OutV = V6_lvsplath Inp
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV)
.add(MI.getOperand(1));
} else {
// SplatV = A2_combine_ll Inp, Inp
// OutV = V6_lvsplatw SplatV
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
const MachineOperand &InpOp = MI.getOperand(1);
BuildMI(MB, At, DL, TII.get(Hexagon::A2_combine_ll), SplatV)
.addReg(InpOp.getReg(), 0, InpOp.getSubReg())
.addReg(InpOp.getReg(), 0, InpOp.getSubReg());
Register OutV = MI.getOperand(0).getReg();
BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
}
MB.erase(At);
break;
case Hexagon::PS_vsplatiw:
case Hexagon::PS_vsplatrw:
if (Opc == Hexagon::PS_vsplatiw) {
// SplatV = A2_tfrsi #imm
Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
.add(MI.getOperand(1));
MI.getOperand(1).ChangeToRegister(SplatV, false);
}
// OutV = V6_lvsplatw SplatV/Inp
MI.setDesc(TII.get(Hexagon::V6_lvsplatw));
break;
}
}
SDValue
HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
SelectionDAG &DAG) const {
if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
unsigned ElemWidth = ElemTy.getSizeInBits();
if (ElemWidth == 8)
return ElemIdx;
unsigned L = Log2_32(ElemWidth/8);
const SDLoc &dl(ElemIdx);
return DAG.getNode(ISD::SHL, dl, MVT::i32,
{ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
}
SDValue
HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
SelectionDAG &DAG) const {
unsigned ElemWidth = ElemTy.getSizeInBits();
assert(ElemWidth >= 8 && ElemWidth <= 32);
if (ElemWidth == 32)
return Idx;
if (ty(Idx) != MVT::i32)
Idx = DAG.getBitcast(MVT::i32, Idx);
const SDLoc &dl(Idx);
SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
return SubIdx;
}
SDValue
HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
SDValue Op1, ArrayRef<int> Mask,
SelectionDAG &DAG) const {
MVT OpTy = ty(Op0);
assert(OpTy == ty(Op1));
MVT ElemTy = OpTy.getVectorElementType();
if (ElemTy == MVT::i8)
return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
assert(ElemTy.getSizeInBits() >= 8);
MVT ResTy = tyVector(OpTy, MVT::i8);
unsigned ElemSize = ElemTy.getSizeInBits() / 8;
SmallVector<int,128> ByteMask;
for (int M : Mask) {
if (M < 0) {
for (unsigned I = 0; I != ElemSize; ++I)
ByteMask.push_back(-1);
} else {
int NewM = M*ElemSize;
for (unsigned I = 0; I != ElemSize; ++I)
ByteMask.push_back(NewM+I);
}
}
assert(ResTy.getVectorNumElements() == ByteMask.size());
return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
opCastElem(Op1, MVT::i8, DAG), ByteMask);
}
SDValue
HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
const SDLoc &dl, MVT VecTy,
SelectionDAG &DAG) const {
unsigned VecLen = Values.size();
MachineFunction &MF = DAG.getMachineFunction();
MVT ElemTy = VecTy.getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
unsigned HwLen = Subtarget.getVectorLength();
unsigned ElemSize = ElemWidth / 8;
assert(ElemSize*VecLen == HwLen);
SmallVector<SDValue,32> Words;
if (VecTy.getVectorElementType() != MVT::i32 &&
!(Subtarget.useHVXFloatingPoint() &&
VecTy.getVectorElementType() == MVT::f32)) {
assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
Words.push_back(DAG.getBitcast(MVT::i32, W));
}
} else {
for (SDValue V : Values)
Words.push_back(DAG.getBitcast(MVT::i32, V));
}
auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) {
unsigned NumValues = Values.size();
assert(NumValues > 0);
bool IsUndef = true;
for (unsigned i = 0; i != NumValues; ++i) {
if (Values[i].isUndef())
continue;
IsUndef = false;
if (!SplatV.getNode())
SplatV = Values[i];
else if (SplatV != Values[i])
return false;
}
if (IsUndef)
SplatV = Values[0];
return true;
};
unsigned NumWords = Words.size();
SDValue SplatV;
bool IsSplat = isSplat(Words, SplatV);
if (IsSplat && isUndef(SplatV))
return DAG.getUNDEF(VecTy);
if (IsSplat) {
assert(SplatV.getNode());
auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
if (IdxN && IdxN->isZero())
return getZero(dl, VecTy, DAG);
MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4);
SDValue S = DAG.getNode(ISD::SPLAT_VECTOR, dl, WordTy, SplatV);
return DAG.getBitcast(VecTy, S);
}
// Delay recognizing constant vectors until here, so that we can generate
// a vsplat.
SmallVector<ConstantInt*, 128> Consts(VecLen);
bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
if (AllConst) {
ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
(Constant**)Consts.end());
Constant *CV = ConstantVector::get(Tmp);
Align Alignment(HwLen);
SDValue CP =
LowerConstantPool(DAG.getConstantPool(CV, VecTy, Alignment), DAG);
return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
MachinePointerInfo::getConstantPool(MF), Alignment);
}
// A special case is a situation where the vector is built entirely from
// elements extracted from another vector. This could be done via a shuffle
// more efficiently, but typically, the size of the source vector will not
// match the size of the vector being built (which precludes the use of a
// shuffle directly).
// This only handles a single source vector, and the vector being built
// should be of a sub-vector type of the source vector type.
auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
SmallVectorImpl<int> &SrcIdx) {
SDValue Vec;
for (SDValue V : Values) {
if (isUndef(V)) {
SrcIdx.push_back(-1);
continue;
}
if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
return false;
// All extracts should come from the same vector.
SDValue T = V.getOperand(0);
if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
return false;
Vec = T;
ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
if (C == nullptr)
return false;
int I = C->getSExtValue();
assert(I >= 0 && "Negative element index");
SrcIdx.push_back(I);
}
SrcVec = Vec;
return true;
};
SmallVector<int,128> ExtIdx;
SDValue ExtVec;
if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
MVT ExtTy = ty(ExtVec);
unsigned ExtLen = ExtTy.getVectorNumElements();
if (ExtLen == VecLen || ExtLen == 2*VecLen) {
// Construct a new shuffle mask that will produce a vector with the same
// number of elements as the input vector, and such that the vector we
// want will be the initial subvector of it.
SmallVector<int,128> Mask;
BitVector Used(ExtLen);
for (int M : ExtIdx) {
Mask.push_back(M);
if (M >= 0)
Used.set(M);
}
// Fill the rest of the mask with the unused elements of ExtVec in hopes
// that it will result in a permutation of ExtVec's elements. It's still
// fine if it doesn't (e.g. if undefs are present, or elements are
// repeated), but permutations can always be done efficiently via vdelta
// and vrdelta.
for (unsigned I = 0; I != ExtLen; ++I) {
if (Mask.size() == ExtLen)
break;
if (!Used.test(I))
Mask.push_back(I);
}
SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
DAG.getUNDEF(ExtTy), Mask);
return ExtLen == VecLen ? S : LoHalf(S, DAG);
}
}
// Find most common element to initialize vector with. This is to avoid
// unnecessary vinsert/valign for cases where the same value is present
// many times. Creates a histogram of the vector's elements to find the
// most common element n.
assert(4*Words.size() == Subtarget.getVectorLength());
int VecHist[32];
int n = 0;
for (unsigned i = 0; i != NumWords; ++i) {
VecHist[i] = 0;
if (Words[i].isUndef())
continue;
for (unsigned j = i; j != NumWords; ++j)
if (Words[i] == Words[j])
VecHist[i]++;
if (VecHist[i] > VecHist[n])
n = i;
}
SDValue HalfV = getZero(dl, VecTy, DAG);
if (VecHist[n] > 1) {
SDValue SplatV = DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Words[n]);
HalfV = DAG.getNode(HexagonISD::VALIGN, dl, VecTy,
{HalfV, SplatV, DAG.getConstant(HwLen/2, dl, MVT::i32)});
}
SDValue HalfV0 = HalfV;
SDValue HalfV1 = HalfV;
// Construct two halves in parallel, then or them together. Rn and Rm count
// number of rotations needed before the next element. One last rotation is
// performed post-loop to position the last element.
int Rn = 0, Rm = 0;
SDValue Sn, Sm;
SDValue N = HalfV0;
SDValue M = HalfV1;
for (unsigned i = 0; i != NumWords/2; ++i) {
// Rotate by element count since last insertion.
if (Words[i] != Words[n] || VecHist[n] <= 1) {
Sn = DAG.getConstant(Rn, dl, MVT::i32);
HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn});
N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
{HalfV0, Words[i]});
Rn = 0;
}
if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
Sm = DAG.getConstant(Rm, dl, MVT::i32);
HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm});
M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
{HalfV1, Words[i+NumWords/2]});
Rm = 0;
}
Rn += 4;
Rm += 4;
}
// Perform last rotation.
Sn = DAG.getConstant(Rn+HwLen/2, dl, MVT::i32);
Sm = DAG.getConstant(Rm, dl, MVT::i32);
HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn});
HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm});
SDValue T0 = DAG.getBitcast(tyVector(VecTy, MVT::i32), HalfV0);
SDValue T1 = DAG.getBitcast(tyVector(VecTy, MVT::i32), HalfV1);
SDValue DstV = DAG.getNode(ISD::OR, dl, ty(T0), {T0, T1});
SDValue OutV =
DAG.getBitcast(tyVector(ty(DstV), VecTy.getVectorElementType()), DstV);
return OutV;
}
SDValue
HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
MVT PredTy = ty(PredV);
unsigned HwLen = Subtarget.getVectorLength();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
if (Subtarget.isHVXVectorType(PredTy, true)) {
// Move the vector predicate SubV to a vector register, and scale it
// down to match the representation (bytes per type element) that VecV
// uses. The scaling down will pick every 2nd or 4th (every Scale-th
// in general) element and put them at the front of the resulting
// vector. This subvector will then be inserted into the Q2V of VecV.
// To avoid having an operation that generates an illegal type (short
// vector), generate a full size vector.
//
SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
SmallVector<int,128> Mask(HwLen);
// Scale = BitBytes(PredV) / Given BitBytes.
unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
for (unsigned i = 0; i != HwLen; ++i) {
unsigned Num = i % Scale;
unsigned Off = i / Scale;
Mask[BlockLen*Num + Off] = i;
}
SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
if (!ZeroFill)
return S;
// Fill the bytes beyond BlockLen with 0s.
// V6_pred_scalar2 cannot fill the entire predicate, so it only works
// when BlockLen < HwLen.
assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
{DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
}
// Make sure that this is a valid scalar predicate.
assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
unsigned Bytes = 8 / PredTy.getVectorNumElements();
SmallVector<SDValue,4> Words[2];
unsigned IdxW = 0;
SDValue W0 = isUndef(PredV)
? DAG.getUNDEF(MVT::i64)
: DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
Words[IdxW].push_back(HiHalf(W0, DAG));
Words[IdxW].push_back(LoHalf(W0, DAG));
while (Bytes < BitBytes) {
IdxW ^= 1;
Words[IdxW].clear();
if (Bytes < 4) {
for (const SDValue &W : Words[IdxW ^ 1]) {
SDValue T = expandPredicate(W, dl, DAG);
Words[IdxW].push_back(HiHalf(T, DAG));
Words[IdxW].push_back(LoHalf(T, DAG));
}
} else {
for (const SDValue &W : Words[IdxW ^ 1]) {
Words[IdxW].push_back(W);
Words[IdxW].push_back(W);
}
}
Bytes *= 2;
}
assert(Bytes == BitBytes);
SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
for (const SDValue &W : Words[IdxW]) {
Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
}
return Vec;
}
SDValue
HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
const SDLoc &dl, MVT VecTy,
SelectionDAG &DAG) const {
// Construct a vector V of bytes, such that a comparison V >u 0 would
// produce the required vector predicate.
unsigned VecLen = Values.size();
unsigned HwLen = Subtarget.getVectorLength();
assert(VecLen <= HwLen || VecLen == 8*HwLen);
SmallVector<SDValue,128> Bytes;
bool AllT = true, AllF = true;
auto IsTrue = [] (SDValue V) {
if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
return !N->isZero();
return false;
};
auto IsFalse = [] (SDValue V) {
if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
return N->isZero();
return false;
};
if (VecLen <= HwLen) {
// In the hardware, each bit of a vector predicate corresponds to a byte
// of a vector register. Calculate how many bytes does a bit of VecTy
// correspond to.
assert(HwLen % VecLen == 0);
unsigned BitBytes = HwLen / VecLen;
for (SDValue V : Values) {
AllT &= IsTrue(V);
AllF &= IsFalse(V);
SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
: DAG.getUNDEF(MVT::i8);
for (unsigned B = 0; B != BitBytes; ++B)
Bytes.push_back(Ext);
}
} else {
// There are as many i1 values, as there are bits in a vector register.
// Divide the values into groups of 8 and check that each group consists
// of the same value (ignoring undefs).
for (unsigned I = 0; I != VecLen; I += 8) {
unsigned B = 0;
// Find the first non-undef value in this group.
for (; B != 8; ++B) {
if (!Values[I+B].isUndef())
break;
}
SDValue F = Values[I+B];
AllT &= IsTrue(F);
AllF &= IsFalse(F);
SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
: DAG.getUNDEF(MVT::i8);
Bytes.push_back(Ext);
// Verify that the rest of values in the group are the same as the
// first.
for (; B != 8; ++B)
assert(Values[I+B].isUndef() || Values[I+B] == F);
}
}
if (AllT)
return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
if (AllF)
return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
}
SDValue
HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
MVT ElemTy = ty(VecV).getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
assert(ElemWidth >= 8 && ElemWidth <= 32);
(void)ElemWidth;
SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
{VecV, ByteIdx});
if (ElemTy == MVT::i32)
return ExWord;
// Have an extracted word, need to extract the smaller element out of it.
// 1. Extract the bits of (the original) IdxV that correspond to the index
// of the desired element in the 32-bit word.
SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
// 2. Extract the element from the word.
SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
}
SDValue
HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
// Implement other return types if necessary.
assert(ResTy == MVT::i1);
unsigned HwLen = Subtarget.getVectorLength();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
}
SDValue
HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
MVT ElemTy = ty(VecV).getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
assert(ElemWidth >= 8 && ElemWidth <= 32);
(void)ElemWidth;
auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
SDValue ByteIdxV) {
MVT VecTy = ty(VecV);
unsigned HwLen = Subtarget.getVectorLength();
SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
{ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
{DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
return TorV;
};
SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
if (ElemTy == MVT::i32)
return InsertWord(VecV, ValV, ByteIdx);
// If this is not inserting a 32-bit word, convert it into such a thing.
// 1. Extract the existing word from the target vector.
SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
{ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
dl, MVT::i32, DAG);
// 2. Treating the extracted word as a 32-bit vector, insert the given
// value into it.
SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
MVT SubVecTy = tyVector(ty(Ext), ElemTy);
SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
ValV, SubIdx, dl, ElemTy, DAG);
// 3. Insert the 32-bit word back into the original vector.
return InsertWord(VecV, Ins, ByteIdx);
}
SDValue
HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
unsigned HwLen = Subtarget.getVectorLength();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
}
SDValue
HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
MVT VecTy = ty(VecV);
unsigned HwLen = Subtarget.getVectorLength();
unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
MVT ElemTy = VecTy.getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
// If the source vector is a vector pair, get the single vector containing
// the subvector of interest. The subvector will never overlap two single
// vectors.
if (isHvxPairTy(VecTy)) {
if (Idx * ElemWidth >= 8*HwLen)
Idx -= VecTy.getVectorNumElements() / 2;
VecV = OrigOp;
if (typeSplit(VecTy).first == ResTy)
return VecV;
}
// The only meaningful subvectors of a single HVX vector are those that
// fit in a scalar register.
assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
MVT WordTy = tyVector(VecTy, MVT::i32);
SDValue WordVec = DAG.getBitcast(WordTy, VecV);
unsigned WordIdx = (Idx*ElemWidth) / 32;
SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
if (ResTy.getSizeInBits() == 32)
return DAG.getBitcast(ResTy, W0);
SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
SDValue WW = getCombine(W1, W0, dl, MVT::i64, DAG);
return DAG.getBitcast(ResTy, WW);
}
SDValue
HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
MVT VecTy = ty(VecV);
unsigned HwLen = Subtarget.getVectorLength();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
// IdxV is required to be a constant.
unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
unsigned ResLen = ResTy.getVectorNumElements();
unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
unsigned Offset = Idx * BitBytes;
SDValue Undef = DAG.getUNDEF(ByteTy);
SmallVector<int,128> Mask;
if (Subtarget.isHVXVectorType(ResTy, true)) {
// Converting between two vector predicates. Since the result is shorter
// than the source, it will correspond to a vector predicate with the
// relevant bits replicated. The replication count is the ratio of the
// source and target vector lengths.
unsigned Rep = VecTy.getVectorNumElements() / ResLen;
assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
for (unsigned i = 0; i != HwLen/Rep; ++i) {
for (unsigned j = 0; j != Rep; ++j)
Mask.push_back(i + Offset);
}
SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
}
// Converting between a vector predicate and a scalar predicate. In the
// vector predicate, a group of BitBytes bits will correspond to a single
// i1 element of the source vector type. Those bits will all have the same
// value. The same will be true for ByteVec, where each byte corresponds
// to a bit in the vector predicate.
// The algorithm is to traverse the ByteVec, going over the i1 values from
// the source vector, and generate the corresponding representation in an
// 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
// elements so that the interesting 8 bytes will be in the low end of the
// vector.
unsigned Rep = 8 / ResLen;
// Make sure the output fill the entire vector register, so repeat the
// 8-byte groups as many times as necessary.
for (unsigned r = 0; r != HwLen/ResLen; ++r) {
// This will generate the indexes of the 8 interesting bytes.
for (unsigned i = 0; i != ResLen; ++i) {
for (unsigned j = 0; j != Rep; ++j)
Mask.push_back(Offset + i*BitBytes);
}
}
SDValue Zero = getZero(dl, MVT::i32, DAG);
SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
// Combine the two low words from ShuffV into a v8i8, and byte-compare
// them against 0.
SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
{ShuffV, DAG.getConstant(4, dl, MVT::i32)});
SDValue Vec64 = getCombine(W1, W0, dl, MVT::v8i8, DAG);
return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
{Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
}
SDValue
HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
MVT VecTy = ty(VecV);
MVT SubTy = ty(SubV);
unsigned HwLen = Subtarget.getVectorLength();
MVT ElemTy = VecTy.getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
bool IsPair = isHvxPairTy(VecTy);
MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
// The two single vectors that VecV consists of, if it's a pair.
SDValue V0, V1;
SDValue SingleV = VecV;
SDValue PickHi;
if (IsPair) {
V0 = LoHalf(VecV, DAG);
V1 = HiHalf(VecV, DAG);
SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
dl, MVT::i32);
PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
if (isHvxSingleTy(SubTy)) {
if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
unsigned Idx = CN->getZExtValue();
assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
}
// If IdxV is not a constant, generate the two variants: with the
// SubV as the high and as the low subregister, and select the right
// pair based on the IdxV.
SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
}
// The subvector being inserted must be entirely contained in one of
// the vectors V0 or V1. Set SingleV to the correct one, and update
// IdxV to be the index relative to the beginning of that vector.
SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
}
// The only meaningful subvectors of a single HVX vector are those that
// fit in a scalar register.
assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
// Convert IdxV to be index in bytes.
auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
if (!IdxN || !IdxN->isZero()) {
IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
DAG.getConstant(ElemWidth/8, dl, MVT::i32));
SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
}
// When inserting a single word, the rotation back to the original position
// would be by HwLen-Idx, but if two words are inserted, it will need to be
// by (HwLen-4)-Idx.
unsigned RolBase = HwLen;
if (SubTy.getSizeInBits() == 32) {
SDValue V = DAG.getBitcast(MVT::i32, SubV);
SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, V);
} else {
SDValue V = DAG.getBitcast(MVT::i64, SubV);
SDValue R0 = LoHalf(V, DAG);
SDValue R1 = HiHalf(V, DAG);
SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
DAG.getConstant(4, dl, MVT::i32));
SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
RolBase = HwLen-4;
}
// If the vector wasn't ror'ed, don't ror it back.
if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
}
if (IsPair) {
SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
}
return SingleV;
}
SDValue
HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
MVT VecTy = ty(VecV);
MVT SubTy = ty(SubV);
assert(Subtarget.isHVXVectorType(VecTy, true));
// VecV is an HVX vector predicate. SubV may be either an HVX vector
// predicate as well, or it can be a scalar predicate.
unsigned VecLen = VecTy.getVectorNumElements();
unsigned HwLen = Subtarget.getVectorLength();
assert(HwLen % VecLen == 0 && "Unexpected vector type");
unsigned Scale = VecLen / SubTy.getVectorNumElements();
unsigned BitBytes = HwLen / VecLen;
unsigned BlockLen = HwLen / Scale;
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
SDValue ByteIdx;
auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
if (!IdxN || !IdxN->isZero()) {
ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
DAG.getConstant(BitBytes, dl, MVT::i32));
ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
}
// ByteVec is the target vector VecV rotated in such a way that the
// subvector should be inserted at index 0. Generate a predicate mask
// and use vmux to do the insertion.
assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
{DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
// Rotate ByteVec back, and convert to a vector predicate.
if (!IdxN || !IdxN->isZero()) {
SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
}
return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
}
SDValue
HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
// Sign- and any-extending of a vector predicate to a vector register is
// equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
// a vector of 1s (where the 1s are of type matching the vector type).
assert(Subtarget.isHVXVectorType(ResTy));
if (!ZeroExt)
return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
SDValue True = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
DAG.getConstant(1, dl, MVT::i32));
SDValue False = getZero(dl, ResTy, DAG);
return DAG.getSelect(dl, ResTy, VecV, True, False);
}
SDValue
HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
MVT ResTy, SelectionDAG &DAG) const {
// Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
// (i.e. the entire predicate register) to bits [0..HwLen-1] of a
// vector register. The remaining bits of the vector register are
// unspecified.
MachineFunction &MF = DAG.getMachineFunction();
unsigned HwLen = Subtarget.getVectorLength();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
MVT PredTy = ty(VecQ);
unsigned PredLen = PredTy.getVectorNumElements();
assert(HwLen % PredLen == 0);
MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen);
Type *Int8Ty = Type::getInt8Ty(*DAG.getContext());
SmallVector<Constant*, 128> Tmp;
// Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
// These are bytes with the LSB rotated left with respect to their index.
for (unsigned i = 0; i != HwLen/8; ++i) {
for (unsigned j = 0; j != 8; ++j)
Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
}
Constant *CV = ConstantVector::get(Tmp);
Align Alignment(HwLen);
SDValue CP =
LowerConstantPool(DAG.getConstantPool(CV, ByteTy, Alignment), DAG);
SDValue Bytes =
DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP,
MachinePointerInfo::getConstantPool(MF), Alignment);
// Select the bytes that correspond to true bits in the vector predicate.
SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes),
getZero(dl, VecTy, DAG));
// Calculate the OR of all bytes in each group of 8. That will compress
// all the individual bits into a single byte.
// First, OR groups of 4, via vrmpy with 0x01010101.
SDValue All1 =
DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32));
SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
// Then rotate the accumulated vector by 4 bytes, and do the final OR.
SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
{Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG);
SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot});
// Pick every 8th byte and coalesce them at the beginning of the output.
// For symmetry, coalesce every 1+8th byte after that, then every 2+8th
// byte and so on.
SmallVector<int,128> Mask;
for (unsigned i = 0; i != HwLen; ++i)
Mask.push_back((8*i) % HwLen + i/(HwLen/8));
SDValue Collect =
DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask);
return DAG.getBitcast(ResTy, Collect);
}
SDValue
HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
const SDLoc &dl, SelectionDAG &DAG) const {
// Take a vector and resize the element type to match the given type.
MVT InpTy = ty(VecV);
if (InpTy == ResTy)
return VecV;
unsigned InpWidth = InpTy.getSizeInBits();
unsigned ResWidth = ResTy.getSizeInBits();
if (InpTy.isFloatingPoint()) {
return InpWidth < ResWidth ? DAG.getNode(ISD::FP_EXTEND, dl, ResTy, VecV)
: DAG.getNode(ISD::FP_ROUND, dl, ResTy, VecV,
getZero(dl, MVT::i32, DAG));
}
assert(InpTy.isInteger());
if (InpWidth < ResWidth) {
unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
return DAG.getNode(ExtOpc, dl, ResTy, VecV);
} else {
unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT;
return DAG.getNode(NarOpc, dl, ResTy, VecV, DAG.getValueType(ResTy));
}
}
SDValue
HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
SelectionDAG &DAG) const {
assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0);
const SDLoc &dl(Vec);
unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements();
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubTy,
{Vec, DAG.getConstant(ElemIdx, dl, MVT::i32)});
}
SDValue
HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
const {
const SDLoc &dl(Op);
MVT VecTy = ty(Op);
unsigned Size = Op.getNumOperands();
SmallVector<SDValue,128> Ops;
for (unsigned i = 0; i != Size; ++i)
Ops.push_back(Op.getOperand(i));
// First, split the BUILD_VECTOR for vector pairs. We could generate
// some pairs directly (via splat), but splats should be generated
// by the combiner prior to getting here.
if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
ArrayRef<SDValue> A(Ops);
MVT SingleTy = typeSplit(VecTy).first;
SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
}
if (VecTy.getVectorElementType() == MVT::i1)
return buildHvxVectorPred(Ops, dl, VecTy, DAG);
// In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is
// not a legal type, just bitcast the node to use i16
// types and bitcast the result back to f16
if (VecTy.getVectorElementType() == MVT::f16) {
SmallVector<SDValue,64> NewOps;
for (unsigned i = 0; i != Size; i++)
NewOps.push_back(DAG.getBitcast(MVT::i16, Ops[i]));
SDValue T0 = DAG.getNode(ISD::BUILD_VECTOR, dl,
tyVector(VecTy, MVT::i16), NewOps);
return DAG.getBitcast(tyVector(VecTy, MVT::f16), T0);
}
return buildHvxVectorReg(Ops, dl, VecTy, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG)
const {
const SDLoc &dl(Op);
MVT VecTy = ty(Op);
MVT ArgTy = ty(Op.getOperand(0));
if (ArgTy == MVT::f16) {
MVT SplatTy = MVT::getVectorVT(MVT::i16, VecTy.getVectorNumElements());
SDValue ToInt16 = DAG.getBitcast(MVT::i16, Op.getOperand(0));
SDValue ToInt32 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, ToInt16);
SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, dl, SplatTy, ToInt32);
return DAG.getBitcast(VecTy, Splat);
}
return SDValue();
}
SDValue
HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
const {
// Vector concatenation of two integer (non-bool) vectors does not need
// special lowering. Custom-lower concats of bool vectors and expand
// concats of more than 2 vectors.
MVT VecTy = ty(Op);
const SDLoc &dl(Op);
unsigned NumOp = Op.getNumOperands();
if (VecTy.getVectorElementType() != MVT::i1) {
if (NumOp == 2)
return Op;
// Expand the other cases into a build-vector.
SmallVector<SDValue,8> Elems;
for (SDValue V : Op.getNode()->ops())
DAG.ExtractVectorElements(V, Elems);
// A vector of i16 will be broken up into a build_vector of i16's.
// This is a problem, since at the time of operation legalization,
// all operations are expected to be type-legalized, and i16 is not
// a legal type. If any of the extracted elements is not of a valid
// type, sign-extend it to a valid one.
for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
SDValue V = Elems[i];
MVT Ty = ty(V);
if (!isTypeLegal(Ty)) {
MVT NTy = typeLegalize(Ty, DAG);
if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
V.getOperand(0), V.getOperand(1)),
DAG.getValueType(Ty));
continue;
}
// A few less complicated cases.
switch (V.getOpcode()) {
case ISD::Constant:
Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
break;
case ISD::UNDEF:
Elems[i] = DAG.getUNDEF(NTy);
break;
case ISD::TRUNCATE:
Elems[i] = V.getOperand(0);
break;
default:
llvm_unreachable("Unexpected vector element");
}
}
}
return DAG.getBuildVector(VecTy, dl, Elems);
}
assert(VecTy.getVectorElementType() == MVT::i1);
unsigned HwLen = Subtarget.getVectorLength();
assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
SDValue Op0 = Op.getOperand(0);
// If the operands are HVX types (i.e. not scalar predicates), then
// defer the concatenation, and create QCAT instead.
if (Subtarget.isHVXVectorType(ty(Op0), true)) {
if (NumOp == 2)
return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
ArrayRef<SDUse> U(Op.getNode()->ops());
SmallVector<SDValue,4> SV(U.begin(), U.end());
ArrayRef<SDValue> Ops(SV);
MVT HalfTy = typeSplit(VecTy).first;
SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
Ops.take_front(NumOp/2));
SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
Ops.take_back(NumOp/2));
return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
}
// Count how many bytes (in a vector register) each bit in VecTy
// corresponds to.
unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
SmallVector<SDValue,8> Prefixes;
for (SDValue V : Op.getNode()->op_values()) {
SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
Prefixes.push_back(P);
}
unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
SDValue Res = getZero(dl, ByteTy, DAG);
for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
}
return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
}
SDValue
HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
const {
// Change the type of the extracted element to i32.
SDValue VecV = Op.getOperand(0);
MVT ElemTy = ty(VecV).getVectorElementType();
const SDLoc &dl(Op);
SDValue IdxV = Op.getOperand(1);
if (ElemTy == MVT::i1)
return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
}
SDValue
HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
const {
const SDLoc &dl(Op);
MVT VecTy = ty(Op);
SDValue VecV = Op.getOperand(0);
SDValue ValV = Op.getOperand(1);
SDValue IdxV = Op.getOperand(2);
MVT ElemTy = ty(VecV).getVectorElementType();
if (ElemTy == MVT::i1)
return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
if (ElemTy == MVT::f16) {
SDValue T0 = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
tyVector(VecTy, MVT::i16),
DAG.getBitcast(tyVector(VecTy, MVT::i16), VecV),
DAG.getBitcast(MVT::i16, ValV), IdxV);
return DAG.getBitcast(tyVector(VecTy, MVT::f16), T0);
}
return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
const {
SDValue SrcV = Op.getOperand(0);
MVT SrcTy = ty(SrcV);
MVT DstTy = ty(Op);
SDValue IdxV = Op.getOperand(1);
unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
assert(Idx % DstTy.getVectorNumElements() == 0);
(void)Idx;
const SDLoc &dl(Op);
MVT ElemTy = SrcTy.getVectorElementType();
if (ElemTy == MVT::i1)
return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
return extractHvxSubvectorReg(Op, SrcV, IdxV, dl, DstTy, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
const {
// Idx does not need to be a constant.
SDValue VecV = Op.getOperand(0);
SDValue ValV = Op.getOperand(1);
SDValue IdxV = Op.getOperand(2);
const SDLoc &dl(Op);
MVT VecTy = ty(VecV);
MVT ElemTy = VecTy.getVectorElementType();
if (ElemTy == MVT::i1)
return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
// Lower any-extends of boolean vectors to sign-extends, since they
// translate directly to Q2V. Zero-extending could also be done equally
// fast, but Q2V is used/recognized in more places.
// For all other vectors, use zero-extend.
MVT ResTy = ty(Op);
SDValue InpV = Op.getOperand(0);
MVT ElemTy = ty(InpV).getVectorElementType();
if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
return LowerHvxSignExt(Op, DAG);
return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
}
SDValue
HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
MVT ResTy = ty(Op);
SDValue InpV = Op.getOperand(0);
MVT ElemTy = ty(InpV).getVectorElementType();
if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
return Op;
}
SDValue
HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
MVT ResTy = ty(Op);
SDValue InpV = Op.getOperand(0);
MVT ElemTy = ty(InpV).getVectorElementType();
if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
return Op;
}
SDValue
HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
// Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
// cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
const SDLoc &dl(Op);
MVT ResTy = ty(Op);
SDValue InpV = Op.getOperand(0);
assert(ResTy == ty(InpV));
// Calculate the vectors of 1 and bitwidth(x).
MVT ElemTy = ty(InpV).getVectorElementType();
unsigned ElemWidth = ElemTy.getSizeInBits();
SDValue Vec1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
DAG.getConstant(1, dl, MVT::i32));
SDValue VecW = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
DAG.getConstant(ElemWidth, dl, MVT::i32));
SDValue VecN1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy,
DAG.getConstant(-1, dl, MVT::i32));
// Do not use DAG.getNOT, because that would create BUILD_VECTOR with
// a BITCAST. Here we can skip the BITCAST (so we don't have to handle
// it separately in custom combine or selection).
SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
{DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
return DAG.getNode(ISD::SUB, dl, ResTy,
{VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
}
SDValue
HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
const SDLoc &dl(Op);
MVT ResTy = ty(Op);
assert(ResTy.getVectorElementType() == MVT::i32);
SDValue Vs = Op.getOperand(0);
SDValue Vt = Op.getOperand(1);
SDVTList ResTys = DAG.getVTList(ResTy, ResTy);
unsigned Opc = Op.getOpcode();
// On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI.
if (Opc == ISD::MULHU)
return DAG.getNode(HexagonISD::UMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1);
if (Opc == ISD::MULHS)
return DAG.getNode(HexagonISD::SMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1);
#ifndef NDEBUG
Op.dump(&DAG);
#endif
llvm_unreachable("Unexpected mulh operation");
}
SDValue
HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const {
const SDLoc &dl(Op);
unsigned Opc = Op.getOpcode();
SDValue Vu = Op.getOperand(0);
SDValue Vv = Op.getOperand(1);
// If the HI part is not used, convert it to a regular MUL.
if (auto HiVal = Op.getValue(1); HiVal.use_empty()) {
// Need to preserve the types and the number of values.
SDValue Hi = DAG.getUNDEF(ty(HiVal));
SDValue Lo = DAG.getNode(ISD::MUL, dl, ty(Op), {Vu, Vv});
return DAG.getMergeValues({Lo, Hi}, dl);
}
bool SignedVu = Opc == HexagonISD::SMUL_LOHI;
bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI;
// Legal on HVX v62+, but lower it here because patterns can't handle multi-
// valued nodes.
if (Subtarget.useHVXV62Ops())
return emitHvxMulLoHiV62(Vu, SignedVu, Vv, SignedVv, dl, DAG);
if (Opc == HexagonISD::SMUL_LOHI) {
// Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI,
// for other signedness LOHI is cheaper.
if (auto LoVal = Op.getValue(0); LoVal.use_empty()) {
SDValue Hi = emitHvxMulHsV60(Vu, Vv, dl, DAG);
SDValue Lo = DAG.getUNDEF(ty(LoVal));
return DAG.getMergeValues({Lo, Hi}, dl);
}
}
return emitHvxMulLoHiV60(Vu, SignedVu, Vv, SignedVv, dl, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
SDValue Val = Op.getOperand(0);
MVT ResTy = ty(Op);
MVT ValTy = ty(Val);
const SDLoc &dl(Op);
if (isHvxBoolTy(ValTy) && ResTy.isScalarInteger()) {
unsigned HwLen = Subtarget.getVectorLength();
MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4);
SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG);
unsigned BitWidth = ResTy.getSizeInBits();
if (BitWidth < 64) {
SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
dl, MVT::i32, DAG);
if (BitWidth == 32)
return W0;
assert(BitWidth < 32u);
return DAG.getZExtOrTrunc(W0, dl, ResTy);
}
// The result is >= 64 bits. The only options are 64 or 128.
assert(BitWidth == 64 || BitWidth == 128);
SmallVector<SDValue,4> Words;
for (unsigned i = 0; i != BitWidth/32; ++i) {
SDValue W = extractHvxElementReg(
VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
Words.push_back(W);
}
SmallVector<SDValue,2> Combines;
assert(Words.size() % 2 == 0);
for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
SDValue C = getCombine(Words[i+1], Words[i], dl, MVT::i64, DAG);
Combines.push_back(C);
}
if (BitWidth == 64)
return Combines[0];
return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines);
}
if (isHvxBoolTy(ResTy) && ValTy.isScalarInteger()) {
// Handle bitcast from i128 -> v128i1 and i64 -> v64i1.
unsigned BitWidth = ValTy.getSizeInBits();
unsigned HwLen = Subtarget.getVectorLength();
assert(BitWidth == HwLen);
MVT ValAsVecTy = MVT::getVectorVT(MVT::i8, BitWidth / 8);
SDValue ValAsVec = DAG.getBitcast(ValAsVecTy, Val);
// Splat each byte of Val 8 times.
// Bytes = [(b0)x8, (b1)x8, ...., (b15)x8]
// where b0, b1,..., b15 are least to most significant bytes of I.
SmallVector<SDValue, 128> Bytes;
// Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,...
// These are bytes with the LSB rotated left with respect to their index.
SmallVector<SDValue, 128> Tmp;
for (unsigned I = 0; I != HwLen / 8; ++I) {
SDValue Idx = DAG.getConstant(I, dl, MVT::i32);
SDValue Byte =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i8, ValAsVec, Idx);
for (unsigned J = 0; J != 8; ++J) {
Bytes.push_back(Byte);
Tmp.push_back(DAG.getConstant(1ull << J, dl, MVT::i8));
}
}
MVT ConstantVecTy = MVT::getVectorVT(MVT::i8, HwLen);
SDValue ConstantVec = DAG.getBuildVector(ConstantVecTy, dl, Tmp);
SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG);
// Each Byte in the I2V will be set iff corresponding bit is set in Val.
I2V = DAG.getNode(ISD::AND, dl, ConstantVecTy, {I2V, ConstantVec});
return DAG.getNode(HexagonISD::V2Q, dl, ResTy, I2V);
}
return Op;
}
SDValue
HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
// Sign- and zero-extends are legal.
assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op),
Op.getOperand(0));
}
SDValue
HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const {
MVT ResTy = ty(Op);
if (ResTy.getVectorElementType() != MVT::i1)
return Op;
const SDLoc &dl(Op);
unsigned HwLen = Subtarget.getVectorLength();
unsigned VecLen = ResTy.getVectorNumElements();
assert(HwLen % VecLen == 0);
unsigned ElemSize = HwLen / VecLen;
MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(ElemSize * 8), VecLen);
SDValue S =
DAG.getNode(ISD::SELECT, dl, VecTy, Op.getOperand(0),
DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(1)),
DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(2)));
return DAG.getNode(HexagonISD::V2Q, dl, ResTy, S);
}
SDValue
HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
if (SDValue S = getVectorShiftByInt(Op, DAG))
return S;
return Op;
}
SDValue
HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
SelectionDAG &DAG) const {
unsigned Opc = Op.getOpcode();
assert(Opc == ISD::FSHL || Opc == ISD::FSHR);
// Make sure the shift amount is within the range of the bitwidth
// of the element type.
SDValue A = Op.getOperand(0);
SDValue B = Op.getOperand(1);
SDValue S = Op.getOperand(2);
MVT InpTy = ty(A);
MVT ElemTy = InpTy.getVectorElementType();
const SDLoc &dl(Op);
unsigned ElemWidth = ElemTy.getSizeInBits();
bool IsLeft = Opc == ISD::FSHL;
// The expansion into regular shifts produces worse code for i8 and for
// right shift of i32 on v65+.
bool UseShifts = ElemTy != MVT::i8;
if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
UseShifts = false;
if (SDValue SplatV = getSplatValue(S, DAG); SplatV && UseShifts) {
// If this is a funnel shift by a scalar, lower it into regular shifts.
SDValue Mask = DAG.getConstant(ElemWidth - 1, dl, MVT::i32);
SDValue ModS =
DAG.getNode(ISD::AND, dl, MVT::i32,
{DAG.getZExtOrTrunc(SplatV, dl, MVT::i32), Mask});
SDValue NegS =
DAG.getNode(ISD::SUB, dl, MVT::i32,
{DAG.getConstant(ElemWidth, dl, MVT::i32), ModS});
SDValue IsZero =
DAG.getSetCC(dl, MVT::i1, ModS, getZero(dl, MVT::i32, DAG), ISD::SETEQ);
// FSHL A, B => A << | B >>n
// FSHR A, B => A <<n | B >>
SDValue Part1 =
DAG.getNode(HexagonISD::VASL, dl, InpTy, {A, IsLeft ? ModS : NegS});
SDValue Part2 =
DAG.getNode(HexagonISD::VLSR, dl, InpTy, {B, IsLeft ? NegS : ModS});
SDValue Or = DAG.getNode(ISD::OR, dl, InpTy, {Part1, Part2});
// If the shift amount was 0, pick A or B, depending on the direction.
// The opposite shift will also be by 0, so the "Or" will be incorrect.
return DAG.getNode(ISD::SELECT, dl, InpTy, {IsZero, (IsLeft ? A : B), Or});
}
SDValue Mask = DAG.getSplatBuildVector(
InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy));
unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR;
return DAG.getNode(MOpc, dl, ty(Op),
{A, B, DAG.getNode(ISD::AND, dl, InpTy, {S, Mask})});
}
SDValue
HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
const SDLoc &dl(Op);
unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
SmallVector<SDValue> Ops(Op->ops().begin(), Op->ops().end());
auto Swap = [&](SDValue P) {
return DAG.getMergeValues({P.getValue(1), P.getValue(0)}, dl);
};
switch (IntNo) {
case Intrinsic::hexagon_V6_pred_typecast:
case Intrinsic::hexagon_V6_pred_typecast_128B: {
MVT ResTy = ty(Op), InpTy = ty(Ops[1]);
if (isHvxBoolTy(ResTy) && isHvxBoolTy(InpTy)) {
if (ResTy == InpTy)
return Ops[1];
return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Ops[1]);
}
break;
}
case Intrinsic::hexagon_V6_vmpyss_parts:
case Intrinsic::hexagon_V6_vmpyss_parts_128B:
return Swap(DAG.getNode(HexagonISD::SMUL_LOHI, dl, Op->getVTList(),
{Ops[1], Ops[2]}));
case Intrinsic::hexagon_V6_vmpyuu_parts:
case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
return Swap(DAG.getNode(HexagonISD::UMUL_LOHI, dl, Op->getVTList(),
{Ops[1], Ops[2]}));
case Intrinsic::hexagon_V6_vmpyus_parts:
case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
return Swap(DAG.getNode(HexagonISD::USMUL_LOHI, dl, Op->getVTList(),
{Ops[1], Ops[2]}));
}
} // switch
return Op;
}
SDValue
HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const {
const SDLoc &dl(Op);
unsigned HwLen = Subtarget.getVectorLength();
MachineFunction &MF = DAG.getMachineFunction();
auto *MaskN = cast<MaskedLoadStoreSDNode>(Op.getNode());
SDValue Mask = MaskN->getMask();
SDValue Chain = MaskN->getChain();
SDValue Base = MaskN->getBasePtr();
auto *MemOp = MF.getMachineMemOperand(MaskN->getMemOperand(), 0, HwLen);
unsigned Opc = Op->getOpcode();
assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
if (Opc == ISD::MLOAD) {
MVT ValTy = ty(Op);
SDValue Load = DAG.getLoad(ValTy, dl, Chain, Base, MemOp);
SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru();
if (isUndef(Thru))
return Load;
SDValue VSel = DAG.getNode(ISD::VSELECT, dl, ValTy, Mask, Load, Thru);
return DAG.getMergeValues({VSel, Load.getValue(1)}, dl);
}
// MSTORE
// HVX only has aligned masked stores.
// TODO: Fold negations of the mask into the store.
unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue();
SDValue Offset0 = DAG.getTargetConstant(0, dl, ty(Base));
if (MaskN->getAlign().value() % HwLen == 0) {
SDValue Store = getInstr(StoreOpc, dl, MVT::Other,
{Mask, Base, Offset0, Value, Chain}, DAG);
DAG.setNodeMemRefs(cast<MachineSDNode>(Store.getNode()), {MemOp});
return Store;
}
// Unaligned case.
auto StoreAlign = [&](SDValue V, SDValue A) {
SDValue Z = getZero(dl, ty(V), DAG);
// TODO: use funnel shifts?
// vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the
// upper half.
SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG);
SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG);
return std::make_pair(LoV, HiV);
};
MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
SDValue MaskV = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Mask);
VectorPair Tmp = StoreAlign(MaskV, Base);
VectorPair MaskU = {DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.first),
DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.second)};
VectorPair ValueU = StoreAlign(Value, Base);
SDValue Offset1 = DAG.getTargetConstant(HwLen, dl, MVT::i32);
SDValue StoreLo =
getInstr(StoreOpc, dl, MVT::Other,
{MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
SDValue StoreHi =
getInstr(StoreOpc, dl, MVT::Other,
{MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
DAG.setNodeMemRefs(cast<MachineSDNode>(StoreLo.getNode()), {MemOp});
DAG.setNodeMemRefs(cast<MachineSDNode>(StoreHi.getNode()), {MemOp});
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, {StoreLo, StoreHi});
}
SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op,
SelectionDAG &DAG) const {
// This conversion only applies to QFloat. IEEE extension from f16 to f32
// is legal (done via a pattern).
assert(Subtarget.useHVXQFloatOps());
assert(Op->getOpcode() == ISD::FP_EXTEND);
MVT VecTy = ty(Op);
MVT ArgTy = ty(Op.getOperand(0));
const SDLoc &dl(Op);
assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
SDValue F16Vec = Op.getOperand(0);
APFloat FloatVal = APFloat(1.0f);
bool Ignored;
FloatVal.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &Ignored);
SDValue Fp16Ones = DAG.getConstantFP(FloatVal, dl, ArgTy);
SDValue VmpyVec =
getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG);
MVT HalfTy = typeSplit(VecTy).first;
VectorPair Pair = opSplit(VmpyVec, dl, DAG);
SDValue LoVec =
getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG);
SDValue HiVec =
getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG);
SDValue ShuffVec =
getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
{HiVec, LoVec, DAG.getConstant(-4, dl, MVT::i32)}, DAG);
return ShuffVec;
}
SDValue
HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const {
// Catch invalid conversion ops (just in case).
assert(Op.getOpcode() == ISD::FP_TO_SINT ||
Op.getOpcode() == ISD::FP_TO_UINT);
MVT ResTy = ty(Op);
MVT FpTy = ty(Op.getOperand(0)).getVectorElementType();
MVT IntTy = ResTy.getVectorElementType();
if (Subtarget.useHVXIEEEFPOps()) {
// There are only conversions from f16.
if (FpTy == MVT::f16) {
// Other int types aren't legal in HVX, so we shouldn't see them here.
assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
// Conversions to i8 and i16 are legal.
if (IntTy == MVT::i8 || IntTy == MVT::i16)
return Op;
}
}
if (IntTy.getSizeInBits() != FpTy.getSizeInBits())
return EqualizeFpIntConversion(Op, DAG);
return ExpandHvxFpToInt(Op, DAG);
}
SDValue
HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const {
// Catch invalid conversion ops (just in case).
assert(Op.getOpcode() == ISD::SINT_TO_FP ||
Op.getOpcode() == ISD::UINT_TO_FP);
MVT