| //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "HexagonISelLowering.h" |
| #include "HexagonRegisterInfo.h" |
| #include "HexagonSubtarget.h" |
| #include "llvm/ADT/SetVector.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/Analysis/MemoryLocation.h" |
| #include "llvm/CodeGen/MachineBasicBlock.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstr.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/TargetInstrInfo.h" |
| #include "llvm/IR/IntrinsicsHexagon.h" |
| #include "llvm/Support/CommandLine.h" |
| |
| #include <algorithm> |
| #include <string> |
| #include <utility> |
| |
| using namespace llvm; |
| |
| static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen", |
| cl::Hidden, cl::init(16), |
| cl::desc("Lower threshold (in bytes) for widening to HVX vectors")); |
| |
| static cl::opt<bool> |
| EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(false), |
| cl::desc("Enable FP fast conversion routine.")); |
| |
| static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 }; |
| static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; |
| static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 }; |
| static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 }; |
| |
| static const unsigned MaxExpandMLA = 8; |
| |
| static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) { |
| // For a float scalar type, return (exp-bits, exp-bias, fraction-bits) |
| MVT ElemTy = Ty.getScalarType(); |
| switch (ElemTy.SimpleTy) { |
| case MVT::f16: |
| return std::make_tuple(5, 15, 10); |
| case MVT::f32: |
| return std::make_tuple(8, 127, 23); |
| case MVT::f64: |
| return std::make_tuple(11, 1023, 52); |
| default: |
| break; |
| } |
| llvm_unreachable(("Unexpected type: " + EVT(ElemTy).getEVTString()).c_str()); |
| } |
| |
| void |
| HexagonTargetLowering::initializeHVXLowering() { |
| if (Subtarget.useHVX64BOps()) { |
| addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass); |
| // These "short" boolean vector types should be legal because |
| // they will appear as results of vector compares. If they were |
| // not legal, type legalization would try to make them legal |
| // and that would require using operations that do not use or |
| // produce such types. That, in turn, would imply using custom |
| // nodes, which would be unoptimizable by the DAG combiner. |
| // The idea is to rely on target-independent operations as much |
| // as possible. |
| addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass); |
| addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); |
| addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); |
| } else if (Subtarget.useHVX128BOps()) { |
| addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass); |
| addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass); |
| addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass); |
| if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) { |
| addRegisterClass(MVT::v32f32, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v64f16, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v64f32, &Hexagon::HvxWRRegClass); |
| addRegisterClass(MVT::v128f16, &Hexagon::HvxWRRegClass); |
| } |
| if (Subtarget.useHVXV81Ops()) { |
| addRegisterClass(MVT::v64bf16, &Hexagon::HvxVRRegClass); |
| addRegisterClass(MVT::v128bf16, &Hexagon::HvxWRRegClass); |
| } |
| } |
| |
| // Set up operation actions. |
| |
| bool Use64b = Subtarget.useHVX64BOps(); |
| ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128; |
| ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128; |
| MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8; |
| MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32; |
| MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8; |
| |
| auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) { |
| setOperationAction(Opc, FromTy, Promote); |
| AddPromotedToType(Opc, FromTy, ToTy); |
| }; |
| |
| // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32). |
| // Note: v16i1 -> i16 is handled in type legalization instead of op |
| // legalization. |
| setOperationAction(ISD::BITCAST, MVT::i16, Custom); |
| setOperationAction(ISD::BITCAST, MVT::i32, Custom); |
| setOperationAction(ISD::BITCAST, MVT::i64, Custom); |
| setOperationAction(ISD::BITCAST, MVT::v16i1, Custom); |
| setOperationAction(ISD::BITCAST, MVT::v128i1, Custom); |
| setOperationAction(ISD::BITCAST, MVT::i128, Custom); |
| setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal); |
| setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal); |
| setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); |
| |
| if (Subtarget.useHVX128BOps()) { |
| setOperationAction(ISD::BITCAST, MVT::v32i1, Custom); |
| setOperationAction(ISD::BITCAST, MVT::v64i1, Custom); |
| setOperationAction(ISD::STORE, MVT::v32i1, Custom); |
| setOperationAction(ISD::LOAD, MVT::v32i1, Custom); |
| setOperationAction(ISD::STORE, MVT::v64i1, Custom); |
| setOperationAction(ISD::LOAD, MVT::v64i1, Custom); |
| setOperationAction(ISD::STORE, MVT::v128i1, Custom); |
| setOperationAction(ISD::LOAD, MVT::v128i1, Custom); |
| } |
| if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() && |
| Subtarget.useHVXFloatingPoint()) { |
| |
| static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 }; |
| static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 }; |
| |
| for (MVT T : FloatV) { |
| setOperationAction(ISD::FADD, T, Legal); |
| setOperationAction(ISD::FSUB, T, Legal); |
| setOperationAction(ISD::FMUL, T, Legal); |
| setOperationAction(ISD::FMINIMUMNUM, T, Legal); |
| setOperationAction(ISD::FMAXIMUMNUM, T, Legal); |
| setOperationAction(ISD::FMINIMUM, T, Legal); |
| setOperationAction(ISD::FMAXIMUM, T, Legal); |
| setOperationAction(ISD::FMINNUM, T, Custom); |
| setOperationAction(ISD::FMAXNUM, T, Custom); |
| setOperationAction(ISD::VECREDUCE_FMIN, T, Custom); |
| setOperationAction(ISD::VECREDUCE_FMAX, T, Custom); |
| setOperationAction(ISD::VECREDUCE_FMINIMUM, T, Custom); |
| setOperationAction(ISD::VECREDUCE_FMAXIMUM, T, Custom); |
| |
| setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); |
| setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); |
| |
| setOperationAction(ISD::SPLAT_VECTOR, T, Legal); |
| setOperationAction(ISD::SPLAT_VECTOR, T, Legal); |
| |
| setOperationAction(ISD::MLOAD, T, Custom); |
| setOperationAction(ISD::MSTORE, T, Custom); |
| // Custom-lower BUILD_VECTOR. The standard (target-independent) |
| // handling of it would convert it to a load, which is not always |
| // the optimal choice. |
| setOperationAction(ISD::BUILD_VECTOR, T, Custom); |
| } |
| |
| |
| // BUILD_VECTOR with f16 operands cannot be promoted without |
| // promoting the result, so lower the node to vsplat or constant pool |
| setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom); |
| setOperationAction(ISD::SPLAT_VECTOR, MVT::f16, Custom); |
| |
| // Vector shuffle is always promoted to ByteV and a bitcast to f16 is |
| // generated. |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128f16, ByteW); |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f16, ByteV); |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64f32, ByteW); |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v32f32, ByteV); |
| |
| // For HVX <v81 there is no hardware float-equality instruction; only |
| // float-GT (V6_vgtsf/V6_vgthf) is available. The integer-equality |
| // fallback (V6_veqw/V6_veqh) silently treats NaN as equal to itself |
| // because the bit patterns match. Mark SETCC as Custom for the |
| // single-vector float types so we can synthesise the correct |
| // ordered-equal predicate in LowerHvxFpSetoeq. |
| if (!Subtarget.useHVXV81Ops()) |
| for (MVT T : FloatV) |
| setOperationAction(ISD::SETCC, T, Custom); |
| |
| if (Subtarget.useHVXV81Ops()) { |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v128bf16, ByteW); |
| setPromoteTo(ISD::VECTOR_SHUFFLE, MVT::v64bf16, ByteV); |
| setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32); |
| setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32); |
| setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32); |
| setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32); |
| setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32); |
| setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32); |
| |
| setOperationAction(ISD::SPLAT_VECTOR, MVT::v64bf16, Legal); |
| setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64bf16, Custom); |
| setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v64bf16, Custom); |
| |
| setOperationAction(ISD::LOAD, MVT::v128bf16, Custom); |
| setOperationAction(ISD::STORE, MVT::v128bf16, Custom); |
| |
| setOperationAction(ISD::MLOAD, MVT::v64bf16, Custom); |
| setOperationAction(ISD::MSTORE, MVT::v64bf16, Custom); |
| setOperationAction(ISD::BUILD_VECTOR, MVT::v64bf16, Custom); |
| setOperationAction(ISD::CONCAT_VECTORS, MVT::v64bf16, Custom); |
| |
| setOperationAction(ISD::MLOAD, MVT::v128bf16, Custom); |
| setOperationAction(ISD::MSTORE, MVT::v128bf16, Custom); |
| setOperationAction(ISD::BUILD_VECTOR, MVT::v128bf16, Custom); |
| setOperationAction(ISD::CONCAT_VECTORS, MVT::v128bf16, Custom); |
| |
| setOperationAction(ISD::SPLAT_VECTOR, MVT::bf16, Custom); |
| setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom); |
| } |
| |
| for (MVT P : FloatW) { |
| setOperationAction(ISD::LOAD, P, Custom); |
| setOperationAction(ISD::STORE, P, Custom); |
| setOperationAction(ISD::FADD, P, Custom); |
| setOperationAction(ISD::FSUB, P, Custom); |
| setOperationAction(ISD::FMUL, P, Custom); |
| setOperationAction(ISD::FMINIMUMNUM, P, Custom); |
| setOperationAction(ISD::FMAXIMUMNUM, P, Custom); |
| setOperationAction(ISD::FMINIMUM, P, Custom); |
| setOperationAction(ISD::FMAXIMUM, P, Custom); |
| setOperationAction(ISD::FMINNUM, P, Custom); |
| setOperationAction(ISD::FMAXNUM, P, Custom); |
| setOperationAction(ISD::VECREDUCE_FMIN, P, Custom); |
| setOperationAction(ISD::VECREDUCE_FMAX, P, Custom); |
| setOperationAction(ISD::VECREDUCE_FMINIMUM, P, Custom); |
| setOperationAction(ISD::VECREDUCE_FMAXIMUM, P, Custom); |
| setOperationAction(ISD::SETCC, P, Custom); |
| setOperationAction(ISD::VSELECT, P, Custom); |
| |
| // Custom-lower BUILD_VECTOR. The standard (target-independent) |
| // handling of it would convert it to a load, which is not always |
| // the optimal choice. |
| setOperationAction(ISD::BUILD_VECTOR, P, Custom); |
| // Make concat-vectors custom to handle concats of more than 2 vectors. |
| setOperationAction(ISD::CONCAT_VECTORS, P, Custom); |
| |
| setOperationAction(ISD::MLOAD, P, Custom); |
| setOperationAction(ISD::MSTORE, P, Custom); |
| } |
| |
| if (Subtarget.useHVXQFloatOps()) { |
| setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Custom); |
| setOperationAction(ISD::FP_ROUND, MVT::v64f16, Legal); |
| } else if (Subtarget.useHVXIEEEFPOps()) { |
| setOperationAction(ISD::FP_EXTEND, MVT::v64f32, Legal); |
| setOperationAction(ISD::FP_ROUND, MVT::v64f16, Legal); |
| } |
| } |
| |
| for (MVT T : LegalV) { |
| setIndexedLoadAction(ISD::POST_INC, T, Legal); |
| setIndexedStoreAction(ISD::POST_INC, T, Legal); |
| |
| setOperationAction(ISD::ABS, T, Legal); |
| setOperationAction(ISD::AND, T, Legal); |
| setOperationAction(ISD::OR, T, Legal); |
| setOperationAction(ISD::XOR, T, Legal); |
| setOperationAction(ISD::ADD, T, Legal); |
| setOperationAction(ISD::SUB, T, Legal); |
| setOperationAction(ISD::MUL, T, Legal); |
| setOperationAction(ISD::CTPOP, T, Legal); |
| setOperationAction(ISD::CTLZ, T, Legal); |
| setOperationAction(ISD::SELECT, T, Legal); |
| setOperationAction(ISD::SPLAT_VECTOR, T, Legal); |
| setOperationAction(ISD::UADDSAT, T, Legal); |
| setOperationAction(ISD::SADDSAT, T, Legal); |
| setOperationAction(ISD::USUBSAT, T, Legal); |
| setOperationAction(ISD::SSUBSAT, T, Legal); |
| if (T != ByteV) { |
| setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); |
| setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); |
| setOperationAction(ISD::BSWAP, T, Legal); |
| } |
| |
| setOperationAction(ISD::SMIN, T, Legal); |
| setOperationAction(ISD::SMAX, T, Legal); |
| if (T.getScalarType() != MVT::i32) { |
| setOperationAction(ISD::UMIN, T, Legal); |
| setOperationAction(ISD::UMAX, T, Legal); |
| } |
| |
| setOperationAction(ISD::CTTZ, T, Custom); |
| setOperationAction(ISD::LOAD, T, Custom); |
| setOperationAction(ISD::MLOAD, T, Custom); |
| setOperationAction(ISD::MSTORE, T, Custom); |
| if (T.getScalarType() != MVT::i32) { |
| setOperationAction(ISD::MULHS, T, Legal); |
| setOperationAction(ISD::MULHU, T, Legal); |
| } |
| |
| setOperationAction(ISD::BUILD_VECTOR, T, Custom); |
| // Make concat-vectors custom to handle concats of more than 2 vectors. |
| setOperationAction(ISD::CONCAT_VECTORS, T, Custom); |
| setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom); |
| setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom); |
| setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom); |
| setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom); |
| setOperationAction(ISD::ANY_EXTEND, T, Custom); |
| setOperationAction(ISD::SIGN_EXTEND, T, Custom); |
| setOperationAction(ISD::ZERO_EXTEND, T, Custom); |
| setOperationAction(ISD::FSHL, T, Custom); |
| setOperationAction(ISD::FSHR, T, Custom); |
| if (T != ByteV) { |
| setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); |
| // HVX only has shifts of words and halfwords. |
| setOperationAction(ISD::SRA, T, Custom); |
| setOperationAction(ISD::SHL, T, Custom); |
| setOperationAction(ISD::SRL, T, Custom); |
| |
| // Promote all shuffles to operate on vectors of bytes. |
| setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV); |
| } |
| |
| if (Subtarget.useHVXFloatingPoint()) { |
| // Same action for both QFloat and IEEE. |
| setOperationAction(ISD::SINT_TO_FP, T, Custom); |
| setOperationAction(ISD::UINT_TO_FP, T, Custom); |
| setOperationAction(ISD::FP_TO_SINT, T, Custom); |
| setOperationAction(ISD::FP_TO_UINT, T, Custom); |
| } |
| |
| setCondCodeAction(ISD::SETNE, T, Expand); |
| setCondCodeAction(ISD::SETLE, T, Expand); |
| setCondCodeAction(ISD::SETGE, T, Expand); |
| setCondCodeAction(ISD::SETLT, T, Expand); |
| setCondCodeAction(ISD::SETULE, T, Expand); |
| setCondCodeAction(ISD::SETUGE, T, Expand); |
| setCondCodeAction(ISD::SETULT, T, Expand); |
| } |
| |
| for (MVT T : LegalW) { |
| // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- |
| // independent) handling of it would convert it to a load, which is |
| // not always the optimal choice. |
| setOperationAction(ISD::BUILD_VECTOR, T, Custom); |
| // Make concat-vectors custom to handle concats of more than 2 vectors. |
| setOperationAction(ISD::CONCAT_VECTORS, T, Custom); |
| |
| // Custom-lower these operations for pairs. Expand them into a concat |
| // of the corresponding operations on individual vectors. |
| setOperationAction(ISD::ANY_EXTEND, T, Custom); |
| setOperationAction(ISD::SIGN_EXTEND, T, Custom); |
| setOperationAction(ISD::ZERO_EXTEND, T, Custom); |
| setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom); |
| setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom); |
| setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal); |
| setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal); |
| setOperationAction(ISD::SPLAT_VECTOR, T, Custom); |
| |
| setOperationAction(ISD::LOAD, T, Custom); |
| setOperationAction(ISD::STORE, T, Custom); |
| setOperationAction(ISD::MLOAD, T, Custom); |
| setOperationAction(ISD::MSTORE, T, Custom); |
| setOperationAction(ISD::ABS, T, Custom); |
| setOperationAction(ISD::CTLZ, T, Custom); |
| setOperationAction(ISD::CTTZ, T, Custom); |
| setOperationAction(ISD::CTPOP, T, Custom); |
| |
| setOperationAction(ISD::ADD, T, Legal); |
| setOperationAction(ISD::UADDSAT, T, Legal); |
| setOperationAction(ISD::SADDSAT, T, Legal); |
| setOperationAction(ISD::SUB, T, Legal); |
| setOperationAction(ISD::USUBSAT, T, Legal); |
| setOperationAction(ISD::SSUBSAT, T, Legal); |
| setOperationAction(ISD::MUL, T, Custom); |
| setOperationAction(ISD::MULHS, T, Custom); |
| setOperationAction(ISD::MULHU, T, Custom); |
| setOperationAction(ISD::AND, T, Custom); |
| setOperationAction(ISD::OR, T, Custom); |
| setOperationAction(ISD::XOR, T, Custom); |
| setOperationAction(ISD::SETCC, T, Custom); |
| setOperationAction(ISD::VSELECT, T, Custom); |
| if (T != ByteW) { |
| setOperationAction(ISD::SRA, T, Custom); |
| setOperationAction(ISD::SHL, T, Custom); |
| setOperationAction(ISD::SRL, T, Custom); |
| |
| // Promote all shuffles to operate on vectors of bytes. |
| setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW); |
| } |
| setOperationAction(ISD::FSHL, T, Custom); |
| setOperationAction(ISD::FSHR, T, Custom); |
| |
| setOperationAction(ISD::SMIN, T, Custom); |
| setOperationAction(ISD::SMAX, T, Custom); |
| if (T.getScalarType() != MVT::i32) { |
| setOperationAction(ISD::UMIN, T, Custom); |
| setOperationAction(ISD::UMAX, T, Custom); |
| } |
| |
| if (Subtarget.useHVXFloatingPoint()) { |
| // Same action for both QFloat and IEEE. |
| setOperationAction(ISD::SINT_TO_FP, T, Custom); |
| setOperationAction(ISD::UINT_TO_FP, T, Custom); |
| setOperationAction(ISD::FP_TO_SINT, T, Custom); |
| setOperationAction(ISD::FP_TO_UINT, T, Custom); |
| } |
| } |
| |
| // Legalize all of these to HexagonISD::[SU]MUL_LOHI. |
| setOperationAction(ISD::MULHS, WordV, Custom); // -> _LOHI |
| setOperationAction(ISD::MULHU, WordV, Custom); // -> _LOHI |
| setOperationAction(ISD::SMUL_LOHI, WordV, Custom); |
| setOperationAction(ISD::UMUL_LOHI, WordV, Custom); |
| |
| setCondCodeAction(ISD::SETNE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETLE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETGE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETLT, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETONE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETOLE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETOGE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETOLT, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETUNE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETULE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETUGE, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETULT, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETUO, MVT::v64f16, Expand); |
| setCondCodeAction(ISD::SETO, MVT::v64f16, Expand); |
| |
| setCondCodeAction(ISD::SETNE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETLE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETGE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETLT, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETONE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETOLE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETOGE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETOLT, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETUNE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETULE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETUGE, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETULT, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETUO, MVT::v32f32, Expand); |
| setCondCodeAction(ISD::SETO, MVT::v32f32, Expand); |
| |
| // Boolean vectors. |
| |
| for (MVT T : LegalW) { |
| // Boolean types for vector pairs will overlap with the boolean |
| // types for single vectors, e.g. |
| // v64i8 -> v64i1 (single) |
| // v64i16 -> v64i1 (pair) |
| // Set these actions first, and allow the single actions to overwrite |
| // any duplicates. |
| MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); |
| setOperationAction(ISD::SETCC, BoolW, Custom); |
| setOperationAction(ISD::AND, BoolW, Custom); |
| setOperationAction(ISD::OR, BoolW, Custom); |
| setOperationAction(ISD::XOR, BoolW, Custom); |
| // Masked load/store takes a mask that may need splitting. |
| setOperationAction(ISD::MLOAD, BoolW, Custom); |
| setOperationAction(ISD::MSTORE, BoolW, Custom); |
| } |
| |
| for (MVT T : LegalV) { |
| MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements()); |
| setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom); |
| setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom); |
| setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom); |
| setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom); |
| setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom); |
| setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom); |
| setOperationAction(ISD::SELECT, BoolV, Custom); |
| setOperationAction(ISD::AND, BoolV, Legal); |
| setOperationAction(ISD::OR, BoolV, Legal); |
| setOperationAction(ISD::XOR, BoolV, Legal); |
| } |
| |
| if (Use64b) { |
| for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32}) |
| setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal); |
| } else { |
| for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32}) |
| setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal); |
| } |
| |
| // Handle store widening for short vectors. |
| unsigned HwLen = Subtarget.getVectorLength(); |
| for (MVT ElemTy : Subtarget.getHVXElementTypes()) { |
| if (ElemTy == MVT::i1) |
| continue; |
| int ElemWidth = ElemTy.getFixedSizeInBits(); |
| int MaxElems = (8*HwLen) / ElemWidth; |
| for (int N = 2; N < MaxElems; N *= 2) { |
| MVT VecTy = MVT::getVectorVT(ElemTy, N); |
| auto Action = getPreferredVectorAction(VecTy); |
| if (Action == TargetLoweringBase::TypeWidenVector) { |
| setOperationAction(ISD::LOAD, VecTy, Custom); |
| setOperationAction(ISD::STORE, VecTy, Custom); |
| setOperationAction(ISD::SETCC, VecTy, Custom); |
| setOperationAction(ISD::TRUNCATE, VecTy, Custom); |
| setOperationAction(ISD::ANY_EXTEND, VecTy, Custom); |
| setOperationAction(ISD::SIGN_EXTEND, VecTy, Custom); |
| setOperationAction(ISD::ZERO_EXTEND, VecTy, Custom); |
| if (Subtarget.useHVXFloatingPoint()) { |
| setOperationAction(ISD::FP_TO_SINT, VecTy, Custom); |
| setOperationAction(ISD::FP_TO_UINT, VecTy, Custom); |
| setOperationAction(ISD::SINT_TO_FP, VecTy, Custom); |
| setOperationAction(ISD::UINT_TO_FP, VecTy, Custom); |
| } |
| |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, N); |
| if (!isTypeLegal(BoolTy)) |
| setOperationAction(ISD::SETCC, BoolTy, Custom); |
| } |
| } |
| } |
| |
| // Include cases which are not hander earlier |
| setOperationAction(ISD::UINT_TO_FP, MVT::v32i1, Custom); |
| setOperationAction(ISD::UINT_TO_FP, MVT::v64i1, Custom); |
| setOperationAction(ISD::SINT_TO_FP, MVT::v32i1, Custom); |
| |
| setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT}); |
| |
| setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA, |
| ISD::PARTIAL_REDUCE_SUMLA}); |
| |
| // Partial MLA reductions. |
| { |
| static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA, |
| ISD::PARTIAL_REDUCE_UMLA, |
| ISD::PARTIAL_REDUCE_SUMLA}; |
| |
| auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) { |
| return MVT::getVectorVT(ScalarT, Subtarget.getVectorLength() * Factor * |
| 8 / ScalarT.getSizeInBits()); |
| }; |
| |
| // Tuple of (Acc element type, input element type, vector pair). |
| // The assumption is both the input and reduction result are of the same |
| // size so the reduction ratio is the same as the ratio of element type |
| // sizes. This may not hold for all available instructions. |
| typedef std::tuple<MVT, MVT, bool> ReductionSignature; |
| |
| static const std::vector<ReductionSignature> NativeReductions = { |
| {MVT::i32, MVT::i8, false}, |
| }; |
| |
| for (const auto &R : NativeReductions) { |
| |
| MVT AccType = std::get<0>(R); |
| MVT InputType = std::get<1>(R); |
| unsigned Factor = std::get<2>(R) ? 2 : 1; |
| |
| // The native size is legal. |
| setPartialReduceMLAAction(MLAOps, HvxType(AccType), HvxType(InputType), |
| Legal); |
| |
| // Allow custom partial MLA reductions on larger vectors than legally |
| // supported. These reduction must be declared as Custom (or Legal) |
| // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern |
| // inserted when the partial reduction intrinsic is converted to |
| // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply |
| // on the original pattern, including the extensions and multiplies, |
| // which will make it impossible to match. |
| // There are two independent ways to extend the |
| // input size: 1. to concatenate the result - output vector is |
| // proportionally extended, 2) to reduce the result - the output vector |
| // size stays the same. We limit allowed combinations so that the total |
| // number of generated reduction instructions is limited by a constant |
| // number. This limit is arbitrary and can be revised. On one hand, it is |
| // convenient to have more choices; on the other hand, there is a |
| // diminishing benefit of very long sequences, which should probably be |
| // written as loops instead. |
| for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA; |
| ConcatFactor <<= 1) |
| for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA; |
| ReductionFactor <<= 1) |
| if (ConcatFactor * ReductionFactor != 1 && |
| ConcatFactor * ReductionFactor <= MaxExpandMLA) |
| setPartialReduceMLAAction( |
| MLAOps, HvxType(AccType, Factor * ConcatFactor), |
| HvxType(InputType, Factor * ConcatFactor * ReductionFactor), |
| Custom); |
| } |
| } |
| } |
| |
| unsigned |
| HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const { |
| // Early exit for invalid input types |
| if (!VecTy.isVector()) |
| return ~0u; |
| |
| MVT ElemTy = VecTy.getVectorElementType(); |
| unsigned VecLen = VecTy.getVectorNumElements(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| |
| // Split vectors of i1 that exceed byte vector length. |
| if (ElemTy == MVT::i1 && VecLen > HwLen) |
| return TargetLoweringBase::TypeSplitVector; |
| |
| ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes(); |
| // For shorter vectors of i1, widen them if any of the corresponding |
| // vectors of integers needs to be widened. |
| if (ElemTy == MVT::i1) { |
| for (MVT T : Tys) { |
| assert(T != MVT::i1); |
| auto A = getPreferredHvxVectorAction(MVT::getVectorVT(T, VecLen)); |
| if (A != ~0u) |
| return A; |
| } |
| return ~0u; |
| } |
| |
| // If the size of VecTy is at least half of the vector length, |
| // widen the vector. Note: the threshold was not selected in |
| // any scientific way. |
| if (llvm::is_contained(Tys, ElemTy)) { |
| unsigned VecWidth = VecTy.getSizeInBits(); |
| unsigned HwWidth = 8*HwLen; |
| if (VecWidth > 2*HwWidth) |
| return TargetLoweringBase::TypeSplitVector; |
| |
| bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0; |
| if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth) |
| return TargetLoweringBase::TypeWidenVector; |
| if (VecWidth >= HwWidth/2 && VecWidth < HwWidth) |
| return TargetLoweringBase::TypeWidenVector; |
| } |
| |
| // Defer to default. |
| return ~0u; |
| } |
| |
| unsigned |
| HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const { |
| unsigned Opc = Op.getOpcode(); |
| switch (Opc) { |
| case HexagonISD::SMUL_LOHI: |
| case HexagonISD::UMUL_LOHI: |
| case HexagonISD::USMUL_LOHI: |
| return TargetLoweringBase::Custom; |
| } |
| return TargetLoweringBase::Legal; |
| } |
| |
| SDValue |
| HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops, |
| const SDLoc &dl, SelectionDAG &DAG) const { |
| SmallVector<SDValue,4> IntOps; |
| IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32)); |
| append_range(IntOps, Ops); |
| return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps); |
| } |
| |
| MVT |
| HexagonTargetLowering::typeJoin(const TypePair &Tys) const { |
| assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType()); |
| |
| MVT ElemTy = Tys.first.getVectorElementType(); |
| return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() + |
| Tys.second.getVectorNumElements()); |
| } |
| |
| HexagonTargetLowering::TypePair |
| HexagonTargetLowering::typeSplit(MVT VecTy) const { |
| assert(VecTy.isVector()); |
| unsigned NumElem = VecTy.getVectorNumElements(); |
| assert((NumElem % 2) == 0 && "Expecting even-sized vector type"); |
| MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2); |
| return { HalfTy, HalfTy }; |
| } |
| |
| MVT |
| HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const { |
| MVT ElemTy = VecTy.getVectorElementType(); |
| MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor); |
| return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); |
| } |
| |
| MVT |
| HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const { |
| MVT ElemTy = VecTy.getVectorElementType(); |
| MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor); |
| return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements()); |
| } |
| |
| SDValue |
| HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy, |
| SelectionDAG &DAG) const { |
| if (ty(Vec).getVectorElementType() == ElemTy) |
| return Vec; |
| MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy); |
| return DAG.getBitcast(CastTy, Vec); |
| } |
| |
| SDValue |
| HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl, |
| SelectionDAG &DAG) const { |
| return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)), |
| Ops.first, Ops.second); |
| } |
| |
| HexagonTargetLowering::VectorPair |
| HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl, |
| SelectionDAG &DAG) const { |
| TypePair Tys = typeSplit(ty(Vec)); |
| if (Vec.getOpcode() == HexagonISD::QCAT) |
| return VectorPair(Vec.getOperand(0), Vec.getOperand(1)); |
| return DAG.SplitVector(Vec, dl, Tys.first, Tys.second); |
| } |
| |
| bool |
| HexagonTargetLowering::isHvxSingleTy(MVT Ty) const { |
| return Subtarget.isHVXVectorType(Ty) && |
| Ty.getSizeInBits() == 8 * Subtarget.getVectorLength(); |
| } |
| |
| bool |
| HexagonTargetLowering::isHvxPairTy(MVT Ty) const { |
| return Subtarget.isHVXVectorType(Ty) && |
| Ty.getSizeInBits() == 16 * Subtarget.getVectorLength(); |
| } |
| |
| bool |
| HexagonTargetLowering::isHvxBoolTy(MVT Ty) const { |
| return Subtarget.isHVXVectorType(Ty, true) && |
| Ty.getVectorElementType() == MVT::i1; |
| } |
| |
| bool HexagonTargetLowering::allowsHvxMemoryAccess( |
| MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const { |
| // Bool vectors are excluded by default, but make it explicit to |
| // emphasize that bool vectors cannot be loaded or stored. |
| // Also, disallow double vector stores (to prevent unnecessary |
| // store widening in DAG combiner). |
| if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength()) |
| return false; |
| if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false)) |
| return false; |
| if (Fast) |
| *Fast = 1; |
| return true; |
| } |
| |
| bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses( |
| MVT VecTy, MachineMemOperand::Flags Flags, unsigned *Fast) const { |
| if (!Subtarget.isHVXVectorType(VecTy)) |
| return false; |
| // XXX Should this be false? vmemu are a bit slower than vmem. |
| if (Fast) |
| *Fast = 1; |
| return true; |
| } |
| |
| void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection( |
| MachineInstr &MI, SDNode *Node) const { |
| unsigned Opc = MI.getOpcode(); |
| const TargetInstrInfo &TII = *Subtarget.getInstrInfo(); |
| MachineBasicBlock &MB = *MI.getParent(); |
| MachineFunction &MF = *MB.getParent(); |
| MachineRegisterInfo &MRI = MF.getRegInfo(); |
| DebugLoc DL = MI.getDebugLoc(); |
| auto At = MI.getIterator(); |
| |
| switch (Opc) { |
| case Hexagon::PS_vsplatib: |
| if (Subtarget.useHVXV62Ops()) { |
| // SplatV = A2_tfrsi #imm |
| // OutV = V6_lvsplatb SplatV |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV) |
| .add(MI.getOperand(1)); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV) |
| .addReg(SplatV); |
| } else { |
| // SplatV = A2_tfrsi #imm:#imm:#imm:#imm |
| // OutV = V6_lvsplatw SplatV |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| const MachineOperand &InpOp = MI.getOperand(1); |
| assert(InpOp.isImm()); |
| uint32_t V = InpOp.getImm() & 0xFF; |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV) |
| .addImm(V << 24 | V << 16 | V << 8 | V); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV); |
| } |
| MB.erase(At); |
| break; |
| case Hexagon::PS_vsplatrb: |
| if (Subtarget.useHVXV62Ops()) { |
| // OutV = V6_lvsplatb Inp |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV) |
| .add(MI.getOperand(1)); |
| } else { |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| const MachineOperand &InpOp = MI.getOperand(1); |
| BuildMI(MB, At, DL, TII.get(Hexagon::S2_vsplatrb), SplatV) |
| .addReg(InpOp.getReg(), {}, InpOp.getSubReg()); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV) |
| .addReg(SplatV); |
| } |
| MB.erase(At); |
| break; |
| case Hexagon::PS_vsplatih: |
| if (Subtarget.useHVXV62Ops()) { |
| // SplatV = A2_tfrsi #imm |
| // OutV = V6_lvsplath SplatV |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV) |
| .add(MI.getOperand(1)); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV) |
| .addReg(SplatV); |
| } else { |
| // SplatV = A2_tfrsi #imm:#imm |
| // OutV = V6_lvsplatw SplatV |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| const MachineOperand &InpOp = MI.getOperand(1); |
| assert(InpOp.isImm()); |
| uint32_t V = InpOp.getImm() & 0xFFFF; |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV) |
| .addImm(V << 16 | V); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV); |
| } |
| MB.erase(At); |
| break; |
| case Hexagon::PS_vsplatrh: |
| if (Subtarget.useHVXV62Ops()) { |
| // OutV = V6_lvsplath Inp |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV) |
| .add(MI.getOperand(1)); |
| } else { |
| // SplatV = A2_combine_ll Inp, Inp |
| // OutV = V6_lvsplatw SplatV |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| const MachineOperand &InpOp = MI.getOperand(1); |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_combine_ll), SplatV) |
| .addReg(InpOp.getReg(), {}, InpOp.getSubReg()) |
| .addReg(InpOp.getReg(), {}, InpOp.getSubReg()); |
| Register OutV = MI.getOperand(0).getReg(); |
| BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV); |
| } |
| MB.erase(At); |
| break; |
| case Hexagon::PS_vsplatiw: |
| case Hexagon::PS_vsplatrw: |
| if (Opc == Hexagon::PS_vsplatiw) { |
| // SplatV = A2_tfrsi #imm |
| Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass); |
| BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV) |
| .add(MI.getOperand(1)); |
| MI.getOperand(1).ChangeToRegister(SplatV, false); |
| } |
| // OutV = V6_lvsplatw SplatV/Inp |
| MI.setDesc(TII.get(Hexagon::V6_lvsplatw)); |
| break; |
| } |
| } |
| |
| SDValue |
| HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy, |
| SelectionDAG &DAG) const { |
| if (ElemIdx.getValueType().getSimpleVT() != MVT::i32) |
| ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx); |
| |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| if (ElemWidth == 8) |
| return ElemIdx; |
| |
| unsigned L = Log2_32(ElemWidth/8); |
| const SDLoc &dl(ElemIdx); |
| return DAG.getNode(ISD::SHL, dl, MVT::i32, |
| {ElemIdx, DAG.getConstant(L, dl, MVT::i32)}); |
| } |
| |
| SDValue |
| HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy, |
| SelectionDAG &DAG) const { |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| assert(ElemWidth >= 8 && ElemWidth <= 32); |
| if (ElemWidth == 32) |
| return Idx; |
| |
| if (ty(Idx) != MVT::i32) |
| Idx = DAG.getBitcast(MVT::i32, Idx); |
| const SDLoc &dl(Idx); |
| SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32); |
| SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask}); |
| return SubIdx; |
| } |
| |
| SDValue |
| HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0, |
| SDValue Op1, ArrayRef<int> Mask, |
| SelectionDAG &DAG) const { |
| MVT OpTy = ty(Op0); |
| assert(OpTy == ty(Op1)); |
| |
| MVT ElemTy = OpTy.getVectorElementType(); |
| if (ElemTy == MVT::i8) |
| return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask); |
| assert(ElemTy.getSizeInBits() >= 8); |
| |
| MVT ResTy = tyVector(OpTy, MVT::i8); |
| unsigned ElemSize = ElemTy.getSizeInBits() / 8; |
| |
| SmallVector<int,128> ByteMask; |
| for (int M : Mask) { |
| if (M < 0) { |
| for (unsigned I = 0; I != ElemSize; ++I) |
| ByteMask.push_back(-1); |
| } else { |
| int NewM = M*ElemSize; |
| for (unsigned I = 0; I != ElemSize; ++I) |
| ByteMask.push_back(NewM+I); |
| } |
| } |
| assert(ResTy.getVectorNumElements() == ByteMask.size()); |
| return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG), |
| opCastElem(Op1, MVT::i8, DAG), ByteMask); |
| } |
| |
| SDValue |
| HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values, |
| const SDLoc &dl, MVT VecTy, |
| SelectionDAG &DAG) const { |
| unsigned VecLen = Values.size(); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| MVT ElemTy = VecTy.getVectorElementType(); |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| |
| unsigned ElemSize = ElemWidth / 8; |
| assert(ElemSize*VecLen == HwLen); |
| SmallVector<SDValue,32> Words; |
| |
| if (VecTy.getVectorElementType() != MVT::i32 && |
| !(Subtarget.useHVXFloatingPoint() && |
| VecTy.getVectorElementType() == MVT::f32)) { |
| assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size"); |
| unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2; |
| MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord); |
| for (unsigned i = 0; i != VecLen; i += OpsPerWord) { |
| SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG); |
| Words.push_back(DAG.getBitcast(MVT::i32, W)); |
| } |
| } else { |
| for (SDValue V : Values) |
| Words.push_back(DAG.getBitcast(MVT::i32, V)); |
| } |
| auto isSplat = [] (ArrayRef<SDValue> Values, SDValue &SplatV) { |
| unsigned NumValues = Values.size(); |
| assert(NumValues > 0); |
| bool IsUndef = true; |
| for (unsigned i = 0; i != NumValues; ++i) { |
| if (Values[i].isUndef()) |
| continue; |
| IsUndef = false; |
| if (!SplatV.getNode()) |
| SplatV = Values[i]; |
| else if (SplatV != Values[i]) |
| return false; |
| } |
| if (IsUndef) |
| SplatV = Values[0]; |
| return true; |
| }; |
| |
| unsigned NumWords = Words.size(); |
| SDValue SplatV; |
| bool IsSplat = isSplat(Words, SplatV); |
| if (IsSplat && isUndef(SplatV)) |
| return DAG.getUNDEF(VecTy); |
| if (IsSplat) { |
| assert(SplatV.getNode()); |
| if (isNullConstant(SplatV)) |
| return getZero(dl, VecTy, DAG); |
| MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4); |
| SDValue S = DAG.getNode(ISD::SPLAT_VECTOR, dl, WordTy, SplatV); |
| return DAG.getBitcast(VecTy, S); |
| } |
| |
| // Delay recognizing constant vectors until here, so that we can generate |
| // a vsplat. |
| SmallVector<ConstantInt*, 128> Consts(VecLen); |
| bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts); |
| if (AllConst) { |
| ArrayRef<Constant*> Tmp((Constant**)Consts.begin(), |
| (Constant**)Consts.end()); |
| Constant *CV = ConstantVector::get(Tmp); |
| Align Alignment(HwLen); |
| SDValue CP = LowerConstantPool( |
| DAG.getConstantPool(CV, getPointerTy(DAG.getDataLayout()), Alignment), |
| DAG); |
| return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP, |
| MachinePointerInfo::getConstantPool(MF), Alignment); |
| } |
| |
| // A special case is a situation where the vector is built entirely from |
| // elements extracted from another vector. This could be done via a shuffle |
| // more efficiently, but typically, the size of the source vector will not |
| // match the size of the vector being built (which precludes the use of a |
| // shuffle directly). |
| // This only handles a single source vector, and the vector being built |
| // should be of a sub-vector type of the source vector type. |
| auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec, |
| SmallVectorImpl<int> &SrcIdx) { |
| SDValue Vec; |
| for (SDValue V : Values) { |
| if (isUndef(V)) { |
| SrcIdx.push_back(-1); |
| continue; |
| } |
| if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) |
| return false; |
| // All extracts should come from the same vector. |
| SDValue T = V.getOperand(0); |
| if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode()) |
| return false; |
| Vec = T; |
| ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1)); |
| if (C == nullptr) |
| return false; |
| int I = C->getSExtValue(); |
| assert(I >= 0 && "Negative element index"); |
| SrcIdx.push_back(I); |
| } |
| SrcVec = Vec; |
| return true; |
| }; |
| |
| SmallVector<int,128> ExtIdx; |
| SDValue ExtVec; |
| if (IsBuildFromExtracts(ExtVec, ExtIdx)) { |
| MVT ExtTy = ty(ExtVec); |
| unsigned ExtLen = ExtTy.getVectorNumElements(); |
| if (ExtLen == VecLen || ExtLen == 2*VecLen) { |
| // Construct a new shuffle mask that will produce a vector with the same |
| // number of elements as the input vector, and such that the vector we |
| // want will be the initial subvector of it. |
| SmallVector<int,128> Mask; |
| BitVector Used(ExtLen); |
| |
| for (int M : ExtIdx) { |
| Mask.push_back(M); |
| if (M >= 0) |
| Used.set(M); |
| } |
| // Fill the rest of the mask with the unused elements of ExtVec in hopes |
| // that it will result in a permutation of ExtVec's elements. It's still |
| // fine if it doesn't (e.g. if undefs are present, or elements are |
| // repeated), but permutations can always be done efficiently via vdelta |
| // and vrdelta. |
| for (unsigned I = 0; I != ExtLen; ++I) { |
| if (Mask.size() == ExtLen) |
| break; |
| if (!Used.test(I)) |
| Mask.push_back(I); |
| } |
| |
| SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec, |
| DAG.getUNDEF(ExtTy), Mask); |
| return ExtLen == VecLen ? S : LoHalf(S, DAG); |
| } |
| } |
| |
| // Find most common element to initialize vector with. This is to avoid |
| // unnecessary vinsert/valign for cases where the same value is present |
| // many times. Creates a histogram of the vector's elements to find the |
| // most common element n. |
| assert(4*Words.size() == Subtarget.getVectorLength()); |
| int VecHist[32]; |
| int n = 0; |
| for (unsigned i = 0; i != NumWords; ++i) { |
| VecHist[i] = 0; |
| if (Words[i].isUndef()) |
| continue; |
| for (unsigned j = i; j != NumWords; ++j) |
| if (Words[i] == Words[j]) |
| VecHist[i]++; |
| |
| if (VecHist[i] > VecHist[n]) |
| n = i; |
| } |
| |
| SDValue HalfV = getZero(dl, VecTy, DAG); |
| if (VecHist[n] > 1) { |
| // Always splat at word (i32) granularity so that the SPLAT_VECTOR node |
| // is selected as PS_vsplatrw (word broadcast) rather than PS_vsplatrb |
| // (byte broadcast of the low byte only), which would corrupt multi-byte |
| // element types. |
| MVT WordVecTy = MVT::getVectorVT(MVT::i32, HwLen / 4); |
| SDValue WordSplat = DAG.getNode(ISD::SPLAT_VECTOR, dl, WordVecTy, Words[n]); |
| SDValue SplatV = DAG.getBitcast(VecTy, WordSplat); |
| HalfV = DAG.getNode(HexagonISD::VALIGN, dl, VecTy, |
| {HalfV, SplatV, DAG.getConstant(HwLen/2, dl, MVT::i32)}); |
| } |
| SDValue HalfV0 = HalfV; |
| SDValue HalfV1 = HalfV; |
| |
| // Construct two halves in parallel, then or them together. Rn and Rm count |
| // number of rotations needed before the next element. One last rotation is |
| // performed post-loop to position the last element. |
| int Rn = 0, Rm = 0; |
| SDValue Sn, Sm; |
| SDValue N = HalfV0; |
| SDValue M = HalfV1; |
| for (unsigned i = 0; i != NumWords/2; ++i) { |
| // Rotate by element count since last insertion. |
| if (Words[i] != Words[n] || VecHist[n] <= 1) { |
| Sn = DAG.getConstant(Rn, dl, MVT::i32); |
| HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn}); |
| N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, |
| {HalfV0, Words[i]}); |
| Rn = 0; |
| } |
| if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) { |
| Sm = DAG.getConstant(Rm, dl, MVT::i32); |
| HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm}); |
| M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, |
| {HalfV1, Words[i+NumWords/2]}); |
| Rm = 0; |
| } |
| Rn += 4; |
| Rm += 4; |
| } |
| // Perform last rotation. |
| Sn = DAG.getConstant(Rn+HwLen/2, dl, MVT::i32); |
| Sm = DAG.getConstant(Rm, dl, MVT::i32); |
| HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, Sn}); |
| HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, Sm}); |
| |
| SDValue T0 = DAG.getBitcast(tyVector(VecTy, MVT::i32), HalfV0); |
| SDValue T1 = DAG.getBitcast(tyVector(VecTy, MVT::i32), HalfV1); |
| |
| SDValue DstV = DAG.getNode(ISD::OR, dl, ty(T0), {T0, T1}); |
| |
| SDValue OutV = |
| DAG.getBitcast(tyVector(ty(DstV), VecTy.getVectorElementType()), DstV); |
| return OutV; |
| } |
| |
| SDValue |
| HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl, |
| unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const { |
| MVT PredTy = ty(PredV); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| |
| if (Subtarget.isHVXVectorType(PredTy, true)) { |
| // Move the vector predicate SubV to a vector register, and scale it |
| // down to match the representation (bytes per type element) that VecV |
| // uses. The scaling down will pick every 2nd or 4th (every Scale-th |
| // in general) element and put them at the front of the resulting |
| // vector. This subvector will then be inserted into the Q2V of VecV. |
| // To avoid having an operation that generates an illegal type (short |
| // vector), generate a full size vector. |
| // |
| SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV); |
| SmallVector<int,128> Mask(HwLen); |
| // Scale = BitBytes(PredV) / Given BitBytes. |
| unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes); |
| unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes; |
| |
| for (unsigned i = 0; i != HwLen; ++i) { |
| unsigned Num = i % Scale; |
| unsigned Off = i / Scale; |
| Mask[BlockLen*Num + Off] = i; |
| } |
| SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask); |
| if (!ZeroFill) |
| return S; |
| // Fill the bytes beyond BlockLen with 0s. |
| // V6_pred_scalar2 cannot fill the entire predicate, so it only works |
| // when BlockLen < HwLen. |
| assert(BlockLen < HwLen && "vsetq(v1) prerequisite"); |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); |
| SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, |
| {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); |
| SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q); |
| return DAG.getNode(ISD::AND, dl, ByteTy, S, M); |
| } |
| |
| // Make sure that this is a valid scalar predicate. |
| assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1); |
| |
| unsigned Bytes = 8 / PredTy.getVectorNumElements(); |
| SmallVector<SDValue,4> Words[2]; |
| unsigned IdxW = 0; |
| |
| SDValue W0 = isUndef(PredV) |
| ? DAG.getUNDEF(MVT::i64) |
| : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV); |
| Words[IdxW].push_back(HiHalf(W0, DAG)); |
| Words[IdxW].push_back(LoHalf(W0, DAG)); |
| |
| while (Bytes < BitBytes) { |
| IdxW ^= 1; |
| Words[IdxW].clear(); |
| |
| if (Bytes < 4) { |
| for (const SDValue &W : Words[IdxW ^ 1]) { |
| SDValue T = expandPredicate(W, dl, DAG); |
| Words[IdxW].push_back(HiHalf(T, DAG)); |
| Words[IdxW].push_back(LoHalf(T, DAG)); |
| } |
| } else { |
| for (const SDValue &W : Words[IdxW ^ 1]) { |
| Words[IdxW].push_back(W); |
| Words[IdxW].push_back(W); |
| } |
| } |
| Bytes *= 2; |
| } |
| |
| assert(Bytes == BitBytes); |
| SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy); |
| SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32); |
| for (const SDValue &W : Words[IdxW]) { |
| Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4); |
| Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W); |
| } |
| |
| return Vec; |
| } |
| |
| SDValue |
| HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values, |
| const SDLoc &dl, MVT VecTy, |
| SelectionDAG &DAG) const { |
| // Construct a vector V of bytes, such that a comparison V >u 0 would |
| // produce the required vector predicate. |
| unsigned VecLen = Values.size(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| assert(VecLen <= HwLen || VecLen == 8*HwLen); |
| SmallVector<SDValue,128> Bytes; |
| bool AllT = true, AllF = true; |
| |
| auto IsTrue = [] (SDValue V) { |
| if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) |
| return !N->isZero(); |
| return false; |
| }; |
| auto IsFalse = [] (SDValue V) { |
| if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode())) |
| return N->isZero(); |
| return false; |
| }; |
| |
| if (VecLen <= HwLen) { |
| // In the hardware, each bit of a vector predicate corresponds to a byte |
| // of a vector register. Calculate how many bytes does a bit of VecTy |
| // correspond to. |
| assert(HwLen % VecLen == 0); |
| unsigned BitBytes = HwLen / VecLen; |
| for (SDValue V : Values) { |
| AllT &= IsTrue(V); |
| AllF &= IsFalse(V); |
| |
| SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8) |
| : DAG.getUNDEF(MVT::i8); |
| for (unsigned B = 0; B != BitBytes; ++B) |
| Bytes.push_back(Ext); |
| } |
| } else { |
| // There are as many i1 values, as there are bits in a vector register. |
| // Divide the values into groups of 8 and check that each group consists |
| // of the same value (ignoring undefs). |
| for (unsigned I = 0; I != VecLen; I += 8) { |
| unsigned B = 0; |
| // Find the first non-undef value in this group. |
| for (; B != 8; ++B) { |
| if (!Values[I+B].isUndef()) |
| break; |
| } |
| SDValue F = Values[I+B]; |
| AllT &= IsTrue(F); |
| AllF &= IsFalse(F); |
| |
| SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8) |
| : DAG.getUNDEF(MVT::i8); |
| Bytes.push_back(Ext); |
| // Verify that the rest of values in the group are the same as the |
| // first. |
| for (; B != 8; ++B) |
| assert(Values[I+B].isUndef() || Values[I+B] == F); |
| } |
| } |
| |
| if (AllT) |
| return DAG.getNode(HexagonISD::QTRUE, dl, VecTy); |
| if (AllF) |
| return DAG.getNode(HexagonISD::QFALSE, dl, VecTy); |
| |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG); |
| return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); |
| } |
| |
| SDValue |
| HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV, |
| const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { |
| MVT ElemTy = ty(VecV).getVectorElementType(); |
| |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| assert(ElemWidth >= 8 && ElemWidth <= 32); |
| (void)ElemWidth; |
| |
| SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); |
| SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, |
| {VecV, ByteIdx}); |
| if (ElemTy == MVT::i32) |
| return ExWord; |
| |
| // Have an extracted word, need to extract the smaller element out of it. |
| // 1. Extract the bits of (the original) IdxV that correspond to the index |
| // of the desired element in the 32-bit word. |
| SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); |
| // 2. Extract the element from the word. |
| SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord); |
| return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV, |
| const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { |
| // Implement other return types if necessary. |
| assert(ResTy == MVT::i1); |
| |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); |
| |
| unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); |
| SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); |
| IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); |
| |
| SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG); |
| SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32); |
| return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV, |
| SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { |
| MVT ElemTy = ty(VecV).getVectorElementType(); |
| |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| assert(ElemWidth >= 8 && ElemWidth <= 32); |
| (void)ElemWidth; |
| |
| auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV, |
| SDValue ByteIdxV) { |
| MVT VecTy = ty(VecV); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| SDValue MaskV = |
| DAG.getNode(ISD::AND, dl, MVT::i32, |
| {ByteIdxV, DAG.getSignedConstant(-4, dl, MVT::i32)}); |
| SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV}); |
| SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV}); |
| SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32, |
| {DAG.getConstant(HwLen, dl, MVT::i32), MaskV}); |
| SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV}); |
| return TorV; |
| }; |
| |
| SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG); |
| if (ElemTy == MVT::i32) |
| return InsertWord(VecV, ValV, ByteIdx); |
| |
| // If this is not inserting a 32-bit word, convert it into such a thing. |
| // 1. Extract the existing word from the target vector. |
| SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32, |
| {ByteIdx, DAG.getConstant(2, dl, MVT::i32)}); |
| SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx, |
| dl, MVT::i32, DAG); |
| |
| // 2. Treating the extracted word as a 32-bit vector, insert the given |
| // value into it. |
| SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG); |
| MVT SubVecTy = tyVector(ty(Ext), ElemTy); |
| SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext), |
| ValV, SubIdx, dl, ElemTy, DAG); |
| |
| // 3. Insert the 32-bit word back into the original vector. |
| return InsertWord(VecV, Ins, ByteIdx); |
| } |
| |
| SDValue |
| HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV, |
| SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const { |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); |
| |
| unsigned Scale = HwLen / ty(VecV).getVectorNumElements(); |
| SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32); |
| IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV); |
| ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV); |
| |
| SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG); |
| return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV); |
| } |
| |
| SDValue |
| HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV, |
| SDValue IdxV, const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { |
| MVT VecTy = ty(VecV); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| unsigned Idx = IdxV.getNode()->getAsZExtVal(); |
| MVT ElemTy = VecTy.getVectorElementType(); |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| |
| // If the source vector is a vector pair, get the single vector containing |
| // the subvector of interest. The subvector will never overlap two single |
| // vectors. |
| if (isHvxPairTy(VecTy)) { |
| unsigned SubIdx = Hexagon::vsub_lo; |
| if (Idx * ElemWidth >= 8 * HwLen) { |
| SubIdx = Hexagon::vsub_hi; |
| Idx -= VecTy.getVectorNumElements() / 2; |
| } |
| |
| VecTy = typeSplit(VecTy).first; |
| VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV); |
| if (VecTy == ResTy) |
| return VecV; |
| } |
| |
| // The only meaningful subvectors of a single HVX vector are those that |
| // fit in a scalar register. |
| assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64); |
| |
| MVT WordTy = tyVector(VecTy, MVT::i32); |
| SDValue WordVec = DAG.getBitcast(WordTy, VecV); |
| unsigned WordIdx = (Idx*ElemWidth) / 32; |
| |
| SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32); |
| SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG); |
| if (ResTy.getSizeInBits() == 32) |
| return DAG.getBitcast(ResTy, W0); |
| |
| SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32); |
| SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG); |
| SDValue WW = getCombine(W1, W0, dl, MVT::i64, DAG); |
| return DAG.getBitcast(ResTy, WW); |
| } |
| |
| SDValue |
| HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV, |
| const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const { |
| MVT VecTy = ty(VecV); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); |
| // IdxV is required to be a constant. |
| unsigned Idx = IdxV.getNode()->getAsZExtVal(); |
| |
| unsigned ResLen = ResTy.getVectorNumElements(); |
| unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); |
| unsigned Offset = Idx * BitBytes; |
| SDValue Undef = DAG.getUNDEF(ByteTy); |
| SmallVector<int,128> Mask; |
| |
| if (Subtarget.isHVXVectorType(ResTy, true)) { |
| // Converting between two vector predicates. Since the result is shorter |
| // than the source, it will correspond to a vector predicate with the |
| // relevant bits replicated. The replication count is the ratio of the |
| // source and target vector lengths. |
| unsigned Rep = VecTy.getVectorNumElements() / ResLen; |
| assert(isPowerOf2_32(Rep) && HwLen % Rep == 0); |
| for (unsigned i = 0; i != HwLen/Rep; ++i) { |
| for (unsigned j = 0; j != Rep; ++j) |
| Mask.push_back(i + Offset); |
| } |
| SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); |
| return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV); |
| } |
| |
| // Converting between a vector predicate and a scalar predicate. In the |
| // vector predicate, a group of BitBytes bits will correspond to a single |
| // i1 element of the source vector type. Those bits will all have the same |
| // value. The same will be true for ByteVec, where each byte corresponds |
| // to a bit in the vector predicate. |
| // The algorithm is to traverse the ByteVec, going over the i1 values from |
| // the source vector, and generate the corresponding representation in an |
| // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the |
| // elements so that the interesting 8 bytes will be in the low end of the |
| // vector. |
| unsigned Rep = 8 / ResLen; |
| // Make sure the output fill the entire vector register, so repeat the |
| // 8-byte groups as many times as necessary. |
| for (unsigned r = 0; r != HwLen / 8; ++r) { |
| // This will generate the indexes of the 8 interesting bytes. |
| for (unsigned i = 0; i != ResLen; ++i) { |
| for (unsigned j = 0; j != Rep; ++j) |
| Mask.push_back(Offset + i*BitBytes); |
| } |
| } |
| |
| SDValue Zero = getZero(dl, MVT::i32, DAG); |
| SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask); |
| // Combine the two low words from ShuffV into a v8i8, and byte-compare |
| // them against 0. |
| SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero}); |
| SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, |
| {ShuffV, DAG.getConstant(4, dl, MVT::i32)}); |
| SDValue Vec64 = getCombine(W1, W0, dl, MVT::v8i8, DAG); |
| return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy, |
| {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV, |
| SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { |
| MVT VecTy = ty(VecV); |
| MVT SubTy = ty(SubV); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ElemTy = VecTy.getVectorElementType(); |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| |
| bool IsPair = isHvxPairTy(VecTy); |
| MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth); |
| // The two single vectors that VecV consists of, if it's a pair. |
| SDValue V0, V1; |
| SDValue SingleV = VecV; |
| SDValue PickHi; |
| |
| if (IsPair) { |
| V0 = LoHalf(VecV, DAG); |
| V1 = HiHalf(VecV, DAG); |
| |
| SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(), |
| dl, MVT::i32); |
| PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT); |
| if (isHvxSingleTy(SubTy)) { |
| if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) { |
| unsigned Idx = CN->getZExtValue(); |
| assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2); |
| unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi; |
| return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV); |
| } |
| // If IdxV is not a constant, generate the two variants: with the |
| // SubV as the high and as the low subregister, and select the right |
| // pair based on the IdxV. |
| SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1}); |
| SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV}); |
| return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); |
| } |
| // The subvector being inserted must be entirely contained in one of |
| // the vectors V0 or V1. Set SingleV to the correct one, and update |
| // IdxV to be the index relative to the beginning of that vector. |
| SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV); |
| IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV); |
| SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0); |
| } |
| |
| // The only meaningful subvectors of a single HVX vector are those that |
| // fit in a scalar register. |
| assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64); |
| // Convert IdxV to be index in bytes. |
| auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); |
| if (!IdxN || !IdxN->isZero()) { |
| IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, |
| DAG.getConstant(ElemWidth/8, dl, MVT::i32)); |
| SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV); |
| } |
| // When inserting a single word, the rotation back to the original position |
| // would be by HwLen-Idx, but if two words are inserted, it will need to be |
| // by (HwLen-4)-Idx. |
| unsigned RolBase = HwLen; |
| if (SubTy.getSizeInBits() == 32) { |
| SDValue V = DAG.getBitcast(MVT::i32, SubV); |
| SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, V); |
| } else { |
| SDValue V = DAG.getBitcast(MVT::i64, SubV); |
| SDValue R0 = LoHalf(V, DAG); |
| SDValue R1 = HiHalf(V, DAG); |
| SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0); |
| SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, |
| DAG.getConstant(4, dl, MVT::i32)); |
| SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1); |
| RolBase = HwLen-4; |
| } |
| // If the vector wasn't ror'ed, don't ror it back. |
| if (RolBase != 4 || !IdxN || !IdxN->isZero()) { |
| SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32, |
| DAG.getConstant(RolBase, dl, MVT::i32), IdxV); |
| SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV); |
| } |
| |
| if (IsPair) { |
| SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1}); |
| SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV}); |
| return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo); |
| } |
| return SingleV; |
| } |
| |
| SDValue |
| HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV, |
| SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const { |
| MVT VecTy = ty(VecV); |
| MVT SubTy = ty(SubV); |
| assert(Subtarget.isHVXVectorType(VecTy, true)); |
| // VecV is an HVX vector predicate. SubV may be either an HVX vector |
| // predicate as well, or it can be a scalar predicate. |
| |
| unsigned VecLen = VecTy.getVectorNumElements(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| assert(HwLen % VecLen == 0 && "Unexpected vector type"); |
| |
| unsigned Scale = VecLen / SubTy.getVectorNumElements(); |
| unsigned BitBytes = HwLen / VecLen; |
| unsigned BlockLen = HwLen / Scale; |
| |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV); |
| SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG); |
| SDValue ByteIdx; |
| |
| auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode()); |
| if (!IdxN || !IdxN->isZero()) { |
| ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, |
| DAG.getConstant(BitBytes, dl, MVT::i32)); |
| ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx); |
| } |
| |
| // ByteVec is the target vector VecV rotated in such a way that the |
| // subvector should be inserted at index 0. Generate a predicate mask |
| // and use vmux to do the insertion. |
| assert(BlockLen < HwLen && "vsetq(v1) prerequisite"); |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); |
| SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, |
| {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG); |
| ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG); |
| // Rotate ByteVec back, and convert to a vector predicate. |
| if (!IdxN || !IdxN->isZero()) { |
| SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32); |
| SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx); |
| ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi); |
| } |
| return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec); |
| } |
| |
| SDValue |
| HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl, |
| MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const { |
| // Sign- and any-extending of a vector predicate to a vector register is |
| // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and |
| // a vector of 1s (where the 1s are of type matching the vector type). |
| assert(Subtarget.isHVXVectorType(ResTy)); |
| if (!ZeroExt) |
| return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV); |
| |
| assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements()); |
| SDValue True = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, |
| DAG.getConstant(1, dl, MVT::i32)); |
| SDValue False = getZero(dl, ResTy, DAG); |
| return DAG.getSelect(dl, ResTy, VecV, True, False); |
| } |
| |
| SDValue |
| HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl, |
| MVT ResTy, SelectionDAG &DAG) const { |
| // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1] |
| // (i.e. the entire predicate register) to bits [0..HwLen-1] of a |
| // vector register. The remaining bits of the vector register are |
| // unspecified. |
| |
| MachineFunction &MF = DAG.getMachineFunction(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| MVT PredTy = ty(VecQ); |
| unsigned PredLen = PredTy.getVectorNumElements(); |
| assert(HwLen % PredLen == 0); |
| MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen); |
| |
| Type *Int8Ty = Type::getInt8Ty(*DAG.getContext()); |
| SmallVector<Constant*, 128> Tmp; |
| // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,... |
| // These are bytes with the LSB rotated left with respect to their index. |
| for (unsigned i = 0; i != HwLen/8; ++i) { |
| for (unsigned j = 0; j != 8; ++j) |
| Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j)); |
| } |
| Constant *CV = ConstantVector::get(Tmp); |
| Align Alignment(HwLen); |
| SDValue CP = LowerConstantPool( |
| DAG.getConstantPool(CV, getPointerTy(DAG.getDataLayout()), Alignment), |
| DAG); |
| SDValue Bytes = |
| DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP, |
| MachinePointerInfo::getConstantPool(MF), Alignment); |
| |
| // Select the bytes that correspond to true bits in the vector predicate. |
| SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes), |
| getZero(dl, VecTy, DAG)); |
| // Calculate the OR of all bytes in each group of 8. That will compress |
| // all the individual bits into a single byte. |
| // First, OR groups of 4, via vrmpy with 0x01010101. |
| SDValue All1 = |
| DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32)); |
| SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG); |
| // Then rotate the accumulated vector by 4 bytes, and do the final OR. |
| SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy, |
| {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG); |
| SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot}); |
| |
| // Pick every 8th byte and coalesce them at the beginning of the output. |
| // For symmetry, coalesce every 1+8th byte after that, then every 2+8th |
| // byte and so on. |
| SmallVector<int,128> Mask; |
| for (unsigned i = 0; i != HwLen; ++i) |
| Mask.push_back((8*i) % HwLen + i/(HwLen/8)); |
| SDValue Collect = |
| DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask); |
| return DAG.getBitcast(ResTy, Collect); |
| } |
| |
| SDValue |
| HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed, |
| const SDLoc &dl, SelectionDAG &DAG) const { |
| // Take a vector and resize the element type to match the given type. |
| MVT InpTy = ty(VecV); |
| if (InpTy == ResTy) |
| return VecV; |
| |
| unsigned InpWidth = InpTy.getSizeInBits(); |
| unsigned ResWidth = ResTy.getSizeInBits(); |
| |
| if (InpTy.isFloatingPoint()) { |
| return InpWidth < ResWidth |
| ? DAG.getNode(ISD::FP_EXTEND, dl, ResTy, VecV) |
| : DAG.getNode(ISD::FP_ROUND, dl, ResTy, VecV, |
| DAG.getTargetConstant(0, dl, MVT::i32)); |
| } |
| |
| assert(InpTy.isInteger()); |
| |
| if (InpWidth < ResWidth) { |
| unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
| return DAG.getNode(ExtOpc, dl, ResTy, VecV); |
| } else { |
| unsigned NarOpc = Signed ? HexagonISD::SSAT : HexagonISD::USAT; |
| return DAG.getNode(NarOpc, dl, ResTy, VecV, DAG.getValueType(ResTy)); |
| } |
| } |
| |
| SDValue |
| HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx, |
| SelectionDAG &DAG) const { |
| assert(ty(Vec).getSizeInBits() % SubTy.getSizeInBits() == 0); |
| |
| const SDLoc &dl(Vec); |
| unsigned ElemIdx = SubIdx * SubTy.getVectorNumElements(); |
| return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubTy, |
| {Vec, DAG.getConstant(ElemIdx, dl, MVT::i32)}); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG) |
| const { |
| const SDLoc &dl(Op); |
| MVT VecTy = ty(Op); |
| |
| unsigned Size = Op.getNumOperands(); |
| SmallVector<SDValue,128> Ops; |
| for (unsigned i = 0; i != Size; ++i) |
| Ops.push_back(Op.getOperand(i)); |
| |
| if (VecTy.getVectorElementType() == MVT::i1) |
| return buildHvxVectorPred(Ops, dl, VecTy, DAG); |
| |
| // In case of MVT::f16 BUILD_VECTOR, since MVT::f16 is |
| // not a legal type, just bitcast the node to use i16 |
| // types and bitcast the result back to f16 |
| if (VecTy.getVectorElementType() == MVT::f16 || |
| VecTy.getVectorElementType() == MVT::bf16) { |
| SmallVector<SDValue, 64> NewOps; |
| for (unsigned i = 0; i != Size; i++) |
| NewOps.push_back(DAG.getBitcast(MVT::i16, Ops[i])); |
| |
| SDValue T0 = |
| DAG.getNode(ISD::BUILD_VECTOR, dl, tyVector(VecTy, MVT::i16), NewOps); |
| return DAG.getBitcast(tyVector(VecTy, VecTy.getVectorElementType()), T0); |
| } |
| |
| // First, split the BUILD_VECTOR for vector pairs. We could generate |
| // some pairs directly (via splat), but splats should be generated |
| // by the combiner prior to getting here. |
| if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) { |
| ArrayRef<SDValue> A(Ops); |
| MVT SingleTy = typeSplit(VecTy).first; |
| SDValue V0 = buildHvxVectorReg(A.take_front(Size / 2), dl, SingleTy, DAG); |
| SDValue V1 = buildHvxVectorReg(A.drop_front(Size / 2), dl, SingleTy, DAG); |
| return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1); |
| } |
| |
| return buildHvxVectorReg(Ops, dl, VecTy, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxSplatVector(SDValue Op, SelectionDAG &DAG) |
| const { |
| const SDLoc &dl(Op); |
| MVT VecTy = ty(Op); |
| MVT ArgTy = ty(Op.getOperand(0)); |
| |
| if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) { |
| MVT SplatTy = MVT::getVectorVT(MVT::i16, VecTy.getVectorNumElements()); |
| SDValue ToInt16 = DAG.getBitcast(MVT::i16, Op.getOperand(0)); |
| SDValue ToInt32 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, ToInt16); |
| SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, dl, SplatTy, ToInt32); |
| return DAG.getBitcast(VecTy, Splat); |
| } |
| |
| return SDValue(); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG) |
| const { |
| // Vector concatenation of two integer (non-bool) vectors does not need |
| // special lowering. Custom-lower concats of bool vectors and expand |
| // concats of more than 2 vectors. |
| MVT VecTy = ty(Op); |
| const SDLoc &dl(Op); |
| unsigned NumOp = Op.getNumOperands(); |
| if (VecTy.getVectorElementType() != MVT::i1) { |
| if (NumOp == 2) |
| return Op; |
| // Expand the other cases into a build-vector. |
| SmallVector<SDValue,8> Elems; |
| for (SDValue V : Op.getNode()->ops()) |
| DAG.ExtractVectorElements(V, Elems); |
| // A vector of i16 will be broken up into a build_vector of i16's. |
| // This is a problem, since at the time of operation legalization, |
| // all operations are expected to be type-legalized, and i16 is not |
| // a legal type. If any of the extracted elements is not of a valid |
| // type, sign-extend it to a valid one. |
| for (SDValue &V : Elems) { |
| MVT Ty = ty(V); |
| if (!isTypeLegal(Ty)) { |
| MVT NTy = typeLegalize(Ty, DAG); |
| if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { |
| V = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy, |
| DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy, |
| V.getOperand(0), V.getOperand(1)), |
| DAG.getValueType(Ty)); |
| continue; |
| } |
| // A few less complicated cases. |
| switch (V.getOpcode()) { |
| case ISD::Constant: |
| V = DAG.getSExtOrTrunc(V, dl, NTy); |
| break; |
| case ISD::UNDEF: |
| V = DAG.getUNDEF(NTy); |
| break; |
| case ISD::TRUNCATE: |
| V = V.getOperand(0); |
| break; |
| default: |
| llvm_unreachable("Unexpected vector element"); |
| } |
| } |
| } |
| return DAG.getBuildVector(VecTy, dl, Elems); |
| } |
| |
| assert(VecTy.getVectorElementType() == MVT::i1); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0); |
| |
| SDValue Op0 = Op.getOperand(0); |
| |
| // If the operands are HVX types (i.e. not scalar predicates), then |
| // defer the concatenation, and create QCAT instead. |
| if (Subtarget.isHVXVectorType(ty(Op0), true)) { |
| if (NumOp == 2) |
| return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1)); |
| |
| ArrayRef<SDUse> U(Op.getNode()->ops()); |
| SmallVector<SDValue, 4> SV(U); |
| ArrayRef<SDValue> Ops(SV); |
| |
| MVT HalfTy = typeSplit(VecTy).first; |
| SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, |
| Ops.take_front(NumOp/2)); |
| SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy, |
| Ops.take_back(NumOp/2)); |
| return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1); |
| } |
| |
| // Count how many bytes (in a vector register) each bit in VecTy |
| // corresponds to. |
| unsigned BitBytes = HwLen / VecTy.getVectorNumElements(); |
| |
| // Make sure that createHvxPrefixPred will only ever need to expand |
| // the predicate, i.e. bytes-per-bit in the input is not greater than |
| // the target bytes-per-bit in the result. |
| SDValue Combined = combineConcatOfScalarPreds(Op, BitBytes, DAG); |
| SmallVector<SDValue,8> Prefixes; |
| for (SDValue V : Combined.getNode()->op_values()) { |
| SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG); |
| Prefixes.push_back(P); |
| } |
| |
| unsigned InpLen = ty(Combined.getOperand(0)).getVectorNumElements(); |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue S = DAG.getConstant(HwLen - InpLen*BitBytes, dl, MVT::i32); |
| SDValue Res = getZero(dl, ByteTy, DAG); |
| for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) { |
| Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S); |
| Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]); |
| } |
| return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG) |
| const { |
| // Change the type of the extracted element to i32. |
| SDValue VecV = Op.getOperand(0); |
| MVT ElemTy = ty(VecV).getVectorElementType(); |
| const SDLoc &dl(Op); |
| SDValue IdxV = Op.getOperand(1); |
| if (ElemTy == MVT::i1) |
| return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG); |
| |
| return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG) |
| const { |
| const SDLoc &dl(Op); |
| SDValue VecV = Op.getOperand(0); |
| SDValue ValV = Op.getOperand(1); |
| SDValue IdxV = Op.getOperand(2); |
| MVT ElemTy = ty(VecV).getVectorElementType(); |
| if (ElemTy == MVT::i1) |
| return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG); |
| |
| return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) |
| const { |
| SDValue SrcV = Op.getOperand(0); |
| MVT SrcTy = ty(SrcV); |
| MVT DstTy = ty(Op); |
| SDValue IdxV = Op.getOperand(1); |
| unsigned Idx = IdxV.getNode()->getAsZExtVal(); |
| assert(Idx % DstTy.getVectorNumElements() == 0); |
| (void)Idx; |
| const SDLoc &dl(Op); |
| |
| MVT ElemTy = SrcTy.getVectorElementType(); |
| if (ElemTy == MVT::i1) |
| return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG); |
| |
| return extractHvxSubvectorReg(Op, SrcV, IdxV, dl, DstTy, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) |
| const { |
| // Idx does not need to be a constant. |
| SDValue VecV = Op.getOperand(0); |
| SDValue ValV = Op.getOperand(1); |
| SDValue IdxV = Op.getOperand(2); |
| |
| const SDLoc &dl(Op); |
| MVT VecTy = ty(VecV); |
| MVT ElemTy = VecTy.getVectorElementType(); |
| if (ElemTy == MVT::i1) |
| return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG); |
| |
| return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const { |
| // Lower any-extends of boolean vectors to sign-extends, since they |
| // translate directly to Q2V. Zero-extending could also be done equally |
| // fast, but Q2V is used/recognized in more places. |
| // For all other vectors, use zero-extend. |
| MVT ResTy = ty(Op); |
| SDValue InpV = Op.getOperand(0); |
| MVT ElemTy = ty(InpV).getVectorElementType(); |
| if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) |
| return LowerHvxSignExt(Op, DAG); |
| return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const { |
| MVT ResTy = ty(Op); |
| SDValue InpV = Op.getOperand(0); |
| MVT ElemTy = ty(InpV).getVectorElementType(); |
| if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) |
| return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG); |
| return Op; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const { |
| MVT ResTy = ty(Op); |
| SDValue InpV = Op.getOperand(0); |
| MVT ElemTy = ty(InpV).getVectorElementType(); |
| if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy)) |
| return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG); |
| return Op; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const { |
| // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight): |
| // cttz(x) = bitwidth(x) - ctlz(~x & (x-1)) |
| const SDLoc &dl(Op); |
| MVT ResTy = ty(Op); |
| SDValue InpV = Op.getOperand(0); |
| assert(ResTy == ty(InpV)); |
| |
| // Calculate the vectors of 1 and bitwidth(x). |
| MVT ElemTy = ty(InpV).getVectorElementType(); |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| |
| SDValue Vec1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, |
| DAG.getConstant(1, dl, MVT::i32)); |
| SDValue VecW = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, |
| DAG.getConstant(ElemWidth, dl, MVT::i32)); |
| SDValue VecN1 = DAG.getNode(ISD::SPLAT_VECTOR, dl, ResTy, |
| DAG.getAllOnesConstant(dl, MVT::i32)); |
| |
| // Do not use DAG.getNOT, because that would create BUILD_VECTOR with |
| // a BITCAST. Here we can skip the BITCAST (so we don't have to handle |
| // it separately in custom combine or selection). |
| SDValue A = DAG.getNode(ISD::AND, dl, ResTy, |
| {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}), |
| DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})}); |
| return DAG.getNode(ISD::SUB, dl, ResTy, |
| {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)}); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| MVT ResTy = ty(Op); |
| assert(ResTy.getVectorElementType() == MVT::i32); |
| |
| SDValue Vs = Op.getOperand(0); |
| SDValue Vt = Op.getOperand(1); |
| |
| SDVTList ResTys = DAG.getVTList(ResTy, ResTy); |
| unsigned Opc = Op.getOpcode(); |
| |
| // On HVX v62+ producing the full product is cheap, so legalize MULH to LOHI. |
| if (Opc == ISD::MULHU) |
| return DAG.getNode(HexagonISD::UMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1); |
| if (Opc == ISD::MULHS) |
| return DAG.getNode(HexagonISD::SMUL_LOHI, dl, ResTys, {Vs, Vt}).getValue(1); |
| |
| #ifndef NDEBUG |
| Op.dump(&DAG); |
| #endif |
| llvm_unreachable("Unexpected mulh operation"); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxMulLoHi(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| unsigned Opc = Op.getOpcode(); |
| SDValue Vu = Op.getOperand(0); |
| SDValue Vv = Op.getOperand(1); |
| |
| // If the HI part is not used, convert it to a regular MUL. |
| if (auto HiVal = Op.getValue(1); HiVal.use_empty()) { |
| // Need to preserve the types and the number of values. |
| SDValue Hi = DAG.getUNDEF(ty(HiVal)); |
| SDValue Lo = DAG.getNode(ISD::MUL, dl, ty(Op), {Vu, Vv}); |
| return DAG.getMergeValues({Lo, Hi}, dl); |
| } |
| |
| bool SignedVu = Opc == HexagonISD::SMUL_LOHI; |
| bool SignedVv = Opc == HexagonISD::SMUL_LOHI || Opc == HexagonISD::USMUL_LOHI; |
| |
| // Legal on HVX v62+, but lower it here because patterns can't handle multi- |
| // valued nodes. |
| if (Subtarget.useHVXV62Ops()) |
| return emitHvxMulLoHiV62(Vu, SignedVu, Vv, SignedVv, dl, DAG); |
| |
| if (Opc == HexagonISD::SMUL_LOHI) { |
| // Direct MULHS expansion is cheaper than doing the whole SMUL_LOHI, |
| // for other signedness LOHI is cheaper. |
| if (auto LoVal = Op.getValue(0); LoVal.use_empty()) { |
| SDValue Hi = emitHvxMulHsV60(Vu, Vv, dl, DAG); |
| SDValue Lo = DAG.getUNDEF(ty(LoVal)); |
| return DAG.getMergeValues({Lo, Hi}, dl); |
| } |
| } |
| |
| return emitHvxMulLoHiV60(Vu, SignedVu, Vv, SignedVv, dl, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const { |
| SDValue Val = Op.getOperand(0); |
| MVT ResTy = ty(Op); |
| MVT ValTy = ty(Val); |
| const SDLoc &dl(Op); |
| |
| if (isHvxBoolTy(ValTy) && ResTy.isScalarInteger()) { |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4); |
| |
| // When the predicate is shorter than the predicate register, each boolean |
| // is represented by multiple consecutive bits in the input register. |
| // Condense the bits so each boolean is represented by one bit. This only |
| // handles 2x and 4x compaction ratios. |
| unsigned PredLen = ValTy.getVectorNumElements(); |
| if (PredLen < HwLen) { |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| Val = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Val); |
| if (HwLen > PredLen * 2) { |
| assert(HwLen == PredLen * 4); |
| PredLen *= 2; |
| Val = getInstr(Hexagon::V6_vdealh, dl, ByteTy, Val, DAG); |
| } |
| if (HwLen > PredLen) { |
| assert(HwLen == PredLen * 2); |
| Val = getInstr(Hexagon::V6_vdealb, dl, ByteTy, Val, DAG); |
| } |
| Val = DAG.getNode(HexagonISD::V2Q, dl, ValTy, Val); |
| } |
| |
| SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG); |
| unsigned BitWidth = ResTy.getSizeInBits(); |
| |
| if (BitWidth < 64) { |
| SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32), |
| dl, MVT::i32, DAG); |
| if (BitWidth == 32) |
| return W0; |
| assert(BitWidth < 32u); |
| return DAG.getZExtOrTrunc(W0, dl, ResTy); |
| } |
| |
| // The result is >= 64 bits. The only options are 64 or 128. |
| assert(BitWidth == 64 || BitWidth == 128); |
| SmallVector<SDValue,4> Words; |
| for (unsigned i = 0; i != BitWidth/32; ++i) { |
| SDValue W = extractHvxElementReg( |
| VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG); |
| Words.push_back(W); |
| } |
| SmallVector<SDValue,2> Combines; |
| assert(Words.size() % 2 == 0); |
| for (unsigned i = 0, e = Words.size(); i < e; i += 2) { |
| SDValue C = getCombine(Words[i+1], Words[i], dl, MVT::i64, DAG); |
| Combines.push_back(C); |
| } |
| |
| if (BitWidth == 64) |
| return Combines[0]; |
| |
| return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines); |
| } |
| |
| // Handle bitcast from i32, v2i16, and v4i8 to v32i1. |
| // Splat the input into a 32-element i32 vector, then AND each element |
| // with a unique bitmask to isolate individual bits. |
| auto bitcastI32ToV32I1 = [&](SDValue Val32) { |
| assert(Val32.getValueType().getSizeInBits() == 32 && |
| "Input must be 32 bits"); |
| MVT VecTy = MVT::getVectorVT(MVT::i32, 32); |
| SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, dl, VecTy, Val32); |
| SmallVector<SDValue, 32> Mask; |
| for (unsigned i = 0; i < 32; ++i) |
| Mask.push_back(DAG.getConstant(1ull << i, dl, MVT::i32)); |
| |
| SDValue MaskVec = DAG.getBuildVector(VecTy, dl, Mask); |
| SDValue Anded = DAG.getNode(ISD::AND, dl, VecTy, Splat, MaskVec); |
| return DAG.getNode(HexagonISD::V2Q, dl, MVT::v32i1, Anded); |
| }; |
| // === Case: v32i1 === |
| if (ResTy == MVT::v32i1 && |
| (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) && |
| Subtarget.useHVX128BOps()) { |
| SDValue Val32 = Val; |
| if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8) |
| Val32 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Val); |
| return bitcastI32ToV32I1(Val32); |
| } |
| // === Case: v64i1 === |
| if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) { |
| // Split i64 into lo/hi 32-bit halves. |
| SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Val); |
| SDValue HiShifted = DAG.getNode(ISD::SRL, dl, MVT::i64, Val, |
| DAG.getConstant(32, dl, MVT::i64)); |
| SDValue Hi = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, HiShifted); |
| |
| // Reuse the same 32-bit logic twice. |
| SDValue LoRes = bitcastI32ToV32I1(Lo); |
| SDValue HiRes = bitcastI32ToV32I1(Hi); |
| |
| // Concatenate into a v64i1 predicate. |
| return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, LoRes, HiRes); |
| } |
| |
| if (isHvxBoolTy(ResTy) && ValTy.isScalarInteger()) { |
| // Handle bitcast from i128 -> v128i1 and i64 -> v64i1. |
| unsigned BitWidth = ValTy.getSizeInBits(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| assert(BitWidth == HwLen); |
| |
| MVT ValAsVecTy = MVT::getVectorVT(MVT::i8, BitWidth / 8); |
| SDValue ValAsVec = DAG.getBitcast(ValAsVecTy, Val); |
| // Splat each byte of Val 8 times. |
| // Bytes = [(b0)x8, (b1)x8, ...., (b15)x8] |
| // where b0, b1,..., b15 are least to most significant bytes of I. |
| SmallVector<SDValue, 128> Bytes; |
| // Tmp: 0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80, 0x01,0x02,0x04,0x08,... |
| // These are bytes with the LSB rotated left with respect to their index. |
| SmallVector<SDValue, 128> Tmp; |
| for (unsigned I = 0; I != HwLen / 8; ++I) { |
| SDValue Idx = DAG.getConstant(I, dl, MVT::i32); |
| SDValue Byte = |
| DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i8, ValAsVec, Idx); |
| for (unsigned J = 0; J != 8; ++J) { |
| Bytes.push_back(Byte); |
| Tmp.push_back(DAG.getConstant(1ull << J, dl, MVT::i8)); |
| } |
| } |
| |
| MVT ConstantVecTy = MVT::getVectorVT(MVT::i8, HwLen); |
| SDValue ConstantVec = DAG.getBuildVector(ConstantVecTy, dl, Tmp); |
| SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG); |
| |
| // Each Byte in the I2V will be set iff corresponding bit is set in Val. |
| I2V = DAG.getNode(ISD::AND, dl, ConstantVecTy, {I2V, ConstantVec}); |
| return DAG.getNode(HexagonISD::V2Q, dl, ResTy, I2V); |
| } |
| |
| return Op; |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxStore(SDValue Op, |
| SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); |
| SDValue Val = SN->getValue(); |
| MVT ValTy = ty(Val); |
| |
| // Check if this is a store of an HVX bool vector (predicate) |
| if (!isHvxBoolTy(ValTy)) |
| return SDValue(); |
| |
| unsigned NumBits = ValTy.getVectorNumElements(); |
| MachineMemOperand *MMO = SN->getMemOperand(); |
| |
| // Check alignment requirements based on predicate size |
| unsigned RequiredAlign = (NumBits == 32) ? 4 : 8; |
| if (MMO->getBaseAlign().value() % RequiredAlign != 0) |
| return SDValue(); |
| |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen / 4); |
| |
| // Compress the predicate into a vector register |
| SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG); |
| |
| // Extract words from the compressed vector |
| SmallVector<SDValue, 4> Words; |
| for (unsigned i = 0; i != NumBits / 32; ++i) { |
| SDValue W = extractHvxElementReg(VQ, DAG.getConstant(i, dl, MVT::i32), dl, |
| MVT::i32, DAG); |
| Words.push_back(W); |
| } |
| |
| SDValue Chain = SN->getChain(); |
| SDValue BasePtr = SN->getBasePtr(); |
| MachinePointerInfo PtrInfo = MMO->getPointerInfo(); |
| |
| if (NumBits == 32) |
| return DAG.getStore(Chain, dl, Words[0], BasePtr, PtrInfo, |
| MMO->getBaseAlign()); |
| |
| if (NumBits == 64) { |
| SDValue W64 = getCombine(Words[1], Words[0], dl, MVT::i64, DAG); |
| return DAG.getStore(Chain, dl, W64, BasePtr, PtrInfo, MMO->getBaseAlign()); |
| } |
| |
| if (NumBits == 128) { |
| SDValue Lo64 = getCombine(Words[1], Words[0], dl, MVT::i64, DAG); |
| SDValue Hi64 = getCombine(Words[3], Words[2], dl, MVT::i64, DAG); |
| |
| Chain = |
| DAG.getStore(Chain, dl, Lo64, BasePtr, PtrInfo, MMO->getBaseAlign()); |
| |
| SDValue Offset8 = DAG.getConstant(8, dl, MVT::i32); |
| SDValue Ptr8 = DAG.getNode(ISD::ADD, dl, MVT::i32, BasePtr, Offset8); |
| return DAG.getStore(Chain, dl, Hi64, Ptr8, PtrInfo.getWithOffset(8), |
| Align(8)); |
| } |
| |
| return SDValue(); |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxLoad(SDValue Op, |
| SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); |
| MVT ResTy = ty(Op); |
| |
| // Check if this is a load of an HVX bool vector (predicate) |
| if (!isHvxBoolTy(ResTy)) |
| return SDValue(); |
| |
| unsigned NumBits = ResTy.getVectorNumElements(); |
| MachineMemOperand *MMO = LN->getMemOperand(); |
| |
| unsigned RequiredAlign = (NumBits == 32) ? 4 : 8; |
| if (MMO->getBaseAlign().value() % RequiredAlign != 0) |
| return SDValue(); |
| |
| SDValue Chain = LN->getChain(); |
| SDValue BasePtr = LN->getBasePtr(); |
| MachinePointerInfo PtrInfo = MMO->getPointerInfo(); |
| |
| if (NumBits == 32) { |
| SDValue W32 = |
| DAG.getLoad(MVT::i32, dl, Chain, BasePtr, PtrInfo, MMO->getBaseAlign()); |
| SDValue Pred = DAG.getNode(ISD::BITCAST, dl, MVT::v32i1, W32); |
| SDValue Ops[] = {Pred, W32.getValue(1)}; |
| return DAG.getMergeValues(Ops, dl); |
| } |
| |
| if (NumBits == 64) { |
| SDValue W64 = |
| DAG.getLoad(MVT::i64, dl, Chain, BasePtr, PtrInfo, MMO->getBaseAlign()); |
| SDValue Pred = DAG.getNode(ISD::BITCAST, dl, MVT::v64i1, W64); |
| SDValue Ops[] = {Pred, W64.getValue(1)}; |
| return DAG.getMergeValues(Ops, dl); |
| } |
| |
| if (NumBits == 128) { |
| SDValue Lo64 = |
| DAG.getLoad(MVT::i64, dl, Chain, BasePtr, PtrInfo, MMO->getBaseAlign()); |
| Chain = Lo64.getValue(1); |
| |
| SDValue Offset8 = DAG.getConstant(8, dl, MVT::i32); |
| SDValue Ptr8 = DAG.getNode(ISD::ADD, dl, MVT::i32, BasePtr, Offset8); |
| SDValue Hi64 = DAG.getLoad(MVT::i64, dl, Chain, Ptr8, |
| PtrInfo.getWithOffset(8), Align(8)); |
| |
| SDValue LoPred = DAG.getNode(ISD::BITCAST, dl, MVT::v64i1, Lo64); |
| SDValue HiPred = DAG.getNode(ISD::BITCAST, dl, MVT::v64i1, Hi64); |
| SDValue Pred = |
| DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v128i1, LoPred, HiPred); |
| |
| SDValue Ops[] = {Pred, Hi64.getValue(1)}; |
| return DAG.getMergeValues(Ops, dl); |
| } |
| |
| return SDValue(); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const { |
| // Sign- and zero-extends are legal. |
| assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG); |
| return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op), |
| Op.getOperand(0)); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxSelect(SDValue Op, SelectionDAG &DAG) const { |
| MVT ResTy = ty(Op); |
| if (ResTy.getVectorElementType() != MVT::i1) |
| return Op; |
| |
| const SDLoc &dl(Op); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| unsigned VecLen = ResTy.getVectorNumElements(); |
| assert(HwLen % VecLen == 0); |
| unsigned ElemSize = HwLen / VecLen; |
| |
| MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(ElemSize * 8), VecLen); |
| SDValue S = |
| DAG.getNode(ISD::SELECT, dl, VecTy, Op.getOperand(0), |
| DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(1)), |
| DAG.getNode(HexagonISD::Q2V, dl, VecTy, Op.getOperand(2))); |
| return DAG.getNode(HexagonISD::V2Q, dl, ResTy, S); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const { |
| if (SDValue S = getVectorShiftByInt(Op, DAG)) |
| return S; |
| return Op; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op, |
| SelectionDAG &DAG) const { |
| unsigned Opc = Op.getOpcode(); |
| assert(Opc == ISD::FSHL || Opc == ISD::FSHR); |
| |
| // Make sure the shift amount is within the range of the bitwidth |
| // of the element type. |
| SDValue A = Op.getOperand(0); |
| SDValue B = Op.getOperand(1); |
| SDValue S = Op.getOperand(2); |
| |
| MVT InpTy = ty(A); |
| MVT ElemTy = InpTy.getVectorElementType(); |
| |
| const SDLoc &dl(Op); |
| unsigned ElemWidth = ElemTy.getSizeInBits(); |
| bool IsLeft = Opc == ISD::FSHL; |
| |
| // The expansion into regular shifts produces worse code for i8 and for |
| // right shift of i32 on v65+. |
| bool UseShifts = ElemTy != MVT::i8; |
| if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32) |
| UseShifts = false; |
| |
| if (SDValue SplatV = getSplatValue(S, DAG); SplatV && UseShifts) { |
| // If this is a funnel shift by a scalar, lower it into regular shifts. |
| SDValue Mask = DAG.getConstant(ElemWidth - 1, dl, MVT::i32); |
| SDValue ModS = |
| DAG.getNode(ISD::AND, dl, MVT::i32, |
| {DAG.getZExtOrTrunc(SplatV, dl, MVT::i32), Mask}); |
| SDValue NegS = |
| DAG.getNode(ISD::SUB, dl, MVT::i32, |
| {DAG.getConstant(ElemWidth, dl, MVT::i32), ModS}); |
| SDValue IsZero = |
| DAG.getSetCC(dl, MVT::i1, ModS, getZero(dl, MVT::i32, DAG), ISD::SETEQ); |
| // FSHL A, B => A << | B >>n |
| // FSHR A, B => A <<n | B >> |
| SDValue Part1 = |
| DAG.getNode(HexagonISD::VASL, dl, InpTy, {A, IsLeft ? ModS : NegS}); |
| SDValue Part2 = |
| DAG.getNode(HexagonISD::VLSR, dl, InpTy, {B, IsLeft ? NegS : ModS}); |
| SDValue Or = DAG.getNode(ISD::OR, dl, InpTy, {Part1, Part2}); |
| // If the shift amount was 0, pick A or B, depending on the direction. |
| // The opposite shift will also be by 0, so the "Or" will be incorrect. |
| return DAG.getNode(ISD::SELECT, dl, InpTy, {IsZero, (IsLeft ? A : B), Or}); |
| } |
| |
| SDValue Mask = DAG.getSplatBuildVector( |
| InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy)); |
| |
| unsigned MOpc = Opc == ISD::FSHL ? HexagonISD::MFSHL : HexagonISD::MFSHR; |
| return DAG.getNode(MOpc, dl, ty(Op), |
| {A, B, DAG.getNode(ISD::AND, dl, InpTy, {S, Mask})}); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| unsigned IntNo = Op.getConstantOperandVal(0); |
| SmallVector<SDValue> Ops(Op->ops()); |
| |
| auto Swap = [&](SDValue P) { |
| return DAG.getMergeValues({P.getValue(1), P.getValue(0)}, dl); |
| }; |
| |
| switch (IntNo) { |
| case Intrinsic::hexagon_V6_pred_typecast: |
| case Intrinsic::hexagon_V6_pred_typecast_128B: { |
| MVT ResTy = ty(Op), InpTy = ty(Ops[1]); |
| if (isHvxBoolTy(ResTy) && isHvxBoolTy(InpTy)) { |
| if (ResTy == InpTy) |
| return Ops[1]; |
| return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Ops[1]); |
| } |
| break; |
| } |
| case Intrinsic::hexagon_V6_vmpyss_parts: |
| case Intrinsic::hexagon_V6_vmpyss_parts_128B: |
| return Swap(DAG.getNode(HexagonISD::SMUL_LOHI, dl, Op->getVTList(), |
| {Ops[1], Ops[2]})); |
| case Intrinsic::hexagon_V6_vmpyuu_parts: |
| case Intrinsic::hexagon_V6_vmpyuu_parts_128B: |
| return Swap(DAG.getNode(HexagonISD::UMUL_LOHI, dl, Op->getVTList(), |
| {Ops[1], Ops[2]})); |
| case Intrinsic::hexagon_V6_vmpyus_parts: |
| case Intrinsic::hexagon_V6_vmpyus_parts_128B: { |
| return Swap(DAG.getNode(HexagonISD::USMUL_LOHI, dl, Op->getVTList(), |
| {Ops[1], Ops[2]})); |
| } |
| } // switch |
| |
| return Op; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxMaskedOp(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto *MaskN = cast<MaskedLoadStoreSDNode>(Op.getNode()); |
| SDValue Mask = MaskN->getMask(); |
| SDValue Chain = MaskN->getChain(); |
| SDValue Base = MaskN->getBasePtr(); |
| auto *MemOp = MF.getMachineMemOperand(MaskN->getMemOperand(), 0, HwLen); |
| |
| unsigned Opc = Op->getOpcode(); |
| assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE); |
| |
| if (Opc == ISD::MLOAD) { |
| MVT ValTy = ty(Op); |
| SDValue Load = DAG.getLoad(ValTy, dl, Chain, Base, MemOp); |
| SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru(); |
| if (isUndef(Thru)) |
| return Load; |
| SDValue VSel = DAG.getNode(ISD::VSELECT, dl, ValTy, Mask, Load, Thru); |
| return DAG.getMergeValues({VSel, Load.getValue(1)}, dl); |
| } |
| |
| // MSTORE |
| // HVX only has aligned masked stores. |
| |
| // TODO: Fold negations of the mask into the store. |
| unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai; |
| SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue(); |
| SDValue Offset0 = DAG.getTargetConstant(0, dl, ty(Base)); |
| |
| if (MaskN->getAlign().value() % HwLen == 0) { |
| SDValue Store = getInstr(StoreOpc, dl, MVT::Other, |
| {Mask, Base, Offset0, Value, Chain}, DAG); |
| DAG.setNodeMemRefs(cast<MachineSDNode>(Store.getNode()), {MemOp}); |
| return Store; |
| } |
| |
| // Unaligned case. |
| auto StoreAlign = [&](SDValue V, SDValue A) { |
| SDValue Z = getZero(dl, ty(V), DAG); |
| // TODO: use funnel shifts? |
| // vlalign(Vu,Vv,Rt) rotates the pair Vu:Vv left by Rt and takes the |
| // upper half. |
| SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG); |
| SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG); |
| return std::make_pair(LoV, HiV); |
| }; |
| |
| MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen); |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); |
| SDValue MaskV = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Mask); |
| VectorPair Tmp = StoreAlign(MaskV, Base); |
| VectorPair MaskU = {DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.first), |
| DAG.getNode(HexagonISD::V2Q, dl, BoolTy, Tmp.second)}; |
| VectorPair ValueU = StoreAlign(Value, Base); |
| |
| SDValue Offset1 = DAG.getTargetConstant(HwLen, dl, MVT::i32); |
| SDValue StoreLo = |
| getInstr(StoreOpc, dl, MVT::Other, |
| {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG); |
| DAG.setNodeMemRefs(cast<MachineSDNode>(StoreLo.getNode()), {MemOp}); |
| |
| // If the store fits within one HwLen-aligned block, the high half's predicate |
| // is always all-zeros and the vmem(Base+HwLen) can be elided entirely. |
| // Proof: addr % StoreAlign == 0 and StoreMemSize <= StoreAlign implies |
| // addr % HwLen <= HwLen - StoreAlign, so addr % HwLen + StoreMemSize |
| // <= HwLen. |
| // Without this guard, Hexagon v73+ probes the TLB for vmem(Base+HwLen) even |
| // when the predicate is all-zeros, causing a TLBMISS if that page is |
| // unmapped. |
| uint64_t StoreMemSize = MaskN->getMemoryVT().getStoreSize().getFixedValue(); |
| if (StoreMemSize <= MaskN->getAlign().value()) |
| return StoreLo; |
| |
| SDValue StoreHi = |
| getInstr(StoreOpc, dl, MVT::Other, |
| {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG); |
| DAG.setNodeMemRefs(cast<MachineSDNode>(StoreHi.getNode()), {MemOp}); |
| return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, {StoreLo, StoreHi}); |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxFpExtend(SDValue Op, |
| SelectionDAG &DAG) const { |
| // This conversion only applies to QFloat. IEEE extension from f16 to f32 |
| // is legal (done via a pattern). |
| assert(Subtarget.useHVXQFloatOps()); |
| |
| assert(Op->getOpcode() == ISD::FP_EXTEND); |
| |
| MVT VecTy = ty(Op); |
| MVT ArgTy = ty(Op.getOperand(0)); |
| const SDLoc &dl(Op); |
| |
| if (ArgTy == MVT::v64bf16) { |
| MVT HalfTy = typeSplit(VecTy).first; |
| SDValue BF16Vec = Op.getOperand(0); |
| SDValue Zeroes = |
| getInstr(Hexagon::V6_vxor, dl, HalfTy, {BF16Vec, BF16Vec}, DAG); |
| // Interleave zero vector with the bf16 vector, with zeroes in the lower |
| // half of each 32 bit lane, effectively extending the bf16 values to fp32 |
| // values. |
| SDValue ShuffVec = |
| getInstr(Hexagon::V6_vshufoeh, dl, VecTy, {BF16Vec, Zeroes}, DAG); |
| VectorPair VecPair = opSplit(ShuffVec, dl, DAG); |
| SDValue Result = getInstr(Hexagon::V6_vshuffvdd, dl, VecTy, |
| {VecPair.second, VecPair.first, |
| DAG.getSignedConstant(-4, dl, MVT::i32)}, |
| DAG); |
| return Result; |
| } |
| |
| assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16); |
| |
| SDValue F16Vec = Op.getOperand(0); |
| |
| APFloat FloatVal = APFloat(1.0f); |
| bool Ignored; |
| FloatVal.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &Ignored); |
| SDValue Fp16Ones = DAG.getConstantFP(FloatVal, dl, ArgTy); |
| SDValue VmpyVec = |
| getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG); |
| |
| MVT HalfTy = typeSplit(VecTy).first; |
| VectorPair Pair = opSplit(VmpyVec, dl, DAG); |
| SDValue LoVec = |
| getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG); |
| SDValue HiVec = |
| getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG); |
| |
| SDValue ShuffVec = |
| getInstr(Hexagon::V6_vshuffvdd, dl, VecTy, |
| {HiVec, LoVec, DAG.getSignedConstant(-4, dl, MVT::i32)}, DAG); |
| |
| return ShuffVec; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxFpToInt(SDValue Op, SelectionDAG &DAG) const { |
| // Catch invalid conversion ops (just in case). |
| assert(Op.getOpcode() == ISD::FP_TO_SINT || |
| Op.getOpcode() == ISD::FP_TO_UINT); |
| |
| MVT ResTy = ty(Op); |
| MVT FpTy = ty(Op.getOperand(0)).getVectorElementType(); |
| MVT IntTy = ResTy.getVectorElementType(); |
| |
| if (Subtarget.useHVXIEEEFPOps()) { |
| // There are only conversions from f16. |
| if (FpTy == MVT::f16) { |
| // Other int types aren't legal in HVX, so we shouldn't see them here. |
| assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32); |
| // Conversions to i8 and i16 are legal. |
| if (IntTy == MVT::i8 || IntTy == MVT::i16) |
| return Op; |
| } |
| } |
| |
| if (IntTy.getSizeInBits() != FpTy.getSizeInBits()) |
| return EqualizeFpIntConversion(Op, DAG); |
| |
| return ExpandHvxFpToInt(Op, DAG); |
| } |
| |
| // For vector type v32i1 uint_to_fp/sint_to_fp to v32f32: |
| // R1 = #1, R2 holds the v32i1 param |
| // V1 = vsplat(R1) |
| // V2 = vsplat(R2) |
| // Q0 = vand(V1,R1) |
| // V0.w=prefixsum(Q0) |
| // V0.w=vsub(V0.w,V1.w) |
| // V2.w = vlsr(V2.w,V0.w) |
| // V2 = vand(V2,V1) |
| // V2.sf = V2.w |
| SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp, |
| SelectionDAG &DAG) const { |
| |
| MVT ResTy = ty(PredOp); |
| const SDLoc &dl(PredOp); |
| |
| SDValue Const = DAG.getTargetConstant(0x1, dl, MVT::i32); |
| SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const); |
| SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32, |
| SDValue(RegConst, 0)); |
| SDNode *PredTransfer = |
| DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1, |
| SDValue(SplatConst, 0), SDValue(RegConst, 0)); |
| SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32, |
| SDValue(PredTransfer, 0)); |
| SDNode *SplatParam = DAG.getMachineNode( |
| Hexagon::V6_lvsplatw, dl, MVT::v32i32, |
| DAG.getNode(ISD::BITCAST, dl, MVT::i32, PredOp.getOperand(0))); |
| SDNode *Vsub = |
| DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32, |
| SDValue(PrefixSum, 0), SDValue(SplatConst, 0)); |
| SDNode *IndexShift = |
| DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32, |
| SDValue(SplatParam, 0), SDValue(Vsub, 0)); |
| SDNode *MaskOff = |
| DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32, |
| SDValue(IndexShift, 0), SDValue(SplatConst, 0)); |
| SDNode *Convert = DAG.getMachineNode(Hexagon::V6_vconv_sf_w, dl, ResTy, |
| SDValue(MaskOff, 0)); |
| return SDValue(Convert, 0); |
| } |
| |
| // For vector type v64i1 uint_to_fo to v64f16: |
| // i64 R32 = bitcast v64i1 R3:2 (R3:2 holds v64i1) |
| // R3 = subreg_high (R32) |
| // R2 = subreg_low (R32) |
| // R1 = #1 |
| // V1 = vsplat(R1) |
| // V2 = vsplat(R2) |
| // V3 = vsplat(R3) |
| // Q0 = vand(V1,R1) |
| // V0.w=prefixsum(Q0) |
| // V0.w=vsub(V0.w,V1.w) |
| // V2.w = vlsr(V2.w,V0.w) |
| // V3.w = vlsr(V3.w,V0.w) |
| // V2 = vand(V2,V1) |
| // V3 = vand(V3,V1) |
| // V2.h = vpacke(V3.w,V2.w) |
| // V2.hf = V2.h |
| SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp, |
| SelectionDAG &DAG) const { |
| |
| MVT ResTy = ty(PredOp); |
| const SDLoc &dl(PredOp); |
| |
| SDValue Inp = DAG.getNode(ISD::BITCAST, dl, MVT::i64, PredOp.getOperand(0)); |
| // Get the hi and lo regs |
| SDValue HiReg = |
| DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, Inp); |
| SDValue LoReg = |
| DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, Inp); |
| // Get constant #1 and splat into vector V1 |
| SDValue Const = DAG.getTargetConstant(0x1, dl, MVT::i32); |
| SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const); |
| SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32, |
| SDValue(RegConst, 0)); |
| // Splat the hi and lo args |
| SDNode *SplatHi = |
| DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32, |
| DAG.getNode(ISD::BITCAST, dl, MVT::i32, HiReg)); |
| SDNode *SplatLo = |
| DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32, |
| DAG.getNode(ISD::BITCAST, dl, MVT::i32, LoReg)); |
| // vand between splatted const and const |
| SDNode *PredTransfer = |
| DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1, |
| SDValue(SplatConst, 0), SDValue(RegConst, 0)); |
| // Get the prefixsum |
| SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32, |
| SDValue(PredTransfer, 0)); |
| // Get the vsub |
| SDNode *Vsub = |
| DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32, |
| SDValue(PrefixSum, 0), SDValue(SplatConst, 0)); |
| // Get vlsr for hi and lo |
| SDNode *IndexShift_hi = |
| DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32, |
| SDValue(SplatHi, 0), SDValue(Vsub, 0)); |
| SDNode *IndexShift_lo = |
| DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32, |
| SDValue(SplatLo, 0), SDValue(Vsub, 0)); |
| // Get vand of hi and lo |
| SDNode *MaskOff_hi = |
| DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32, |
| SDValue(IndexShift_hi, 0), SDValue(SplatConst, 0)); |
| SDNode *MaskOff_lo = |
| DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32, |
| SDValue(IndexShift_lo, 0), SDValue(SplatConst, 0)); |
| // Pack them |
| SDNode *Pack = |
| DAG.getMachineNode(Hexagon::V6_vpackeh, dl, MVT::v64i16, |
| SDValue(MaskOff_hi, 0), SDValue(MaskOff_lo, 0)); |
| SDNode *Convert = |
| DAG.getMachineNode(Hexagon::V6_vconv_hf_h, dl, ResTy, SDValue(Pack, 0)); |
| return SDValue(Convert, 0); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const { |
| // Catch invalid conversion ops (just in case). |
| assert(Op.getOpcode() == ISD::SINT_TO_FP || |
| Op.getOpcode() == ISD::UINT_TO_FP); |
| |
| MVT ResTy = ty(Op); |
| MVT IntTy = ty(Op.getOperand(0)).getVectorElementType(); |
| MVT FpTy = ResTy.getVectorElementType(); |
| |
| if (Op.getOpcode() == ISD::UINT_TO_FP || Op.getOpcode() == ISD::SINT_TO_FP) { |
| if (ResTy == MVT::v32f32 && ty(Op.getOperand(0)) == MVT::v32i1) |
| return LowerHvxPred32ToFp(Op, DAG); |
| if (ResTy == MVT::v64f16 && ty(Op.getOperand(0)) == MVT::v64i1) |
| return LowerHvxPred64ToFp(Op, DAG); |
| } |
| |
| if (Subtarget.useHVXIEEEFPOps()) { |
| // There are only conversions to f16. |
| if (FpTy == MVT::f16) { |
| // Other int types aren't legal in HVX, so we shouldn't see them here. |
| assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32); |
| // i8, i16 -> f16 is legal. |
| if (IntTy == MVT::i8 || IntTy == MVT::i16) |
| return Op; |
| } |
| } |
| |
| if (IntTy.getSizeInBits() != FpTy.getSizeInBits()) |
| return EqualizeFpIntConversion(Op, DAG); |
| |
| return ExpandHvxIntToFp(Op, DAG); |
| } |
| |
| HexagonTargetLowering::TypePair |
| HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const { |
| // Compare the widths of elements of the two types, and extend the narrower |
| // type to match the with of the wider type. For vector types, apply this |
| // to the element type. |
| assert(Ty0.isVector() == Ty1.isVector()); |
| |
| MVT ElemTy0 = Ty0.getScalarType(); |
| MVT ElemTy1 = Ty1.getScalarType(); |
| |
| unsigned Width0 = ElemTy0.getSizeInBits(); |
| unsigned Width1 = ElemTy1.getSizeInBits(); |
| unsigned MaxWidth = std::max(Width0, Width1); |
| |
| auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) { |
| if (ScalarTy.isInteger()) |
| return MVT::getIntegerVT(Width); |
| assert(ScalarTy.isFloatingPoint()); |
| return MVT::getFloatingPointVT(Width); |
| }; |
| |
| MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth); |
| MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth); |
| |
| if (!Ty0.isVector()) { |
| // Both types are scalars. |
| return {WideETy0, WideETy1}; |
| } |
| |
| // Vector types. |
| unsigned NumElem = Ty0.getVectorNumElements(); |
| assert(NumElem == Ty1.getVectorNumElements()); |
| |
| return {MVT::getVectorVT(WideETy0, NumElem), |
| MVT::getVectorVT(WideETy1, NumElem)}; |
| } |
| |
| HexagonTargetLowering::TypePair |
| HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const { |
| // Compare the numbers of elements of two vector types, and widen the |
| // narrower one to match the number of elements in the wider one. |
| assert(Ty0.isVector() && Ty1.isVector()); |
| |
| unsigned Len0 = Ty0.getVectorNumElements(); |
| unsigned Len1 = Ty1.getVectorNumElements(); |
| if (Len0 == Len1) |
| return {Ty0, Ty1}; |
| |
| unsigned MaxLen = std::max(Len0, Len1); |
| return {MVT::getVectorVT(Ty0.getVectorElementType(), MaxLen), |
| MVT::getVectorVT(Ty1.getVectorElementType(), MaxLen)}; |
| } |
| |
| MVT |
| HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const { |
| EVT LegalTy = getTypeToTransformTo(*DAG.getContext(), Ty); |
| assert(LegalTy.isSimple()); |
| return LegalTy.getSimpleVT(); |
| } |
| |
| MVT |
| HexagonTargetLowering::typeWidenToHvx(MVT Ty) const { |
| unsigned HwWidth = 8 * Subtarget.getVectorLength(); |
| assert(Ty.getSizeInBits() <= HwWidth); |
| if (Ty.getSizeInBits() == HwWidth) |
| return Ty; |
| |
| MVT ElemTy = Ty.getScalarType(); |
| return MVT::getVectorVT(ElemTy, HwWidth / ElemTy.getSizeInBits()); |
| } |
| |
| HexagonTargetLowering::VectorPair |
| HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B, |
| const SDLoc &dl, bool Signed, SelectionDAG &DAG) const { |
| // Compute A+B, return {A+B, O}, where O = vector predicate indicating |
| // whether an overflow has occurred. |
| MVT ResTy = ty(A); |
| assert(ResTy == ty(B)); |
| MVT PredTy = MVT::getVectorVT(MVT::i1, ResTy.getVectorNumElements()); |
| |
| if (!Signed) { |
| // V62+ has V6_vaddcarry, but it requires input predicate, so it doesn't |
| // save any instructions. |
| SDValue Add = DAG.getNode(ISD::ADD, dl, ResTy, {A, B}); |
| SDValue Ovf = DAG.getSetCC(dl, PredTy, Add, A, ISD::SETULT); |
| return {Add, Ovf}; |
| } |
| |
| // Signed overflow has happened, if: |
| // (A, B have the same sign) and (A+B has a different sign from either) |
| // i.e. (~A xor B) & ((A+B) xor B), then check the sign bit |
| SDValue Add = DAG.getNode(ISD::ADD, dl, ResTy, {A, B}); |
| SDValue NotA = |
| DAG.getNode(ISD::XOR, dl, ResTy, {A, DAG.getAllOnesConstant(dl, ResTy)}); |
| SDValue Xor0 = DAG.getNode(ISD::XOR, dl, ResTy, {NotA, B}); |
| SDValue Xor1 = DAG.getNode(ISD::XOR, dl, ResTy, {Add, B}); |
| SDValue And = DAG.getNode(ISD::AND, dl, ResTy, {Xor0, Xor1}); |
| SDValue MSB = |
| DAG.getSetCC(dl, PredTy, And, getZero(dl, ResTy, DAG), ISD::SETLT); |
| return {Add, MSB}; |
| } |
| |
| HexagonTargetLowering::VectorPair |
| HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt, |
| bool Signed, SelectionDAG &DAG) const { |
| // Shift Val right by Amt bits, round the result to the nearest integer, |
| // tie-break by rounding halves to even integer. |
| |
| const SDLoc &dl(Val); |
| MVT ValTy = ty(Val); |
| |
| // This should also work for signed integers. |
| // |
| // uint tmp0 = inp + ((1 << (Amt-1)) - 1); |
| // bool ovf = (inp > tmp0); |
| // uint rup = inp & (1 << (Amt+1)); |
| // |
| // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff |
| // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0 |
| // uint tmp3 = tmp2 + rup; |
| // uint frac = (tmp1 != tmp2) ? tmp2 >> 1 : tmp3 >> 1; |
| unsigned ElemWidth = ValTy.getVectorElementType().getSizeInBits(); |
| MVT ElemTy = MVT::getIntegerVT(ElemWidth); |
| MVT IntTy = tyVector(ValTy, ElemTy); |
| MVT PredTy = MVT::getVectorVT(MVT::i1, IntTy.getVectorNumElements()); |
| unsigned ShRight = Signed ? ISD::SRA : ISD::SRL; |
| |
| SDValue Inp = DAG.getBitcast(IntTy, Val); |
| SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy); |
| |
| SDValue AmtP1 = DAG.getConstant(1ull << Amt, dl, IntTy); |
| SDValue And = DAG.getNode(ISD::AND, dl, IntTy, {Inp, AmtP1}); |
| SDValue Zero = getZero(dl, IntTy, DAG); |
| SDValue Bit = DAG.getSetCC(dl, PredTy, And, Zero, ISD::SETNE); |
| SDValue Rup = DAG.getZExtOrTrunc(Bit, dl, IntTy); |
| auto [Tmp0, Ovf] = emitHvxAddWithOverflow(Inp, LowBits, dl, Signed, DAG); |
| |
| SDValue AmtM1 = DAG.getConstant(Amt - 1, dl, IntTy); |
| SDValue Tmp1 = DAG.getNode(ShRight, dl, IntTy, Inp, AmtM1); |
| SDValue Tmp2 = DAG.getNode(ShRight, dl, IntTy, Tmp0, AmtM1); |
| SDValue Tmp3 = DAG.getNode(ISD::ADD, dl, IntTy, Tmp2, Rup); |
| |
| SDValue Eq = DAG.getSetCC(dl, PredTy, Tmp1, Tmp2, ISD::SETEQ); |
| SDValue One = DAG.getConstant(1, dl, IntTy); |
| SDValue Tmp4 = DAG.getNode(ShRight, dl, IntTy, {Tmp2, One}); |
| SDValue Tmp5 = DAG.getNode(ShRight, dl, IntTy, {Tmp3, One}); |
| SDValue Mux = DAG.getNode(ISD::VSELECT, dl, IntTy, {Eq, Tmp5, Tmp4}); |
| return {Mux, Ovf}; |
| } |
| |
| SDValue |
| HexagonTargetLowering::emitHvxMulHsV60(SDValue A, SDValue B, const SDLoc &dl, |
| SelectionDAG &DAG) const { |
| MVT VecTy = ty(A); |
| MVT PairTy = typeJoin({VecTy, VecTy}); |
| assert(VecTy.getVectorElementType() == MVT::i32); |
| |
| SDValue S16 = DAG.getConstant(16, dl, MVT::i32); |
| |
| // mulhs(A,B) = |
| // = [(Hi(A)*2^16 + Lo(A)) *s (Hi(B)*2^16 + Lo(B))] >> 32 |
| // = [Hi(A)*2^16 *s Hi(B)*2^16 + Hi(A) *su Lo(B)*2^16 |
| // + Lo(A) *us (Hi(B)*2^16 + Lo(B))] >> 32 |
| // = [Hi(A) *s Hi(B)*2^32 + Hi(A) *su Lo(B)*2^16 + Lo(A) *us B] >> 32 |
| // The low half of Lo(A)*Lo(B) will be discarded (it's not added to |
| // anything, so it cannot produce any carry over to higher bits), |
| // so everything in [] can be shifted by 16 without loss of precision. |
| // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + Lo(A)*B >> 16] >> 16 |
| // = [Hi(A) *s Hi(B)*2^16 + Hi(A)*su Lo(B) + V6_vmpyewuh(A,B)] >> 16 |
| // The final additions need to make sure to properly maintain any carry- |
| // out bits. |
| // |
| // Hi(B) Lo(B) |
| // Hi(A) Lo(A) |
| // -------------- |
| // Lo(B)*Lo(A) | T0 = V6_vmpyewuh(B,A) does this, |
| // Hi(B)*Lo(A) | + dropping the low 16 bits |
| // Hi(A)*Lo(B) | T2 |
| // Hi(B)*Hi(A) |
| |
| SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, VecTy, {B, A}, DAG); |
| // T1 = get Hi(A) into low halves. |
| SDValue T1 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {A, S16}, DAG); |
| // P0 = interleaved T1.h*B.uh (full precision product) |
| SDValue P0 = getInstr(Hexagon::V6_vmpyhus, dl, PairTy, {T1, B}, DAG); |
| // T2 = T1.even(h) * B.even(uh), i.e. Hi(A)*Lo(B) |
| SDValue T2 = LoHalf(P0, DAG); |
| // We need to add T0+T2, recording the carry-out, which will be 1<<16 |
| // added to the final sum. |
| // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves |
| SDValue P1 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, {T0, T2}, DAG); |
| // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves |
| SDValue P2 = getInstr(Hexagon::V6_vaddhw, dl, PairTy, {T0, T2}, DAG); |
| // T3 = full-precision(T0+T2) >> 16 |
| // The low halves are added-unsigned, the high ones are added-signed. |
| SDValue T3 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy, |
| {HiHalf(P2, DAG), LoHalf(P1, DAG), S16}, DAG); |
| SDValue T4 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {B, S16}, DAG); |
| // P3 = interleaved Hi(B)*Hi(A) (full precision), |
| // which is now Lo(T1)*Lo(T4), so we want to keep the even product. |
| SDValue P3 = getInstr(Hexagon::V6_vmpyhv, dl, PairTy, {T1, T4}, DAG); |
| SDValue T5 = LoHalf(P3, DAG); |
| // Add: |
| SDValue T6 = DAG.getNode(ISD::ADD, dl, VecTy, {T3, T5}); |
| return T6; |
| } |
| |
| SDValue |
| HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B, |
| bool SignedB, const SDLoc &dl, |
| SelectionDAG &DAG) const { |
| MVT VecTy = ty(A); |
| MVT PairTy = typeJoin({VecTy, VecTy}); |
| assert(VecTy.getVectorElementType() == MVT::i32); |
| |
| SDValue S16 = DAG.getConstant(16, dl, MVT::i32); |
| |
| if (SignedA && !SignedB) { |
| // Make A:unsigned, B:signed. |
| std::swap(A, B); |
| std::swap(SignedA, SignedB); |
| } |
| |
| // Do halfword-wise multiplications for unsigned*unsigned product, then |
| // add corrections for signed and unsigned*signed. |
| |
| SDValue Lo, Hi; |
| |
| // P0:lo = (uu) products of low halves of A and B, |
| // P0:hi = (uu) products of high halves. |
| SDValue P0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, B}, DAG); |
| |
| // Swap low/high halves in B |
| SDValue T0 = getInstr(Hexagon::V6_lvsplatw, dl, VecTy, |
| {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG); |
| SDValue T1 = getInstr(Hexagon::V6_vdelta, dl, VecTy, {B, T0}, DAG); |
| // P1 = products of even/odd halfwords. |
| // P1:lo = (uu) products of even(A.uh) * odd(B.uh) |
| // P1:hi = (uu) products of odd(A.uh) * even(B.uh) |
| SDValue P1 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, T1}, DAG); |
| |
| // P2:lo = low halves of P1:lo + P1:hi, |
| // P2:hi = high halves of P1:lo + P1:hi. |
| SDValue P2 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, |
| {HiHalf(P1, DAG), LoHalf(P1, DAG)}, DAG); |
| // Still need to add the high halves of P0:lo to P2:lo |
| SDValue T2 = |
| getInstr(Hexagon::V6_vlsrw, dl, VecTy, {LoHalf(P0, DAG), S16}, DAG); |
| SDValue T3 = DAG.getNode(ISD::ADD, dl, VecTy, {LoHalf(P2, DAG), T2}); |
| |
| // The high halves of T3 will contribute to the HI part of LOHI. |
| SDValue T4 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy, |
| {HiHalf(P2, DAG), T3, S16}, DAG); |
| |
| // The low halves of P2 need to be added to high halves of the LO part. |
| Lo = getInstr(Hexagon::V6_vaslw_acc, dl, VecTy, |
| {LoHalf(P0, DAG), LoHalf(P2, DAG), S16}, DAG); |
| Hi = DAG.getNode(ISD::ADD, dl, VecTy, {HiHalf(P0, DAG), T4}); |
| |
| if (SignedA) { |
| assert(SignedB && "Signed A and unsigned B should have been inverted"); |
| |
| MVT PredTy = MVT::getVectorVT(MVT::i1, VecTy.getVectorNumElements()); |
| SDValue Zero = getZero(dl, VecTy, DAG); |
| SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT); |
| SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT); |
| SDValue X0 = DAG.getNode(ISD::VSELECT, dl, VecTy, {Q0, B, Zero}); |
| SDValue X1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, X0, A}, DAG); |
| Hi = getInstr(Hexagon::V6_vsubw, dl, VecTy, {Hi, X1}, DAG); |
| } else if (SignedB) { |
| // Same correction as for mulhus: |
| // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0) |
| MVT PredTy = MVT::getVectorVT(MVT::i1, VecTy.getVectorNumElements()); |
| SDValue Zero = getZero(dl, VecTy, DAG); |
| SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT); |
| Hi = getInstr(Hexagon::V6_vsubwq, dl, VecTy, {Q1, Hi, A}, DAG); |
| } else { |
| assert(!SignedA && !SignedB); |
| } |
| |
| return DAG.getMergeValues({Lo, Hi}, dl); |
| } |
| |
| SDValue |
| HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA, |
| SDValue B, bool SignedB, |
| const SDLoc &dl, |
| SelectionDAG &DAG) const { |
| MVT VecTy = ty(A); |
| MVT PairTy = typeJoin({VecTy, VecTy}); |
| assert(VecTy.getVectorElementType() == MVT::i32); |
| |
| if (SignedA && !SignedB) { |
| // Make A:unsigned, B:signed. |
| std::swap(A, B); |
| std::swap(SignedA, SignedB); |
| } |
| |
| // Do S*S first, then make corrections for U*S or U*U if needed. |
| SDValue P0 = getInstr(Hexagon::V6_vmpyewuh_64, dl, PairTy, {A, B}, DAG); |
| SDValue P1 = |
| getInstr(Hexagon::V6_vmpyowh_64_acc, dl, PairTy, {P0, A, B}, DAG); |
| SDValue Lo = LoHalf(P1, DAG); |
| SDValue Hi = HiHalf(P1, DAG); |
| |
| if (!SignedB) { |
| assert(!SignedA && "Signed A and unsigned B should have been inverted"); |
| SDValue Zero = getZero(dl, VecTy, DAG); |
| MVT PredTy = MVT::getVectorVT(MVT::i1, VecTy.getVectorNumElements()); |
| |
| // Mulhu(X, Y) = Mulhs(X, Y) + (X, if Y < 0) + (Y, if X < 0). |
| // def: Pat<(VecI32 (mulhu HVI32:$A, HVI32:$B)), |
| // (V6_vaddw (HiHalf (Muls64O $A, $B)), |
| // (V6_vaddwq (V6_vgtw (V6_vd0), $B), |
| // (V6_vandvqv (V6_vgtw (V6_vd0), $A), $B), |
| // $A))>; |
| SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT); |
| SDValue Q1 = DAG.getSetCC(dl, PredTy, B, Zero, ISD::SETLT); |
| SDValue T0 = getInstr(Hexagon::V6_vandvqv, dl, VecTy, {Q0, B}, DAG); |
| SDValue T1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, T0, A}, DAG); |
| Hi = getInstr(Hexagon::V6_vaddw, dl, VecTy, {Hi, T1}, DAG); |
| } else if (!SignedA) { |
| SDValue Zero = getZero(dl, VecTy, DAG); |
| MVT PredTy = MVT::getVectorVT(MVT::i1, VecTy.getVectorNumElements()); |
| |
| // Mulhus(unsigned X, signed Y) = Mulhs(X, Y) + (Y, if X < 0). |
| // def: Pat<(VecI32 (HexagonMULHUS HVI32:$A, HVI32:$B)), |
| // (V6_vaddwq (V6_vgtw (V6_vd0), $A), |
| // (HiHalf (Muls64O $A, $B)), |
| // $B)>; |
| SDValue Q0 = DAG.getSetCC(dl, PredTy, A, Zero, ISD::SETLT); |
| Hi = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q0, Hi, B}, DAG); |
| } |
| |
| return DAG.getMergeValues({Lo, Hi}, dl); |
| } |
| |
| SDValue |
| HexagonTargetLowering::EqualizeFpIntConversion(SDValue Op, SelectionDAG &DAG) |
| const { |
| // Rewrite conversion between integer and floating-point in such a way that |
| // the integer type is extended/narrowed to match the bitwidth of the |
| // floating-point type, combined with additional integer-integer extensions |
| // or narrowings to match the original input/result types. |
| // E.g. f32 -> i8 ==> f32 -> i32 -> i8 |
| // |
| // The input/result types are not required to be legal, but if they are |
| // legal, this function should not introduce illegal types. |
| |
| unsigned Opc = Op.getOpcode(); |
| assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT || |
| Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP); |
| |
| SDValue Inp = Op.getOperand(0); |
| MVT InpTy = ty(Inp); |
| MVT ResTy = ty(Op); |
| |
| if (InpTy == ResTy) |
| return Op; |
| |
| const SDLoc &dl(Op); |
| bool Signed = Opc == ISD::FP_TO_SINT || Opc == ISD::SINT_TO_FP; |
| |
| auto [WInpTy, WResTy] = typeExtendToWider(InpTy, ResTy); |
| SDValue WInp = resizeToWidth(Inp, WInpTy, Signed, dl, DAG); |
| SDValue Conv = DAG.getNode(Opc, dl, WResTy, WInp); |
| SDValue Res = resizeToWidth(Conv, ResTy, Signed, dl, DAG); |
| return Res; |
| } |
| |
| SDValue |
| HexagonTargetLowering::ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const { |
| unsigned Opc = Op.getOpcode(); |
| assert(Opc == ISD::FP_TO_SINT || Opc == ISD::FP_TO_UINT); |
| |
| const SDLoc &dl(Op); |
| SDValue Op0 = Op.getOperand(0); |
| MVT InpTy = ty(Op0); |
| MVT ResTy = ty(Op); |
| assert(InpTy.changeTypeToInteger() == ResTy); |
| |
| // At this point this is an experiment under a flag. |
| // In arch before V81 the rounding mode is towards nearest value. |
| // The C/C++ standard requires rounding towards zero: |
| // C (C99 and later): ISO/IEC 9899:2018 (C18), section 6.3.1.4 — "When a |
| // finite value of real floating type is converted to an integer type, the |
| // fractional part is discarded (i.e., the value is truncated toward zero)." |
| // C++: ISO/IEC 14882:2020 (C++20), section 7.3.7 — "A prvalue of a |
| // floating-point type can be converted to a prvalue of an integer type. The |
| // conversion truncates; that is, the fractional part is discarded." |
| if (InpTy == MVT::v64f16) { |
| if (Subtarget.useHVXV81Ops()) { |
| // This is c/c++ compliant |
| SDValue ConvVec = |
| getInstr(Hexagon::V6_vconv_h_hf_rnd, dl, ResTy, {Op0}, DAG); |
| return ConvVec; |
| } else if (EnableFpFastConvert) { |
| // Vd32.h=Vu32.hf same as Q6_Vh_equals_Vhf |
| SDValue ConvVec = getInstr(Hexagon::V6_vconv_h_hf, dl, ResTy, {Op0}, DAG); |
| return ConvVec; |
| } |
| } else if (EnableFpFastConvert && InpTy == MVT::v32f32) { |
| // Vd32.w=Vu32.sf same as Q6_Vw_equals_Vsf |
| SDValue ConvVec = getInstr(Hexagon::V6_vconv_w_sf, dl, ResTy, {Op0}, DAG); |
| return ConvVec; |
| } |
| |
| // int32_t conv_f32_to_i32(uint32_t inp) { |
| // // s | exp8 | frac23 |
| // |
| // int neg = (int32_t)inp < 0; |
| // |
| // // "expm1" is the actual exponent minus 1: instead of "bias", subtract |
| // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will |
| // // produce a large positive "expm1", which will result in max u/int. |
| // // In all IEEE formats, bias is the largest positive number that can be |
| // // represented in bias-width bits (i.e. 011..1). |
| // int32_t expm1 = (inp << 1) - 0x80000000; |
| // expm1 >>= 24; |
| // |
| // // Always insert the "implicit 1". Subnormal numbers will become 0 |
| // // regardless. |
| // uint32_t frac = (inp << 8) | 0x80000000; |
| // |
| // // "frac" is the fraction part represented as Q1.31. If it was |
| // // interpreted as uint32_t, it would be the fraction part multiplied |
| // // by 2^31. |
| // |
| // // Calculate the amount of right shift, since shifting further to the |
| // // left would lose significant bits. Limit it to 32, because we want |
| // // shifts by 32+ to produce 0, whereas V6_vlsrwv treats the shift |
| // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift |
| // // left by 31). "rsh" can be negative. |
| // int32_t rsh = min(31 - (expm1 + 1), 32); |
| // |
| // frac >>= rsh; // rsh == 32 will produce 0 |
| // |
| // // Everything up to this point is the same for conversion to signed |
| // // unsigned integer. |
| // |
| // if (neg) // Only for signed int |
| // frac = -frac; // |
| // if (rsh <= 0 && neg) // bound = neg ? 0x80000000 : 0x7fffffff |
| // frac = 0x80000000; // frac = rsh <= 0 ? bound : frac |
| // if (rsh <= 0 && !neg) // |
| // frac = 0x7fffffff; // |
| // |
| // if (neg) // Only for unsigned int |
| // frac = 0; // |
| // if (rsh < 0 && !neg) // frac = rsh < 0 ? 0x7fffffff : frac; |
| // frac = 0x7fffffff; // frac = neg ? 0 : frac; |
| // |
| // return frac; |
| // } |
| |
| MVT PredTy = MVT::getVectorVT(MVT::i1, ResTy.getVectorElementCount()); |
| |
| // Zero = V6_vd0(); |
| // Neg = V6_vgtw(Zero, Inp); |
| // One = V6_lvsplatw(1); |
| // M80 = V6_lvsplatw(0x80000000); |
| // Exp00 = V6_vaslwv(Inp, One); |
| // Exp01 = V6_vsubw(Exp00, M80); |
| // ExpM1 = V6_vasrw(Exp01, 24); |
| // Frc00 = V6_vaslw(Inp, 8); |
| // Frc01 = V6_vor(Frc00, M80); |
| // Rsh00 = V6_vsubw(V6_lvsplatw(30), ExpM1); |
| // Rsh01 = V6_vminw(Rsh00, V6_lvsplatw(32)); |
| // Frc02 = V6_vlsrwv(Frc01, Rsh01); |
| |
| // if signed int: |
| // Bnd = V6_vmux(Neg, M80, V6_lvsplatw(0x7fffffff)) |
| // Pos = V6_vgtw(Rsh01, Zero); |
| // Frc13 = V6_vsubw(Zero, Frc02); |
| // Frc14 = V6_vmux(Neg, Frc13, Frc02); |
| // Int = V6_vmux(Pos, Frc14, Bnd); |
| // |
| // if unsigned int: |
| // Rsn = V6_vgtw(Zero, Rsh01) |
| // Frc23 = V6_vmux(Rsn, V6_lvsplatw(0x7fffffff), Frc02) |
| // Int = V6_vmux(Neg, Zero, Frc23) |
| |
| auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(InpTy); |
| unsigned ElemWidth = 1 + ExpWidth + FracWidth; |
| assert((1ull << (ExpWidth - 1)) == (1 + ExpBias)); |
| |
| SDValue Inp = DAG.getBitcast(ResTy, Op0); |
| SDValue Zero = getZero(dl, ResTy, DAG); |
| SDValue Neg = DAG.getSetCC(dl, PredTy, Inp, Zero, ISD::SETLT); |
| SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, ResTy); |
| SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy); |
| SDValue One = DAG.getConstant(1, dl, ResTy); |
| SDValue Exp00 = DAG.getNode(ISD::SHL, dl, ResTy, {Inp, One}); |
| SDValue Exp01 = DAG.getNode(ISD::SUB, dl, ResTy, {Exp00, M80}); |
| SDValue MNE = DAG.getConstant(ElemWidth - ExpWidth, dl, ResTy); |
| SDValue ExpM1 = DAG.getNode(ISD::SRA, dl, ResTy, {Exp01, MNE}); |
| |
| SDValue ExpW = DAG.getConstant(ExpWidth, dl, ResTy); |
| SDValue Frc00 = DAG.getNode(ISD::SHL, dl, ResTy, {Inp, ExpW}); |
| SDValue Frc01 = DAG.getNode(ISD::OR, dl, ResTy, {Frc00, M80}); |
| |
| SDValue MN2 = DAG.getConstant(ElemWidth - 2, dl, ResTy); |
| SDValue Rsh00 = DAG.getNode(ISD::SUB, dl, ResTy, {MN2, ExpM1}); |
| SDValue MW = DAG.getConstant(ElemWidth, dl, ResTy); |
| SDValue Rsh01 = DAG.getNode(ISD::SMIN, dl, ResTy, {Rsh00, MW}); |
| SDValue Frc02 = DAG.getNode(ISD::SRL, dl, ResTy, {Frc01, Rsh01}); |
| |
| SDValue Int; |
| |
| if (Opc == ISD::FP_TO_SINT) { |
| SDValue Bnd = DAG.getNode(ISD::VSELECT, dl, ResTy, {Neg, M80, M7F}); |
| SDValue Pos = DAG.getSetCC(dl, PredTy, Rsh01, Zero, ISD::SETGT); |
| SDValue Frc13 = DAG.getNode(ISD::SUB, dl, ResTy, {Zero, Frc02}); |
| SDValue Frc14 = DAG.getNode(ISD::VSELECT, dl, ResTy, {Neg, Frc13, Frc02}); |
| Int = DAG.getNode(ISD::VSELECT, dl, ResTy, {Pos, Frc14, Bnd}); |
| } else { |
| assert(Opc == ISD::FP_TO_UINT); |
| SDValue Rsn = DAG.getSetCC(dl, PredTy, Rsh01, Zero, ISD::SETLT); |
| SDValue Frc23 = DAG.getNode(ISD::VSELECT, dl, ResTy, Rsn, M7F, Frc02); |
| Int = DAG.getNode(ISD::VSELECT, dl, ResTy, Neg, Zero, Frc23); |
| } |
| |
| return Int; |
| } |
| |
| SDValue |
| HexagonTargetLowering::ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const { |
| unsigned Opc = Op.getOpcode(); |
| assert(Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP); |
| |
| const SDLoc &dl(Op); |
| SDValue Op0 = Op.getOperand(0); |
| MVT InpTy = ty(Op0); |
| MVT ResTy = ty(Op); |
| assert(ResTy.changeTypeToInteger() == InpTy); |
| |
| // uint32_t vnoc1_rnd(int32_t w) { |
| // int32_t iszero = w == 0; |
| // int32_t isneg = w < 0; |
| // uint32_t u = __builtin_HEXAGON_A2_abs(w); |
| // |
| // uint32_t norm_left = __builtin_HEXAGON_S2_cl0(u) + 1; |
| // uint32_t frac0 = (uint64_t)u << norm_left; |
| // |
| // // Rounding: |
| // uint32_t frac1 = frac0 + ((1 << 8) - 1); |
| // uint32_t renorm = (frac0 > frac1); |
| // uint32_t rup = (int)(frac0 << 22) < 0; |
| // |
| // uint32_t frac2 = frac0 >> 8; |
| // uint32_t frac3 = frac1 >> 8; |
| // uint32_t frac = (frac2 != frac3) ? frac3 >> 1 : (frac3 + rup) >> 1; |
| // |
| // int32_t exp = 32 - norm_left + renorm + 127; |
| // exp <<= 23; |
| // |
| // uint32_t sign = 0x80000000 * isneg; |
| // uint32_t f = sign | exp | frac; |
| // return iszero ? 0 : f; |
| // } |
| |
| MVT PredTy = MVT::getVectorVT(MVT::i1, InpTy.getVectorElementCount()); |
| bool Signed = Opc == ISD::SINT_TO_FP; |
| |
| auto [ExpWidth, ExpBias, FracWidth] = getIEEEProperties(ResTy); |
| unsigned ElemWidth = 1 + ExpWidth + FracWidth; |
| |
| SDValue Zero = getZero(dl, InpTy, DAG); |
| SDValue One = DAG.getConstant(1, dl, InpTy); |
| SDValue IsZero = DAG.getSetCC(dl, PredTy, Op0, Zero, ISD::SETEQ); |
| SDValue Abs = Signed ? DAG.getNode(ISD::ABS, dl, InpTy, Op0) : Op0; |
| SDValue Clz = DAG.getNode(ISD::CTLZ, dl, InpTy, Abs); |
| SDValue NLeft = DAG.getNode(ISD::ADD, dl, InpTy, {Clz, One}); |
| SDValue Frac0 = DAG.getNode(ISD::SHL, dl, InpTy, {Abs, NLeft}); |
| |
| auto [Frac, Ovf] = emitHvxShiftRightRnd(Frac0, ExpWidth + 1, false, DAG); |
| if (Signed) { |
| SDValue IsNeg = DAG.getSetCC(dl, PredTy, Op0, Zero, ISD::SETLT); |
| SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, InpTy); |
| SDValue Sign = DAG.getNode(ISD::VSELECT, dl, InpTy, {IsNeg, M80, Zero}); |
| Frac = DAG.getNode(ISD::OR, dl, InpTy, {Sign, Frac}); |
| } |
| |
| SDValue Rnrm = DAG.getZExtOrTrunc(Ovf, dl, InpTy); |
| SDValue Exp0 = DAG.getConstant(ElemWidth + ExpBias, dl, InpTy); |
| SDValue Exp1 = DAG.getNode(ISD::ADD, dl, InpTy, {Rnrm, Exp0}); |
| SDValue Exp2 = DAG.getNode(ISD::SUB, dl, InpTy, {Exp1, NLeft}); |
| SDValue Exp3 = DAG.getNode(ISD::SHL, dl, InpTy, |
| {Exp2, DAG.getConstant(FracWidth, dl, InpTy)}); |
| SDValue Flt0 = DAG.getNode(ISD::OR, dl, InpTy, {Frac, Exp3}); |
| SDValue Flt1 = DAG.getNode(ISD::VSELECT, dl, InpTy, {IsZero, Zero, Flt0}); |
| SDValue Flt = DAG.getBitcast(ResTy, Flt1); |
| |
| return Flt; |
| } |
| |
| SDValue |
| HexagonTargetLowering::CreateTLWrapper(SDValue Op, SelectionDAG &DAG) const { |
| unsigned Opc = Op.getOpcode(); |
| unsigned TLOpc; |
| switch (Opc) { |
| case ISD::ANY_EXTEND: |
| case ISD::SIGN_EXTEND: |
| case ISD::ZERO_EXTEND: |
| TLOpc = HexagonISD::TL_EXTEND; |
| break; |
| case ISD::TRUNCATE: |
| TLOpc = HexagonISD::TL_TRUNCATE; |
| break; |
| #ifndef NDEBUG |
| Op.dump(&DAG); |
| #endif |
| llvm_unreachable("Unexpected operator"); |
| } |
| |
| const SDLoc &dl(Op); |
| return DAG.getNode(TLOpc, dl, ty(Op), Op.getOperand(0), |
| DAG.getUNDEF(MVT::i128), // illegal type |
| DAG.getConstant(Opc, dl, MVT::i32)); |
| } |
| |
| SDValue |
| HexagonTargetLowering::RemoveTLWrapper(SDValue Op, SelectionDAG &DAG) const { |
| assert(Op.getOpcode() == HexagonISD::TL_EXTEND || |
| Op.getOpcode() == HexagonISD::TL_TRUNCATE); |
| unsigned Opc = Op.getConstantOperandVal(2); |
| return DAG.getNode(Opc, SDLoc(Op), ty(Op), Op.getOperand(0)); |
| } |
| |
| HexagonTargetLowering::VectorPair |
| HexagonTargetLowering::SplitVectorOp(SDValue Op, SelectionDAG &DAG) const { |
| assert(!Op.isMachineOpcode()); |
| SmallVector<SDValue, 2> OpsL, OpsH; |
| const SDLoc &dl(Op); |
| |
| auto SplitVTNode = [&DAG, this](const VTSDNode *N) { |
| MVT Ty = typeSplit(N->getVT().getSimpleVT()).first; |
| SDValue TV = DAG.getValueType(Ty); |
| return std::make_pair(TV, TV); |
| }; |
| |
| for (SDValue A : Op.getNode()->ops()) { |
| auto [Lo, Hi] = |
| ty(A).isVector() ? opSplit(A, dl, DAG) : std::make_pair(A, A); |
| // Special case for type operand. |
| switch (Op.getOpcode()) { |
| case ISD::SIGN_EXTEND_INREG: |
| case HexagonISD::SSAT: |
| case HexagonISD::USAT: |
| if (const auto *N = dyn_cast<const VTSDNode>(A.getNode())) |
| std::tie(Lo, Hi) = SplitVTNode(N); |
| break; |
| } |
| OpsL.push_back(Lo); |
| OpsH.push_back(Hi); |
| } |
| |
| MVT ResTy = ty(Op); |
| MVT HalfTy = typeSplit(ResTy).first; |
| SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL); |
| SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH); |
| return {L, H}; |
| } |
| |
| SDValue |
| HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const { |
| auto *MemN = cast<MemSDNode>(Op.getNode()); |
| unsigned MemOpc = MemN->getOpcode(); |
| EVT MemTy = MemN->getMemoryVT(); |
| |
| if ((MemOpc == ISD::STORE || MemOpc == ISD::LOAD) && |
| (!MemTy.isSimple() || !isHvxPairTy(MemTy.getSimpleVT()))) |
| return Op; |
| |
| EVT ValueType; |
| if (MemOpc == ISD::STORE) |
| ValueType = ty(cast<StoreSDNode>(Op)->getValue()); |
| else if (MemOpc == ISD::MSTORE) |
| ValueType = ty(cast<MaskedStoreSDNode>(Op)->getValue()); |
| else // ISD::LOAD, ISD::MLOAD. |
| ValueType = MemN->getValueType(0); |
| |
| EVT LoVT, HiVT; |
| std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(ValueType); |
| |
| EVT LoMemVT, HiMemVT; |
| bool HiIsEmpty = false; |
| std::tie(LoMemVT, HiMemVT) = |
| DAG.GetDependentSplitDestVTs(MemTy, LoVT, &HiIsEmpty); |
| |
| uint64_t LoSize = LoMemVT.getSizeInBits().getFixedValue() / 8; |
| uint64_t HiSize = HiMemVT.getSizeInBits().getFixedValue() / 8; |
| |
| const SDLoc &dl(Op); |
| SDValue Chain = MemN->getChain(); |
| SDValue Base0 = MemN->getBasePtr(); |
| SDValue Base1 = |
| DAG.getMemBasePlusOffset(Base0, TypeSize::getFixed(LoSize), dl); |
| |
| MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr; |
| if (MachineMemOperand *MMO = MemN->getMemOperand()) { |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto MemSize = [=](uint64_t Size) { |
| return (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE) |
| ? (uint64_t)MemoryLocation::UnknownSize |
| : Size; |
| }; |
| // MOp1 will not be used if HiIsEmpty for masked loads and stores (MLOAD and |
| // MSTORE). Non-masked loads and store are always of double-vector size (see |
| // isHvxPairTy() check above). |
| MOp0 = MF.getMachineMemOperand(MMO, 0, MemSize(LoSize)); |
| MOp1 = MF.getMachineMemOperand(MMO, LoSize, MemSize(HiSize)); |
| } |
| |
| if (MemOpc == ISD::LOAD) { |
| assert(cast<LoadSDNode>(Op)->isUnindexed()); |
| SDValue Load0 = DAG.getLoad(LoVT, dl, Chain, Base0, MOp0); |
| SDValue Load1 = DAG.getLoad(HiVT, dl, Chain, Base1, MOp1); |
| return DAG.getMergeValues( |
| {DAG.getNode(ISD::CONCAT_VECTORS, dl, MemN->getValueType(0), Load0, |
| Load1), |
| DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Load0.getValue(1), |
| Load1.getValue(1))}, |
| dl); |
| } |
| if (MemOpc == ISD::STORE) { |
| assert(cast<StoreSDNode>(Op)->isUnindexed()); |
| VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG); |
| SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0); |
| SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1); |
| return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1); |
| } |
| |
| assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE); |
| |
| auto MaskN = cast<MaskedLoadStoreSDNode>(Op); |
| assert(MaskN->isUnindexed()); |
| VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG); |
| SDValue Offset = DAG.getPOISON(MVT::i32); |
| |
| if (MemOpc == ISD::MLOAD) { |
| VectorPair Thru = |
| opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG); |
| SDValue MLoad0 = DAG.getMaskedLoad(LoVT, dl, Chain, Base0, Offset, |
| Masks.first, Thru.first, LoMemVT, MOp0, |
| ISD::UNINDEXED, ISD::NON_EXTLOAD, false); |
| |
| // The hi masked load has zero storage size. We therefore simply set it to |
| // the low masked load and rely on subsequent removal from the chain as it |
| // is unused. See DAGTypeLegalizer::SplitVecRes_MLOAD() for the same logic. |
| SDValue MLoad1 = |
| HiIsEmpty ? MLoad0 |
| : DAG.getMaskedLoad(HiVT, dl, Chain, Base1, Offset, |
| Masks.second, Thru.second, HiMemVT, MOp1, |
| ISD::UNINDEXED, ISD::NON_EXTLOAD, false); |
| return DAG.getMergeValues( |
| {DAG.getNode(ISD::CONCAT_VECTORS, dl, MemN->getValueType(0), MLoad0, |
| MLoad1), |
| DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MLoad0.getValue(1), |
| MLoad1.getValue(1))}, |
| dl); |
| } |
| if (MemOpc == ISD::MSTORE) { |
| VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG); |
| SDValue MStore0 = |
| DAG.getMaskedStore(Chain, dl, Vals.first, Base0, Offset, Masks.first, |
| LoMemVT, MOp0, ISD::UNINDEXED, false, false); |
| if (HiIsEmpty) |
| return MStore0; |
| SDValue MStore1 = |
| DAG.getMaskedStore(Chain, dl, Vals.second, Base1, Offset, Masks.second, |
| HiMemVT, MOp1, ISD::UNINDEXED, false, false); |
| return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MStore0, MStore1); |
| } |
| |
| std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG); |
| llvm_unreachable(Name.c_str()); |
| } |
| |
| SDValue |
| HexagonTargetLowering::WidenHvxLoad(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| auto *LoadN = cast<LoadSDNode>(Op.getNode()); |
| assert(LoadN->isUnindexed() && "Not widening indexed loads yet"); |
| assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 && |
| "Not widening loads of i1 yet"); |
| |
| SDValue Chain = LoadN->getChain(); |
| SDValue Base = LoadN->getBasePtr(); |
| SDValue Offset = DAG.getPOISON(MVT::i32); |
| |
| MVT ResTy = ty(Op); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| unsigned ResLen = ResTy.getStoreSize(); |
| assert(ResLen < HwLen && "vsetq(v1) prerequisite"); |
| |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); |
| SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, |
| {DAG.getConstant(ResLen, dl, MVT::i32)}, DAG); |
| |
| MVT LoadTy = MVT::getVectorVT(MVT::i8, HwLen); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto *MemOp = MF.getMachineMemOperand(LoadN->getMemOperand(), 0, HwLen); |
| |
| SDValue Load = DAG.getMaskedLoad(LoadTy, dl, Chain, Base, Offset, Mask, |
| DAG.getUNDEF(LoadTy), LoadTy, MemOp, |
| ISD::UNINDEXED, ISD::NON_EXTLOAD, false); |
| SDValue Value = opCastElem(Load, ResTy.getVectorElementType(), DAG); |
| return DAG.getMergeValues({Value, Load.getValue(1)}, dl); |
| } |
| |
| SDValue |
| HexagonTargetLowering::WidenHvxStore(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| auto *StoreN = cast<StoreSDNode>(Op.getNode()); |
| assert(StoreN->isUnindexed() && "Not widening indexed stores yet"); |
| assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 && |
| "Not widening stores of i1 yet"); |
| |
| SDValue Chain = StoreN->getChain(); |
| SDValue Base = StoreN->getBasePtr(); |
| SDValue Offset = DAG.getPOISON(MVT::i32); |
| |
| SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG); |
| MVT ValueTy = ty(Value); |
| unsigned ValueLen = ValueTy.getVectorNumElements(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| assert(isPowerOf2_32(ValueLen)); |
| |
| for (unsigned Len = ValueLen; Len < HwLen; ) { |
| Value = opJoin({Value, DAG.getUNDEF(ty(Value))}, dl, DAG); |
| Len = ty(Value).getVectorNumElements(); // This is Len *= 2 |
| } |
| assert(ty(Value).getVectorNumElements() == HwLen); // Paranoia |
| |
| assert(ValueLen < HwLen && "vsetq(v1) prerequisite"); |
| MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen); |
| SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy, |
| {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG); |
| MachineFunction &MF = DAG.getMachineFunction(); |
| auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen); |
| return DAG.getMaskedStore(Chain, dl, Value, Base, Offset, Mask, |
| StoreN->getMemoryVT(), MemOp, ISD::UNINDEXED, false, |
| false); |
| } |
| |
| SDValue |
| HexagonTargetLowering::WidenHvxSetCC(SDValue Op, SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1); |
| MVT ElemTy = ty(Op0).getVectorElementType(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| |
| unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits(); |
| assert(WideOpLen * ElemTy.getSizeInBits() == 8 * HwLen); |
| MVT WideOpTy = MVT::getVectorVT(ElemTy, WideOpLen); |
| if (!Subtarget.isHVXVectorType(WideOpTy, true)) |
| return SDValue(); |
| |
| SDValue WideOp0 = appendUndef(Op0, WideOpTy, DAG); |
| SDValue WideOp1 = appendUndef(Op1, WideOpTy, DAG); |
| EVT ResTy = |
| getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), WideOpTy); |
| SDValue SetCC = DAG.getNode(ISD::SETCC, dl, ResTy, |
| {WideOp0, WideOp1, Op.getOperand(2)}); |
| |
| EVT RetTy = typeLegalize(ty(Op), DAG); |
| return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RetTy, |
| {SetCC, getZero(dl, MVT::i32, DAG)}); |
| } |
| |
| SDValue HexagonTargetLowering::WidenHvxTruncateToBool(SDValue Op, |
| SelectionDAG &DAG) const { |
| // Handle truncation to boolean vector where the result boolean type |
| // needs widening (e.g., v16i32 -> v16i1 where v16i1 is not a standard |
| // HVX predicate type, or v16i8 -> v16i1 in 128-byte mode). |
| // Widen the input to HVX width, perform the truncate to the widened |
| // boolean type, then extract the result. |
| const SDLoc &dl(Op); |
| SDValue Inp = Op.getOperand(0); |
| MVT InpTy = ty(Inp); |
| MVT ResTy = ty(Op); |
| |
| assert(ResTy.getVectorElementType() == MVT::i1 && |
| "Expected boolean result type"); |
| |
| MVT ElemTy = InpTy.getVectorElementType(); |
| unsigned HwLen = Subtarget.getVectorLength(); |
| |
| // Calculate the widened input type that fills the HVX register. |
| unsigned WideLen = (8 * HwLen) / ElemTy.getSizeInBits(); |
| MVT WideInpTy = MVT::getVectorVT(ElemTy, WideLen); |
| if (!Subtarget.isHVXVectorType(WideInpTy, false)) |
| return SDValue(); |
| |
| // Widen the input to HVX width. |
| SDValue WideInp = appendUndef(Inp, WideInpTy, DAG); |
| |
| // Perform the truncate to widened boolean type. |
| MVT WideBoolTy = MVT::getVectorVT(MVT::i1, WideLen); |
| SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, WideBoolTy, WideInp); |
| |
| // Extract the result. |
| EVT RetTy = typeLegalize(ResTy, DAG); |
| return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RetTy, |
| {WideTrunc, getZero(dl, MVT::i32, DAG)}); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const { |
| unsigned Opc = Op.getOpcode(); |
| bool IsPairOp = isHvxPairTy(ty(Op)) || |
| llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) { |
| return isHvxPairTy(ty(V)); |
| }); |
| |
| if (IsPairOp) { |
| switch (Opc) { |
| default: |
| break; |
| case ISD::LOAD: |
| case ISD::STORE: |
| case ISD::MLOAD: |
| case ISD::MSTORE: |
| return SplitHvxMemOp(Op, DAG); |
| case ISD::SINT_TO_FP: |
| case ISD::UINT_TO_FP: |
| case ISD::FP_TO_SINT: |
| case ISD::FP_TO_UINT: |
| if (ty(Op).getSizeInBits() == ty(Op.getOperand(0)).getSizeInBits()) |
| return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG); |
| break; |
| case ISD::ABS: |
| case ISD::CTPOP: |
| case ISD::CTLZ: |
| case ISD::CTTZ: |
| case ISD::MUL: |
| case ISD::FADD: |
| case ISD::FSUB: |
| case ISD::FMUL: |
| case ISD::FMINIMUMNUM: |
| case ISD::FMAXIMUMNUM: |
| case ISD::FMINIMUM: |
| case ISD::FMAXIMUM: |
| case ISD::FMINNUM: |
| case ISD::FMAXNUM: |
| case ISD::MULHS: |
| case ISD::MULHU: |
| case ISD::AND: |
| case ISD::OR: |
| case ISD::XOR: |
| case ISD::SRA: |
| case ISD::SHL: |
| case ISD::SRL: |
| case ISD::FSHL: |
| case ISD::FSHR: |
| case ISD::SMIN: |
| case ISD::SMAX: |
| case ISD::UMIN: |
| case ISD::UMAX: |
| case ISD::SETCC: |
| case ISD::VSELECT: |
| case ISD::SIGN_EXTEND_INREG: |
| case ISD::SPLAT_VECTOR: |
| return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG); |
| case ISD::SIGN_EXTEND: |
| case ISD::ZERO_EXTEND: |
| // In general, sign- and zero-extends can't be split and still |
| // be legal. The only exception is extending bool vectors. |
| if (ty(Op.getOperand(0)).getVectorElementType() == MVT::i1) |
| return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG); |
| break; |
| } |
| } |
| |
| switch (Opc) { |
| default: |
| break; |
| // clang-format off |
| case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG); |
| case ISD::SPLAT_VECTOR: return LowerHvxSplatVector(Op, DAG); |
| case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG); |
| case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG); |
| case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG); |
| case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG); |
| case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG); |
| case ISD::BITCAST: return LowerHvxBitcast(Op, DAG); |
| case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG); |
| case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG); |
| case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG); |
| case ISD::CTTZ: return LowerHvxCttz(Op, DAG); |
| case ISD::SELECT: return LowerHvxSelect(Op, DAG); |
| case ISD::SRA: |
| case ISD::SHL: |
| case ISD::SRL: return LowerHvxShift(Op, DAG); |
| case ISD::FSHL: |
| case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG); |
| case ISD::MULHS: |
| case ISD::MULHU: return LowerHvxMulh(Op, DAG); |
| case ISD::SMUL_LOHI: |
| case ISD::UMUL_LOHI: return LowerHvxMulLoHi(Op, DAG); |
| case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG); |
| case ISD::SETCC: { |
| ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); |
| if (CC == ISD::SETOEQ && |
| ty(Op.getOperand(0)).getScalarType().isFloatingPoint()) |
| return LowerHvxFpSetoeq(Op, DAG); |
| return Op; |
| } |
| case ISD::INTRINSIC_VOID: return Op; |
| case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG); |
| case ISD::MLOAD: |
| case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG); |
| // Unaligned loads will be handled by the default lowering. |
| case ISD::LOAD: return LowerHvxLoad(Op, DAG); |
| case ISD::STORE: return LowerHvxStore(Op, DAG); |
| case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG); |
| case ISD::FP_TO_SINT: |
| case ISD::FP_TO_UINT: return LowerHvxFpToInt(Op, DAG); |
| case ISD::SINT_TO_FP: |
| case ISD::UINT_TO_FP: return LowerHvxIntToFp(Op, DAG); |
| |
| // Special nodes: |
| case HexagonISD::SMUL_LOHI: |
| case HexagonISD::UMUL_LOHI: |
| case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG); |
| |
| case ISD::PARTIAL_REDUCE_SMLA: |
| case ISD::PARTIAL_REDUCE_UMLA: |
| case ISD::PARTIAL_REDUCE_SUMLA: |
| return LowerHvxPartialReduceMLA(Op, DAG); |
| case ISD::VECREDUCE_FMIN: |
| return LowerHvxVecReduceFMin(Op, DAG); |
| case ISD::VECREDUCE_FMAX: |
| return LowerHvxVecReduceFMax(Op, DAG); |
| case ISD::VECREDUCE_FMINIMUM: |
| return LowerHvxVecReduceFMinimum(Op, DAG); |
| case ISD::VECREDUCE_FMAXIMUM: |
| return LowerHvxVecReduceFMaximum(Op, DAG); |
| case ISD::FMINNUM: |
| return LowerHvxFMinNum(Op, DAG); |
| case ISD::FMAXNUM: |
| return LowerHvxFMaxNum(Op, DAG); |
| // clang-format on |
| } |
| #ifndef NDEBUG |
| Op.dumpr(&DAG); |
| #endif |
| llvm_unreachable("Unhandled HVX operation"); |
| } |
| |
| SDValue |
| HexagonTargetLowering::ExpandHvxResizeIntoSteps(SDValue Op, SelectionDAG &DAG) |
| const { |
| // Rewrite the extension/truncation/saturation op into steps where each |
| // step changes the type widths by a factor of 2. |
| // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32. |
| // |
| // Some of the vector types in Op may not be legal. |
| |
| unsigned Opc = Op.getOpcode(); |
| switch (Opc) { |
| case HexagonISD::SSAT: |
| case HexagonISD::USAT: |
| case HexagonISD::TL_EXTEND: |
| case HexagonISD::TL_TRUNCATE: |
| break; |
| case ISD::ANY_EXTEND: |
| case ISD::ZERO_EXTEND: |
| case ISD::SIGN_EXTEND: |
| case ISD::TRUNCATE: |
| llvm_unreachable("ISD:: ops will be auto-folded"); |
| break; |
| #ifndef NDEBUG |
| Op.dump(&DAG); |
| #endif |
| llvm_unreachable("Unexpected operation"); |
| } |
| |
| SDValue Inp = Op.getOperand(0); |
| MVT InpTy = ty(Inp); |
| MVT ResTy = ty(Op); |
| |
| unsigned InpWidth = InpTy.getVectorElementType().getSizeInBits(); |
| unsigned ResWidth = ResTy.getVectorElementType().getSizeInBits(); |
| assert(InpWidth != ResWidth); |
| |
| if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth) |
| return Op; |
| |
| const SDLoc &dl(Op); |
| unsigned NumElems = InpTy.getVectorNumElements(); |
| assert(NumElems == ResTy.getVectorNumElements()); |
| |
| auto repeatOp = [&](unsigned NewWidth, SDValue Arg) { |
| MVT Ty = MVT::getVectorVT(MVT::getIntegerVT(NewWidth), NumElems); |
| switch (Opc) { |
| case HexagonISD::SSAT: |
| case HexagonISD::USAT: |
| return DAG.getNode(Opc, dl, Ty, {Arg, DAG.getValueType(Ty)}); |
| case HexagonISD::TL_EXTEND: |
| case HexagonISD::TL_TRUNCATE: |
| return DAG.getNode(Opc, dl, Ty, {Arg, Op.getOperand(1), Op.getOperand(2)}); |
| default: |
| llvm_unreachable("Unexpected opcode"); |
| } |
| }; |
| |
| SDValue S = Inp; |
| if (InpWidth < ResWidth) { |
| assert(ResWidth % InpWidth == 0 && isPowerOf2_32(ResWidth / InpWidth)); |
| while (InpWidth * 2 <= ResWidth) |
| S = repeatOp(InpWidth *= 2, S); |
| } else { |
| // InpWidth > ResWidth |
| assert(InpWidth % ResWidth == 0 && isPowerOf2_32(InpWidth / ResWidth)); |
| while (InpWidth / 2 >= ResWidth) |
| S = repeatOp(InpWidth /= 2, S); |
| } |
| return S; |
| } |
| |
| SDValue |
| HexagonTargetLowering::LegalizeHvxResize(SDValue Op, SelectionDAG &DAG) const { |
| SDValue Inp0 = Op.getOperand(0); |
| MVT InpTy = ty(Inp0); |
| MVT ResTy = ty(Op); |
| unsigned InpWidth = InpTy.getSizeInBits(); |
| unsigned ResWidth = ResTy.getSizeInBits(); |
| unsigned Opc = Op.getOpcode(); |
| |
| if (shouldWidenToHvx(InpTy, DAG) || shouldWidenToHvx(ResTy, DAG)) { |
| // First, make sure that the narrower type is widened to HVX. |
| // This may cause the result to be wider than what the legalizer |
| // expects, so insert EXTRACT_SUBVECTOR to bring it back to the |
| // desired type. |
| auto [WInpTy, WResTy] = |
| InpWidth < ResWidth ? typeWidenToWider(typeWidenToHvx(InpTy), ResTy) |
| : typeWidenToWider(InpTy, typeWidenToHvx(ResTy)); |
| SDValue W = appendUndef(Inp0, WInpTy, DAG); |
| SDValue S; |
| if (Opc == HexagonISD::TL_EXTEND || Opc == HexagonISD::TL_TRUNCATE) { |
| S = DAG.getNode(Opc, SDLoc(Op), WResTy, W, Op.getOperand(1), |
| Op.getOperand(2)); |
| } else { |
| S = DAG.getNode(Opc, SDLoc(Op), WResTy, W, DAG.getValueType(WResTy)); |
| } |
| SDValue T = ExpandHvxResizeIntoSteps(S, DAG); |
| return extractSubvector(T, typeLegalize(ResTy, DAG), 0, DAG); |
| } else if (shouldSplitToHvx(InpWidth < ResWidth ? ResTy : InpTy, DAG)) { |
| // For multi-step extends/truncates (e.g., i8->i32), expand into |
| // single-step operations first. Splitting a multi-step TL_EXTEND |
| // would halve the operand type to a sub-HVX size (e.g., v128i8 -> |
| // v64i8), creating illegal types that cause issues in the type |
| // legalizer's map tracking. Single-step operations (e.g., i16->i32) |
| // are safe to split because their halved operand types remain legal. |
| SDValue T = ExpandHvxResizeIntoSteps(Op, DAG); |
| if (T != Op) |
| return T; |
| return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG); |
| } else { |
| assert(isTypeLegal(InpTy) && isTypeLegal(ResTy)); |
| return RemoveTLWrapper(Op, DAG); |
| } |
| llvm_unreachable("Unexpected situation"); |
| } |
| |
| void |
| HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N, |
| SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { |
| unsigned Opc = N->getOpcode(); |
| SDValue Op(N, 0); |
| SDValue Inp0; // Optional first argument. |
| if (N->getNumOperands() > 0) |
| Inp0 = Op.getOperand(0); |
| |
| switch (Opc) { |
| case ISD::ANY_EXTEND: |
| case ISD::SIGN_EXTEND: |
| case ISD::ZERO_EXTEND: |
| if (Subtarget.isHVXElementType(ty(Op)) && |
| Subtarget.isHVXElementType(ty(Inp0))) { |
| Results.push_back(CreateTLWrapper(Op, DAG)); |
| } |
| break; |
| case ISD::TRUNCATE: |
| // Handle truncate to boolean vector when the input is not a |
| // standard HVX vector type (single or pair). This covers cases |
| // where the input needs widening (e.g., v64i8 -> v64i1 in |
| // 128-byte mode) and cases where the result boolean type itself |
| // needs widening (e.g., v16i32 -> v16i1). When the input is |
| // already an HVX type, tablegen patterns handle the truncation |
| // directly (e.g., v64i16 -> v64i1 via V6_vandvrt). |
| if (ty(Op).getVectorElementType() == MVT::i1 && |
| !Subtarget.isHVXVectorType(ty(Inp0), false)) { |
| if (SDValue T = WidenHvxTruncateToBool(Op, DAG)) |
| Results.push_back(T); |
| } else if (Subtarget.isHVXElementType(ty(Op)) && |
| Subtarget.isHVXElementType(ty(Inp0))) { |
| Results.push_back(CreateTLWrapper(Op, DAG)); |
| } |
| break; |
| case ISD::SETCC: |
| if (shouldWidenToHvx(ty(Inp0), DAG)) { |
| if (SDValue T = WidenHvxSetCC(Op, DAG)) |
| Results.push_back(T); |
| } |
| break; |
| case ISD::STORE: { |
| if (shouldWidenToHvx(ty(cast<StoreSDNode>(N)->getValue()), DAG)) { |
| SDValue Store = WidenHvxStore(Op, DAG); |
| Results.push_back(Store); |
| } |
| break; |
| } |
| case ISD::MLOAD: |
| if (isHvxPairTy(ty(Op))) { |
| SDValue S = SplitHvxMemOp(Op, DAG); |
| assert(S->getOpcode() == ISD::MERGE_VALUES); |
| Results.push_back(S.getOperand(0)); |
| Results.push_back(S.getOperand(1)); |
| } |
| break; |
| case ISD::MSTORE: |
| if (isHvxPairTy(ty(Op->getOperand(1)))) { // Stored value |
| SDValue S = SplitHvxMemOp(Op, DAG); |
| Results.push_back(S); |
| } |
| break; |
| case ISD::SINT_TO_FP: |
| case ISD::UINT_TO_FP: |
| case ISD::FP_TO_SINT: |
| case ISD::FP_TO_UINT: |
| if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) { |
| SDValue T = EqualizeFpIntConversion(Op, DAG); |
| Results.push_back(T); |
| } |
| break; |
| case HexagonISD::SSAT: |
| case HexagonISD::USAT: |
| case HexagonISD::TL_EXTEND: |
| case HexagonISD::TL_TRUNCATE: |
| Results.push_back(LegalizeHvxResize(Op, DAG)); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| void |
| HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N, |
| SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { |
| unsigned Opc = N->getOpcode(); |
| SDValue Op(N, 0); |
| SDValue Inp0; // Optional first argument. |
| if (N->getNumOperands() > 0) |
| Inp0 = Op.getOperand(0); |
| |
| switch (Opc) { |
| case ISD::ANY_EXTEND: |
| case ISD::SIGN_EXTEND: |
| case ISD::ZERO_EXTEND: |
| if (Subtarget.isHVXElementType(ty(Op)) && |
| Subtarget.isHVXElementType(ty(Inp0))) { |
| Results.push_back(CreateTLWrapper(Op, DAG)); |
| } |
| break; |
| case ISD::TRUNCATE: |
| // Handle truncate to boolean vector when the input is not a |
| // standard HVX vector type. See comment in LowerHvxOperationWrapper. |
| if (ty(Op).getVectorElementType() == MVT::i1 && |
| !Subtarget.isHVXVectorType(ty(Inp0), false)) { |
| if (SDValue T = WidenHvxTruncateToBool(Op, DAG)) |
| Results.push_back(T); |
| } else if (Subtarget.isHVXElementType(ty(Op)) && |
| Subtarget.isHVXElementType(ty(Inp0))) { |
| Results.push_back(CreateTLWrapper(Op, DAG)); |
| } |
| break; |
| case ISD::SETCC: |
| if (shouldWidenToHvx(ty(Op), DAG)) { |
| if (SDValue T = WidenHvxSetCC(Op, DAG)) |
| Results.push_back(T); |
| } |
| break; |
| case ISD::LOAD: { |
| if (shouldWidenToHvx(ty(Op), DAG)) { |
| SDValue Load = WidenHvxLoad(Op, DAG); |
| assert(Load->getOpcode() == ISD::MERGE_VALUES); |
| Results.push_back(Load.getOperand(0)); |
| Results.push_back(Load.getOperand(1)); |
| } |
| break; |
| } |
| case ISD::BITCAST: |
| if (isHvxBoolTy(ty(Inp0))) { |
| SDValue C = LowerHvxBitcast(Op, DAG); |
| Results.push_back(C); |
| } |
| break; |
| case ISD::FP_TO_SINT: |
| case ISD::FP_TO_UINT: |
| if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) { |
| SDValue T = EqualizeFpIntConversion(Op, DAG); |
| Results.push_back(T); |
| } |
| break; |
| case HexagonISD::SSAT: |
| case HexagonISD::USAT: |
| case HexagonISD::TL_EXTEND: |
| case HexagonISD::TL_TRUNCATE: |
| Results.push_back(LegalizeHvxResize(Op, DAG)); |
| break; |
| default: |
| break; |
| } |
| } |
| |
| SDValue |
| HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op, |
| DAGCombinerInfo &DCI) const { |
| // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB |
| // to extract-subvector (shuffle V, pick even, pick odd) |
| |
| assert(Op.getOpcode() == ISD::TRUNCATE); |
| SelectionDAG &DAG = DCI.DAG; |
| const SDLoc &dl(Op); |
| |
| if (Op.getOperand(0).getOpcode() == ISD::BITCAST) |
| return SDValue(); |
| SDValue Cast = Op.getOperand(0); |
| SDValue Src = Cast.getOperand(0); |
| |
| EVT TruncTy = Op.getValueType(); |
| EVT CastTy = Cast.getValueType(); |
| EVT SrcTy = Src.getValueType(); |
| if (SrcTy.isSimple()) |
| return SDValue(); |
| if (SrcTy.getVectorElementType() != TruncTy.getVectorElementType()) |
| return SDValue(); |
| unsigned SrcLen = SrcTy.getVectorNumElements(); |
| unsigned CastLen = CastTy.getVectorNumElements(); |
| if (2 * CastLen != SrcLen) |
| return SDValue(); |
| |
| SmallVector<int, 128> Mask(SrcLen); |
| for (int i = 0; i != static_cast<int>(CastLen); ++i) { |
| Mask[i] = 2 * i; |
| Mask[i + CastLen] = 2 * i + 1; |
| } |
| SDValue Deal = |
| DAG.getVectorShuffle(SrcTy, dl, Src, DAG.getUNDEF(SrcTy), Mask); |
| return opSplit(Deal, dl, DAG).first; |
| } |
| |
| SDValue |
| HexagonTargetLowering::combineConcatOfShuffles(SDValue Op, |
| SelectionDAG &DAG) const { |
| // Fold |
| // concat (shuffle x, y, m1), (shuffle x, y, m2) |
| // into |
| // shuffle (concat x, y), undef, m3 |
| if (Op.getNumOperands() != 2) |
| return SDValue(); |
| |
| const SDLoc &dl(Op); |
| SDValue V0 = Op.getOperand(0); |
| SDValue V1 = Op.getOperand(1); |
| |
| if (V0.getOpcode() != ISD::VECTOR_SHUFFLE) |
| return SDValue(); |
| if (V1.getOpcode() != ISD::VECTOR_SHUFFLE) |
| return SDValue(); |
| |
| SetVector<SDValue> Order; |
| Order.insert(V0.getOperand(0)); |
| Order.insert(V0.getOperand(1)); |
| Order.insert(V1.getOperand(0)); |
| Order.insert(V1.getOperand(1)); |
| |
| if (Order.size() > 2) |
| return SDValue(); |
| |
| // In ISD::VECTOR_SHUFFLE, the types of each input and the type of the |
| // result must be the same. |
| EVT InpTy = V0.getValueType(); |
| assert(InpTy.isVector()); |
| unsigned InpLen = InpTy.getVectorNumElements(); |
| |
| SmallVector<int, 128> LongMask; |
| auto AppendToMask = [&](SDValue Shuffle) { |
| auto *SV = cast<ShuffleVectorSDNode>(Shuffle.getNode()); |
| ArrayRef<int> Mask = SV->getMask(); |
| SDValue X = Shuffle.getOperand(0); |
| SDValue Y = Shuffle.getOperand(1); |
| for (int M : Mask) { |
| if (M == -1) { |
| LongMask.push_back(M); |
| continue; |
| } |
| SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y; |
| if (static_cast<unsigned>(M) >= InpLen) |
| M -= InpLen; |
| |
| int OutOffset = Order[0] == Src ? 0 : InpLen; |
| LongMask.push_back(M + OutOffset); |
| } |
| }; |
| |
| AppendToMask(V0); |
| AppendToMask(V1); |
| |
| SDValue C0 = Order.front(); |
| SDValue C1 = Order.back(); // Can be same as front |
| EVT LongTy = InpTy.getDoubleNumVectorElementsVT(*DAG.getContext()); |
| |
| SDValue Cat = DAG.getNode(ISD::CONCAT_VECTORS, dl, LongTy, {C0, C1}); |
| return DAG.getVectorShuffle(LongTy, dl, Cat, DAG.getUNDEF(LongTy), LongMask); |
| } |
| |
| // Reassociate concat(p1, p2, ...) into |
| // concat(concat(p1, ...), concat(pi, ...), ...) |
| // where each inner concat produces a predicate where each bit corresponds |
| // to at most BitBytes bytes. |
| // Concatenating predicates decreases the number of bytes per each predicate |
| // bit. |
| SDValue |
| HexagonTargetLowering::combineConcatOfScalarPreds(SDValue Op, unsigned BitBytes, |
| SelectionDAG &DAG) const { |
| const SDLoc &dl(Op); |
| SmallVector<SDValue> Ops(Op->ops()); |
| MVT ResTy = ty(Op); |
| MVT InpTy = ty(Ops[0]); |
| unsigned InpLen = InpTy.getVectorNumElements(); // Scalar predicate |
| unsigned ResLen = ResTy.getVectorNumElements(); // HVX vector predicate |
| assert(InpLen <= 8 && "Too long for scalar predicate"); |
| assert(ResLen > 8 && "Too short for HVX vector predicate"); |
| |
| unsigned Bytes = 8 / InpLen; // Bytes-per-bit in input |
| |
| // Already in the right form? |
| if (Bytes <= BitBytes) |
| return Op; |
| |
| ArrayRef<SDValue> Inputs(Ops); |
| unsigned SliceLen = Bytes / BitBytes; |
| |
| SmallVector<SDValue> Cats; |
| // (8 / BitBytes) is the desired length of the result of the inner concat. |
| MVT InnerTy = MVT::getVectorVT(MVT::i1, 8 / BitBytes); |
| for (unsigned i = 0; i != ResLen / (8 / BitBytes); ++i) { |
| SDValue Cat = DAG.getNode(ISD::CONCAT_VECTORS, dl, InnerTy, |
| Inputs.slice(SliceLen * i, SliceLen)); |
| Cats.push_back(Cat); |
| } |
| |
| return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, Cats); |
| } |
| |
| SDValue HexagonTargetLowering::combineConcatVectorsBeforeLegal( |
| SDValue Op, DAGCombinerInfo &DCI) const { |
| MVT ResTy = ty(Op); |
| MVT ElemTy = ResTy.getVectorElementType(); |
| |
| if (ElemTy != MVT::i1) { |
| return combineConcatOfShuffles(Op, DCI.DAG); |
| } |
| return SDValue(); |
| } |
| |
| // Create the inner partial reduction MLA that can be efficiently lowered. This |
| // function is used by partial and full reductions. |
| SDValue HexagonTargetLowering::createExtendingPartialReduceMLA( |
| unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType, |
| const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio, |
| const SDLoc &DL, SelectionDAG &DAG) const { |
| const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>(); |
| if (!Subtarget.useHVXOps()) |
| return SDValue(); |
| |
| EVT InputEltType = InputType.getVectorElementType(); |
| |
| // Find if an optimized instruction for the sub-reduction is available. |
| unsigned NativeRatio; |
| if (AccEltType == MVT::i32 && InputEltType == MVT::i8) |
| NativeRatio = 4; |
| else |
| return SDValue(); |
| |
| // We only handle the case when additional reduction will be needed, i.e. |
| // input is longer by a larger factor than the result. |
| ElementCount InputEC = InputType.getVectorElementCount(); |
| if (!InputEC.isKnownMultipleOf(AccNumElements * NativeRatio)) |
| return SDValue(); |
| |
| unsigned InputNumElements = InputEC.getFixedValue(); |
| RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio); |
| if (RemainingReductionRatio == 1) |
| return SDValue(); |
| |
| // Create a reduction by the natively supported factor. |
| EVT IntermediateType = EVT::getVectorVT(*DAG.getContext(), AccEltType, |
| InputNumElements / NativeRatio); |
| |
| SDValue Zero = DAG.getConstant(0, DL, IntermediateType); |
| return DAG.getNode(Opcode, DL, IntermediateType, Zero, A, B); |
| } |
| |
| static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType, |
| unsigned &Opcode, SDValue &A, SDValue &B) { |
| SDValue Mul = N; |
| EVT AccType = Mul.getValueType(); // Vector input type after extension. |
| if (ScalarType != AccType.getVectorElementType()) |
| return false; |
| bool swap = false; |
| if (Mul->getOpcode() != ISD::MUL) |
| return false; |
| A = Mul->getOperand(0); |
| B = Mul->getOperand(1); |
| if (A.getOpcode() == ISD::ZERO_EXTEND) { |
| if (B.getOpcode() == ISD::ZERO_EXTEND) |
| Opcode = ISD::PARTIAL_REDUCE_UMLA; |
| else if (B.getOpcode() == ISD::SIGN_EXTEND) { |
| swap = true; |
| Opcode = ISD::PARTIAL_REDUCE_SUMLA; |
| } else |
| return false; |
| } else if (A.getOpcode() == ISD::SIGN_EXTEND) { |
| if (B.getOpcode() == ISD::ZERO_EXTEND) |
| Opcode = ISD::PARTIAL_REDUCE_SUMLA; |
| else if (B.getOpcode() == ISD::SIGN_EXTEND) |
| Opcode = ISD::PARTIAL_REDUCE_SMLA; |
| else |
| return false; |
| } else |
| return false; |
| |
| // Get multiplication arguments before extension. |
| A = A->getOperand(0); |
| B = B->getOperand(0); |
| if (A.getValueType() != B.getValueType()) |
| return false; |
| |
| if (swap) |
| std::swap(A, B); |
| |
| return true; |
| } |
| |
| SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N, |
| SelectionDAG &DAG) const { |
| if (!Subtarget.useHVXOps()) |
| return SDValue(); |
| |
| EVT ScalarType = N->getValueType(0); |
| unsigned Opcode; |
| SDValue A, B; |
| if (!DetectExtendingMultiply(N->getOperand(0), ScalarType, Opcode, A, B)) |
| return SDValue(); |
| |
| SDLoc DL(N); |
| unsigned RemainingReductionRatio; |
| SDValue Partial = |
| createExtendingPartialReduceMLA(Opcode, ScalarType, 1, A.getValueType(), |
| A, B, RemainingReductionRatio, DL, DAG); |
| if (!Partial) |
| return SDValue(); |
| |
| // We could have inserted a trivial MLA and rely on the folding action, |
| // similar to how vector_partial_reduce_add is lowered to an MLA in |
| // SelectionDAGBuilder. However, we just replace the final result since we |
| // have analyzed the input completely. |
| return DAG.getNode(ISD::VECREDUCE_ADD, DL, ScalarType, Partial); |
| } |
| |
| // Shared helper for VECREDUCE_FMIN/FMAX/FMINIMUM/FMAXIMUM on HVX float |
| // vector types. IgnoreNaN=true (FMIN/FMAX): NaN elements are replaced with |
| // the neutral value before the reduction so they don't corrupt the result |
| // even when the hardware pairwise instruction propagates NaN. |
| // IgnoreNaN=false (FMINIMUM/FMAXIMUM): NaN propagates naturally. |
| SDValue HexagonTargetLowering::LowerHvxVecReduceFMinMax( |
| SDValue Op, unsigned PairwiseOpc, bool IgnoreNaN, SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| SDValue Vec = Op.getOperand(0); |
| MVT VecTy = ty(Vec); |
| SDNodeFlags Flags = Op->getFlags(); |
| bool ShouldStripNaN = IgnoreNaN && !Flags.hasNoNaNs(); |
| |
| // Save original input before NaN stripping; needed for the all-NaN fixup. |
| SDValue OrigVec = Vec; |
| MVT OrigVecTy = VecTy; |
| |
| // Replace NaN elements in V with +/-Inf so the tree reduction ignores them. |
| bool IsMax = (Op.getOpcode() == ISD::VECREDUCE_FMAX); |
| auto ReplaceNaN = [&](SDValue V, MVT Ty) -> SDValue { |
| APFloat Neutral = APFloat::getInf( |
| Ty.getVectorElementType().getFltSemantics(), /*Negative=*/IsMax); |
| SDValue NeutralVec = DAG.getConstantFP(Neutral, DL, Ty); |
| EVT BoolTy = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), Ty); |
| SDValue IsNaN = DAG.getSetCC(DL, BoolTy, V, V, ISD::SETUO); |
| return DAG.getNode(ISD::VSELECT, DL, Ty, IsNaN, NeutralVec, V); |
| }; |
| |
| // For a pair vector, strip NaN from each half before the cross-half |
| // pairwise reduction so a NaN in one half can't corrupt the other. |
| if (isHvxPairTy(VecTy)) { |
| auto [Lo, Hi] = opSplit(Vec, DL, DAG); |
| MVT SingleTy = ty(Lo); |
| if (ShouldStripNaN) { |
| Lo = ReplaceNaN(Lo, SingleTy); |
| Hi = ReplaceNaN(Hi, SingleTy); |
| } |
| Vec = DAG.getNode(PairwiseOpc, DL, SingleTy, Lo, Hi, Flags); |
| VecTy = SingleTy; |
| } else if (ShouldStripNaN) { |
| Vec = ReplaceNaN(Vec, VecTy); |
| } |
| |
| // Tree reduction using VROR + pairwise op. Each iteration rotates the vector |
| // by half the remaining element count (in bytes) and takes element-wise |
| // min/max, halving the active width until element 0 holds the result. |
| unsigned ElemBytes = VecTy.getScalarSizeInBits() / 8; |
| unsigned HwLen = Subtarget.getVectorLength(); |
| unsigned NumElems = HwLen / ElemBytes; |
| |
| SDValue Curr = Vec; |
| for (unsigned Width = NumElems / 2; Width >= 1; Width /= 2) { |
| SDValue RotAmt = DAG.getConstant(Width * ElemBytes, DL, MVT::i32); |
| SDValue Rotated = DAG.getNode(HexagonISD::VROR, DL, VecTy, Curr, RotAmt); |
| Curr = DAG.getNode(PairwiseOpc, DL, VecTy, Curr, Rotated, Flags); |
| } |
| |
| // Extract element 0 as the scalar result. |
| MVT ScalarTy = Op.getSimpleValueType(); |
| SDValue Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarTy, Curr, |
| DAG.getConstant(0, DL, MVT::i32)); |
| |
| // Per llvm.maxnum/minnum semantics, all-NaN input must return NaN. The |
| // NaN-stripping above replaced every NaN with the neutral value, so an |
| // all-NaN vector produces the neutral value instead. Fix: if no element |
| // in the original vector was non-NaN, return NaN. |
| if (ShouldStripNaN) { |
| EVT BoolVecTy = |
| getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OrigVecTy); |
| // IsOrd[i] == 1 iff OrigVec[i] is not NaN. |
| SDValue IsOrd = DAG.getSetCC(DL, BoolVecTy, OrigVec, OrigVec, ISD::SETO); |
| // BoolVecTy is vNi1 with exactly N bits; bitcast to an integer and |
| // check != 0 to detect whether any element was non-NaN. |
| unsigned NumBits = BoolVecTy.getSizeInBits(); |
| MVT IntTy = MVT::getIntegerVT(NumBits); |
| SDValue IsOrdInt = DAG.getBitcast(IntTy, IsOrd); |
| SDValue AnyNonNaN = DAG.getSetCC(DL, MVT::i1, IsOrdInt, |
| DAG.getConstant(0, DL, IntTy), ISD::SETNE); |
| SDValue NaN = DAG.getConstantFP(APFloat::getNaN(ScalarTy.getFltSemantics()), |
| DL, ScalarTy); |
| Result = DAG.getSelect(DL, ScalarTy, AnyNonNaN, Result, NaN); |
| } |
| |
| return Result; |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxVecReduceFMin(SDValue Op, |
| SelectionDAG &DAG) const { |
| return LowerHvxVecReduceFMinMax(Op, ISD::FMINNUM, /*IgnoreNaN=*/true, DAG); |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxVecReduceFMax(SDValue Op, |
| SelectionDAG &DAG) const { |
| return LowerHvxVecReduceFMinMax(Op, ISD::FMAXNUM, /*IgnoreNaN=*/true, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxVecReduceFMinimum(SDValue Op, |
| SelectionDAG &DAG) const { |
| return LowerHvxVecReduceFMinMax(Op, ISD::FMINIMUM, /*IgnoreNaN=*/false, DAG); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxVecReduceFMaximum(SDValue Op, |
| SelectionDAG &DAG) const { |
| return LowerHvxVecReduceFMinMax(Op, ISD::FMAXIMUM, /*IgnoreNaN=*/false, DAG); |
| } |
| |
| // Lower FMINNUM/FMAXNUM on a single HVX float vector. These ops must ignore |
| // NaN (return the non-NaN operand). Because the hardware vmin/vmax may |
| // propagate NaN, replace NaN in each operand with the neutral value (+/-Inf) |
| // before delegating to FMINIMUM/FMAXIMUM. |
| SDValue HexagonTargetLowering::LowerHvxFMinNum(SDValue Op, |
| SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| auto A = Op.getOperand(0), B = Op.getOperand(1); |
| auto Ty = ty(Op); |
| auto Flags = Op->getFlags(); |
| |
| if (!Flags.hasNoNaNs()) { |
| auto &Sem = Ty.getVectorElementType().getFltSemantics(); |
| auto PosInf = DAG.getConstantFP(APFloat::getInf(Sem, false), DL, Ty); |
| auto BoolTy = |
| getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), Ty); |
| auto IsNaN_A = DAG.getSetCC(DL, BoolTy, A, A, ISD::SETUO); |
| auto IsNaN_B = DAG.getSetCC(DL, BoolTy, B, B, ISD::SETUO); |
| // Per llvm.minnum: if both operands are NaN, return NaN. Replace NaN |
| // with +Inf so the hardware min ignores single-operand NaN, then restore |
| // NaN for lanes where both inputs were NaN. |
| auto BothNaN = DAG.getNode(ISD::AND, DL, BoolTy, IsNaN_A, IsNaN_B); |
| A = DAG.getNode(ISD::VSELECT, DL, Ty, IsNaN_A, PosInf, A); |
| B = DAG.getNode(ISD::VSELECT, DL, Ty, IsNaN_B, PosInf, B); |
| auto Res = DAG.getNode(ISD::FMINIMUM, DL, Ty, A, B, Flags); |
| auto NaN = DAG.getConstantFP(APFloat::getNaN(Sem), DL, Ty); |
| return DAG.getNode(ISD::VSELECT, DL, Ty, BothNaN, NaN, Res); |
| } |
| return DAG.getNode(ISD::FMINIMUM, DL, Ty, A, B, Flags); |
| } |
| |
| SDValue HexagonTargetLowering::LowerHvxFMaxNum(SDValue Op, |
| SelectionDAG &DAG) const { |
| SDLoc DL(Op); |
| auto A = Op.getOperand(0), B = Op.getOperand(1); |
| auto Ty = ty(Op); |
| auto Flags = Op->getFlags(); |
| |
| if (!Flags.hasNoNaNs()) { |
| auto &Sem = Ty.getVectorElementType().getFltSemantics(); |
| auto NegInf = DAG.getConstantFP(APFloat::getInf(Sem, true), DL, Ty); |
| auto BoolTy = |
| getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), Ty); |
| auto IsNaN_A = DAG.getSetCC(DL, BoolTy, A, A, ISD::SETUO); |
| auto IsNaN_B = DAG.getSetCC(DL, BoolTy, B, B, ISD::SETUO); |
| // Per llvm.maxnum: if both operands are NaN, return NaN. Replace NaN |
| // with -Inf so the hardware max ignores single-operand NaN, then restore |
| // NaN for lanes where both inputs were NaN. |
| auto BothNaN = DAG.getNode(ISD::AND, DL, BoolTy, IsNaN_A, IsNaN_B); |
| A = DAG.getNode(ISD::VSELECT, DL, Ty, IsNaN_A, NegInf, A); |
| B = DAG.getNode(ISD::VSELECT, DL, Ty, IsNaN_B, NegInf, B); |
| auto Res = DAG.getNode(ISD::FMAXIMUM, DL, Ty, A, B, Flags); |
| auto NaN = DAG.getConstantFP(APFloat::getNaN(Sem), DL, Ty); |
| return DAG.getNode(ISD::VSELECT, DL, Ty, BothNaN, NaN, Res); |
| } |
| return DAG.getNode(ISD::FMAXIMUM, DL, Ty, A, B, Flags); |
| } |
| |
| // When possible, separate an MLA reduction with extended operands but |
| // unsupported reduction factor into an extending partial reduction that |
| // can be efficiently lowered, and a follow-up partial reduction. |
| // partial_reduce_mla(a, x, y) -> |
| // partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1) |
| SDValue |
| HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N, |
| SelectionDAG &DAG) const { |
| if (!Subtarget.useHVXOps()) |
| return SDValue(); |
| |
| SDValue Acc = N->getOperand(0); |
| SDValue A = N->getOperand(1); |
| SDValue B = N->getOperand(2); |
| if (A.getValueType() != B.getValueType()) |
| return SDValue(); |
| |
| // The types should be declared as custom, but do not split already legal |
| // operation. |
| EVT AccType = Acc.getValueType(); |
| EVT InputType = A.getValueType(); |
| if (getPartialReduceMLAAction(N->getOpcode(), AccType, InputType) != Custom) |
| return SDValue(); |
| |
| SDLoc DL(N); |
| unsigned RemainingReductionRatio; |
| SDValue Partial = createExtendingPartialReduceMLA( |
| N->getOpcode(), AccType.getVectorElementType(), |
| AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio, |
| DL, DAG); |
| if (!Partial) |
| return SDValue(); |
| assert(RemainingReductionRatio <= MaxExpandMLA); |
| |
| // Create the reduction for the remaining ratio. |
| EVT IntermediateType = Partial->getOperand(0).getValueType(); |
| SDValue One = DAG.getConstant(1, DL, IntermediateType); |
| return DAG.getNode(N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA |
| ? ISD::PARTIAL_REDUCE_UMLA |
| : ISD::PARTIAL_REDUCE_SUMLA, |
| DL, AccType, Acc, Partial, One); |
| } |
| |
| SDValue |
| HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op, |
| SelectionDAG &DAG) const { |
| const SDLoc &DL(Op); |
| SDValue Acc = Op.getOperand(0); |
| SDValue A = Op.getOperand(1); |
| SDValue B = Op.getOperand(2); |
| |
| // Split the input vectors into units of one HVX vector length. |
| unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8; |
| |
| EVT AccType = Acc.getValueType(); |
| EVT AccEltType = AccType.getVectorElementType(); |
| unsigned AccSubvectorNumElements = |
| HwVectorSizeInBits / AccEltType.getSizeInBits(); |
| EVT AccSubvectorType = |
| EVT::getVectorVT(*DAG.getContext(), AccEltType, AccSubvectorNumElements); |
| |
| EVT InputType = A.getValueType(); |
| assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0); |
| EVT InputEltType = InputType.getVectorElementType(); |
| unsigned InputSubvectorNumElements = |
| HwVectorSizeInBits / InputEltType.getSizeInBits(); |
| EVT InputSubvectorType = EVT::getVectorVT(*DAG.getContext(), InputEltType, |
| InputSubvectorNumElements); |
| |
| unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits; |
| SmallVector<SDValue, MaxExpandMLA> Subvectors; |
| |
| for (unsigned I = 0; I != SubvectorNum; ++I) { |
| SDValue SubvectorAcc = DAG.getExtractSubvector(DL, AccSubvectorType, Acc, |
| I * AccSubvectorNumElements); |
| SDValue SubvectorA = DAG.getExtractSubvector(DL, InputSubvectorType, A, |
| I * InputSubvectorNumElements); |
| SDValue SubvectorB = DAG.getExtractSubvector(DL, InputSubvectorType, B, |
| I * InputSubvectorNumElements); |
| SDValue SubvectorMLA = DAG.getNode(Op.getOpcode(), DL, AccSubvectorType, |
| SubvectorAcc, SubvectorA, SubvectorB); |
| Subvectors.push_back(SubvectorMLA); |
| } |
| |
| return DAG.getNode(ISD::CONCAT_VECTORS, DL, AccType, Subvectors); |
| } |
| |
| // Lower fcmp oeq on HVX float vectors for architectures before v81, which |
| // lack a dedicated floating-point equality instruction. |
| // |
| // Correct IEEE-754 semantics: oeq(a,b) is true iff a==b and neither is NaN. |
| // We use the available float-GT instruction (V6_vgtsf/V6_vgthf) for the |
| // inequality check and bit manipulation for NaN detection: |
| // |
| // oeq(a, b) = NOT(ogt(a,b) OR ogt(b,a) OR isNaN(a) OR isNaN(b)) |
| // |
| // where isNaN(x) = ((int_bits(x) & AbsMask) > NaNThreshold) |
| // f32: AbsMask=0x7FFFFFFF, NaNThreshold=0x7F800000 |
| // f16: AbsMask=0x7FFF, NaNThreshold=0x7C00 |
| // |
| // This handles +0/-0 correctly because float-GT treats them as equal, so |
| // neither ogt(+0,-0) nor ogt(-0,+0) is ever true. |
| // |
| // Example f32 assembly (no NaNs case): |
| // q0 = vcmp.eq(v0.w, v1.w) // bitwise comparison should just work |
| // |
| // Example f32 assembly (NaN-present case): |
| // q0 = vcmp.gt(v0.sf, v1.sf) // ogt(a,b) |
| // q0 |= vcmp.gt(v1.sf, v0.sf) // |= ogt(b,a) |
| // r0 = ##0x7FFFFFFF |
| // v2 = vsplat(r0) // AbsMask broadcast |
| // r1 = ##0x7F800000 |
| // v3 = vsplat(r1) // NaNThresh broadcast |
| // v4 = vand(v0, v2) // int_bits(a) & AbsMask |
| // v5 = vand(v1, v2) // int_bits(b) & AbsMask |
| // q0 |= vcmp.gt(v4.w, v3.w) // |= isNaN(a) |
| // q0 |= vcmp.gt(v5.w, v3.w) // |= isNaN(b) |
| // // q0 now holds AnyFalse; result = XOR(q0, allones) = oeq |
| SDValue HexagonTargetLowering::LowerHvxFpSetoeq(SDValue Op, |
| SelectionDAG &DAG) const { |
| auto ResTy = ty(Op); |
| auto A = Op.getOperand(0), B = Op->getOperand(1); |
| MVT FloatTy = ty(A); |
| MVT ElemTy = FloatTy.getVectorElementType(); |
| bool IsF32 = (ElemTy == MVT::f32); |
| if (!IsF32) { |
| assert((ElemTy == MVT::f16)); |
| } |
| const SDLoc &DL(Op); |
| MVT IntElemTy = IsF32 ? MVT::i32 : MVT::i16; |
| MVT IntVecTy = tyVector(FloatTy, IntElemTy); |
| |
| // Under nnan semantics NaN cannot appear, so integer equality is both |
| // correct and cheaper (one instruction vs the float-GT sequence). |
| bool NoNaN = Op->getFlags().hasNoNaNs(); |
| if (NoNaN) { |
| SDValue IA = DAG.getNode(ISD::BITCAST, DL, IntVecTy, A); |
| SDValue IB = DAG.getNode(ISD::BITCAST, DL, IntVecTy, B); |
| return DAG.getSetCC(DL, ResTy, IA, IB, ISD::SETEQ); |
| } |
| |
| // Float GT comparisons (IEEE-754: false whenever either operand is NaN). |
| SDValue QAgtB = DAG.getSetCC(DL, ResTy, A, B, ISD::SETOGT); |
| SDValue QBgtA = DAG.getSetCC(DL, ResTy, B, A, ISD::SETOGT); |
| |
| // OR all "false" conditions together, then invert. |
| SDValue AnyFalse = DAG.getNode(ISD::OR, DL, ResTy, QAgtB, QBgtA); |
| |
| // Detect NaN by checking whether the unbiased exponent/mantissa field |
| // exceeds the largest finite value. |
| // f32: (bits & 0x7FFFFFFF) > 0x7F800000 |
| // f16: (bits & 0x7FFF) > 0x7C00 |
| uint64_t AbsMask = IsF32 ? 0x7FFFFFFFull : 0x7FFFull; |
| uint64_t NaNThresh = IsF32 ? 0x7F800000ull : 0x7C00ull; |
| |
| SDValue IA = DAG.getNode(ISD::BITCAST, DL, IntVecTy, A); |
| SDValue IB = DAG.getNode(ISD::BITCAST, DL, IntVecTy, B); |
| SDValue MaskVec = DAG.getConstant(AbsMask, DL, IntVecTy); |
| SDValue ThreshVec = DAG.getConstant(NaNThresh, DL, IntVecTy); |
| SDValue QNanA = |
| DAG.getSetCC(DL, ResTy, DAG.getNode(ISD::AND, DL, IntVecTy, IA, MaskVec), |
| ThreshVec, ISD::SETGT); |
| SDValue QNanB = |
| DAG.getSetCC(DL, ResTy, DAG.getNode(ISD::AND, DL, IntVecTy, IB, MaskVec), |
| ThreshVec, ISD::SETGT); |
| AnyFalse = DAG.getNode(ISD::OR, DL, ResTy, AnyFalse, QNanA); |
| AnyFalse = DAG.getNode(ISD::OR, DL, ResTy, AnyFalse, QNanB); |
| |
| // Result = NOT(<Is A gt B>, <IS B gt A>, <IS A NaN>, <IS B NaN>) |
| // Use XOR with ones to simulate logical not. |
| return DAG.getNode(ISD::XOR, DL, ResTy, AnyFalse, |
| DAG.getConstant(1, DL, ResTy)); |
| } |
| |
| SDValue |
| HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) |
| const { |
| const SDLoc &dl(N); |
| SelectionDAG &DAG = DCI.DAG; |
| SDValue Op(N, 0); |
| unsigned Opc = Op.getOpcode(); |
| |
| SmallVector<SDValue, 4> Ops(N->ops()); |
| |
| if (Opc == ISD::TRUNCATE) |
| return combineTruncateBeforeLegal(Op, DCI); |
| if (Opc == ISD::CONCAT_VECTORS) |
| return combineConcatVectorsBeforeLegal(Op, DCI); |
| |
| if (DCI.isBeforeLegalizeOps()) |
| return SDValue(); |
| |
| switch (Opc) { |
| case HexagonISD::V2Q: |
| if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) { |
| if (const auto *C = dyn_cast<ConstantSDNode>(Ops[0].getOperand(0))) |
| return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op)) |
| : DAG.getNode(HexagonISD::QTRUE, dl, ty(Op)); |
| } |
| break; |
| case HexagonISD::Q2V: |
| if (Ops[0].getOpcode() == HexagonISD::QTRUE) |
| return DAG.getNode(ISD::SPLAT_VECTOR, dl, ty(Op), |
| DAG.getAllOnesConstant(dl, MVT::i32)); |
| if (Ops[0].getOpcode() == HexagonISD::QFALSE) |
| return getZero(dl, ty(Op), DAG); |
| break; |
| case HexagonISD::VINSERTW0: |
| if (isUndef(Ops[1])) |
| return Ops[0]; |
| break; |
| case HexagonISD::VROR: { |
| if (Ops[0].getOpcode() == HexagonISD::VROR) { |
| SDValue Vec = Ops[0].getOperand(0); |
| SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(1); |
| SDValue Rot = DAG.getNode(ISD::ADD, dl, ty(Rot0), {Rot0, Rot1}); |
| return DAG.getNode(HexagonISD::VROR, dl, ty(Op), {Vec, Rot}); |
| } |
| break; |
| } |
| } |
| |
| return SDValue(); |
| } |
| |
| bool |
| HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const { |
| if (Subtarget.isHVXVectorType(Ty, true)) |
| return false; |
| auto Action = getPreferredHvxVectorAction(Ty); |
| if (Action == TargetLoweringBase::TypeSplitVector) |
| return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true); |
| return false; |
| } |
| |
| bool |
| HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const { |
| if (Subtarget.isHVXVectorType(Ty, true)) |
| return false; |
| auto Action = getPreferredHvxVectorAction(Ty); |
| if (Action == TargetLoweringBase::TypeWidenVector) |
| return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true); |
| return false; |
| } |
| |
| bool |
| HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const { |
| if (!Subtarget.useHVXOps()) |
| return false; |
| // If the type of any result, or any operand type are HVX vector types, |
| // this is an HVX operation. |
| auto IsHvxTy = [this](EVT Ty) { |
| return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true); |
| }; |
| auto IsHvxOp = [this](SDValue Op) { |
| return Op.getValueType().isSimple() && |
| Subtarget.isHVXVectorType(ty(Op), true); |
| }; |
| if (llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp)) |
| return true; |
| |
| // Check if this could be an HVX operation after type widening. |
| auto IsWidenedToHvx = [this, &DAG](SDValue Op) { |
| if (!Op.getValueType().isSimple()) |
| return false; |
| MVT ValTy = ty(Op); |
| return ValTy.isVector() && shouldWidenToHvx(ValTy, DAG); |
| }; |
| |
| for (int i = 0, e = N->getNumValues(); i != e; ++i) { |
| if (IsWidenedToHvx(SDValue(N, i))) |
| return true; |
| } |
| return llvm::any_of(N->ops(), IsWidenedToHvx); |
| } |
