| //===- NeonEmitter.cpp - Generate arm_neon.h for use with clang -*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This tablegen backend is responsible for emitting arm_neon.h, which includes |
| // a declaration and definition of each function specified by the ARM NEON |
| // compiler interface. See ARM document DUI0348B. |
| // |
| // Each NEON instruction is implemented in terms of 1 or more functions which |
| // are suffixed with the element type of the input vectors. Functions may be |
| // implemented in terms of generic vector operations such as +, *, -, etc. or |
| // by calling a __builtin_-prefixed function which will be handled by clang's |
| // CodeGen library. |
| // |
| // Additional validation code can be generated by this file when runHeader() is |
| // called, rather than the normal run() entry point. |
| // |
| // See also the documentation in include/clang/Basic/arm_neon.td. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "TableGenBackends.h" |
| #include "llvm/ADT/ArrayRef.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/ADT/None.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/StringExtras.h" |
| #include "llvm/ADT/StringRef.h" |
| #include "llvm/Support/Casting.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/raw_ostream.h" |
| #include "llvm/TableGen/Error.h" |
| #include "llvm/TableGen/Record.h" |
| #include "llvm/TableGen/SetTheory.h" |
| #include <algorithm> |
| #include <cassert> |
| #include <cctype> |
| #include <cstddef> |
| #include <cstdint> |
| #include <deque> |
| #include <map> |
| #include <set> |
| #include <sstream> |
| #include <string> |
| #include <utility> |
| #include <vector> |
| |
| using namespace llvm; |
| |
| namespace { |
| |
| // While globals are generally bad, this one allows us to perform assertions |
| // liberally and somehow still trace them back to the def they indirectly |
| // came from. |
| static Record *CurrentRecord = nullptr; |
| static void assert_with_loc(bool Assertion, const std::string &Str) { |
| if (!Assertion) { |
| if (CurrentRecord) |
| PrintFatalError(CurrentRecord->getLoc(), Str); |
| else |
| PrintFatalError(Str); |
| } |
| } |
| |
| enum ClassKind { |
| ClassNone, |
| ClassI, // generic integer instruction, e.g., "i8" suffix |
| ClassS, // signed/unsigned/poly, e.g., "s8", "u8" or "p8" suffix |
| ClassW, // width-specific instruction, e.g., "8" suffix |
| ClassB, // bitcast arguments with enum argument to specify type |
| ClassL, // Logical instructions which are op instructions |
| // but we need to not emit any suffix for in our |
| // tests. |
| ClassNoTest // Instructions which we do not test since they are |
| // not TRUE instructions. |
| }; |
| |
| /// NeonTypeFlags - Flags to identify the types for overloaded Neon |
| /// builtins. These must be kept in sync with the flags in |
| /// include/clang/Basic/TargetBuiltins.h. |
| namespace NeonTypeFlags { |
| |
| enum { EltTypeMask = 0xf, UnsignedFlag = 0x10, QuadFlag = 0x20 }; |
| |
| enum EltType { |
| Int8, |
| Int16, |
| Int32, |
| Int64, |
| Poly8, |
| Poly16, |
| Poly64, |
| Poly128, |
| Float16, |
| Float32, |
| Float64 |
| }; |
| |
| } // end namespace NeonTypeFlags |
| |
| class NeonEmitter; |
| |
| //===----------------------------------------------------------------------===// |
| // TypeSpec |
| //===----------------------------------------------------------------------===// |
| |
| /// A TypeSpec is just a simple wrapper around a string, but gets its own type |
| /// for strong typing purposes. |
| /// |
| /// A TypeSpec can be used to create a type. |
| class TypeSpec : public std::string { |
| public: |
| static std::vector<TypeSpec> fromTypeSpecs(StringRef Str) { |
| std::vector<TypeSpec> Ret; |
| TypeSpec Acc; |
| for (char I : Str.str()) { |
| if (islower(I)) { |
| Acc.push_back(I); |
| Ret.push_back(TypeSpec(Acc)); |
| Acc.clear(); |
| } else { |
| Acc.push_back(I); |
| } |
| } |
| return Ret; |
| } |
| }; |
| |
| //===----------------------------------------------------------------------===// |
| // Type |
| //===----------------------------------------------------------------------===// |
| |
| /// A Type. Not much more to say here. |
| class Type { |
| private: |
| TypeSpec TS; |
| |
| bool Float, Signed, Immediate, Void, Poly, Constant, Pointer; |
| // ScalarForMangling and NoManglingQ are really not suited to live here as |
| // they are not related to the type. But they live in the TypeSpec (not the |
| // prototype), so this is really the only place to store them. |
| bool ScalarForMangling, NoManglingQ; |
| unsigned Bitwidth, ElementBitwidth, NumVectors; |
| |
| public: |
| Type() |
| : Float(false), Signed(false), Immediate(false), Void(true), Poly(false), |
| Constant(false), Pointer(false), ScalarForMangling(false), |
| NoManglingQ(false), Bitwidth(0), ElementBitwidth(0), NumVectors(0) {} |
| |
| Type(TypeSpec TS, char CharMod) |
| : TS(std::move(TS)), Float(false), Signed(false), Immediate(false), |
| Void(false), Poly(false), Constant(false), Pointer(false), |
| ScalarForMangling(false), NoManglingQ(false), Bitwidth(0), |
| ElementBitwidth(0), NumVectors(0) { |
| applyModifier(CharMod); |
| } |
| |
| /// Returns a type representing "void". |
| static Type getVoid() { return Type(); } |
| |
| bool operator==(const Type &Other) const { return str() == Other.str(); } |
| bool operator!=(const Type &Other) const { return !operator==(Other); } |
| |
| // |
| // Query functions |
| // |
| bool isScalarForMangling() const { return ScalarForMangling; } |
| bool noManglingQ() const { return NoManglingQ; } |
| |
| bool isPointer() const { return Pointer; } |
| bool isFloating() const { return Float; } |
| bool isInteger() const { return !Float && !Poly; } |
| bool isSigned() const { return Signed; } |
| bool isImmediate() const { return Immediate; } |
| bool isScalar() const { return NumVectors == 0; } |
| bool isVector() const { return NumVectors > 0; } |
| bool isFloat() const { return Float && ElementBitwidth == 32; } |
| bool isDouble() const { return Float && ElementBitwidth == 64; } |
| bool isHalf() const { return Float && ElementBitwidth == 16; } |
| bool isPoly() const { return Poly; } |
| bool isChar() const { return ElementBitwidth == 8; } |
| bool isShort() const { return !Float && ElementBitwidth == 16; } |
| bool isInt() const { return !Float && ElementBitwidth == 32; } |
| bool isLong() const { return !Float && ElementBitwidth == 64; } |
| bool isVoid() const { return Void; } |
| unsigned getNumElements() const { return Bitwidth / ElementBitwidth; } |
| unsigned getSizeInBits() const { return Bitwidth; } |
| unsigned getElementSizeInBits() const { return ElementBitwidth; } |
| unsigned getNumVectors() const { return NumVectors; } |
| |
| // |
| // Mutator functions |
| // |
| void makeUnsigned() { Signed = false; } |
| void makeSigned() { Signed = true; } |
| |
| void makeInteger(unsigned ElemWidth, bool Sign) { |
| Float = false; |
| Poly = false; |
| Signed = Sign; |
| Immediate = false; |
| ElementBitwidth = ElemWidth; |
| } |
| |
| void makeImmediate(unsigned ElemWidth) { |
| Float = false; |
| Poly = false; |
| Signed = true; |
| Immediate = true; |
| ElementBitwidth = ElemWidth; |
| } |
| |
| void makeScalar() { |
| Bitwidth = ElementBitwidth; |
| NumVectors = 0; |
| } |
| |
| void makeOneVector() { |
| assert(isVector()); |
| NumVectors = 1; |
| } |
| |
| void doubleLanes() { |
| assert_with_loc(Bitwidth != 128, "Can't get bigger than 128!"); |
| Bitwidth = 128; |
| } |
| |
| void halveLanes() { |
| assert_with_loc(Bitwidth != 64, "Can't get smaller than 64!"); |
| Bitwidth = 64; |
| } |
| |
| /// Return the C string representation of a type, which is the typename |
| /// defined in stdint.h or arm_neon.h. |
| std::string str() const; |
| |
| /// Return the string representation of a type, which is an encoded |
| /// string for passing to the BUILTIN() macro in Builtins.def. |
| std::string builtin_str() const; |
| |
| /// Return the value in NeonTypeFlags for this type. |
| unsigned getNeonEnum() const; |
| |
| /// Parse a type from a stdint.h or arm_neon.h typedef name, |
| /// for example uint32x2_t or int64_t. |
| static Type fromTypedefName(StringRef Name); |
| |
| private: |
| /// Creates the type based on the typespec string in TS. |
| /// Sets "Quad" to true if the "Q" or "H" modifiers were |
| /// seen. This is needed by applyModifier as some modifiers |
| /// only take effect if the type size was changed by "Q" or "H". |
| void applyTypespec(bool &Quad); |
| /// Applies a prototype modifier to the type. |
| void applyModifier(char Mod); |
| }; |
| |
| //===----------------------------------------------------------------------===// |
| // Variable |
| //===----------------------------------------------------------------------===// |
| |
| /// A variable is a simple class that just has a type and a name. |
| class Variable { |
| Type T; |
| std::string N; |
| |
| public: |
| Variable() : T(Type::getVoid()), N("") {} |
| Variable(Type T, std::string N) : T(std::move(T)), N(std::move(N)) {} |
| |
| Type getType() const { return T; } |
| std::string getName() const { return "__" + N; } |
| }; |
| |
| //===----------------------------------------------------------------------===// |
| // Intrinsic |
| //===----------------------------------------------------------------------===// |
| |
| /// The main grunt class. This represents an instantiation of an intrinsic with |
| /// a particular typespec and prototype. |
| class Intrinsic { |
| friend class DagEmitter; |
| |
| /// The Record this intrinsic was created from. |
| Record *R; |
| /// The unmangled name and prototype. |
| std::string Name, Proto; |
| /// The input and output typespecs. InTS == OutTS except when |
| /// CartesianProductOfTypes is 1 - this is the case for vreinterpret. |
| TypeSpec OutTS, InTS; |
| /// The base class kind. Most intrinsics use ClassS, which has full type |
| /// info for integers (s32/u32). Some use ClassI, which doesn't care about |
| /// signedness (i32), while some (ClassB) have no type at all, only a width |
| /// (32). |
| ClassKind CK; |
| /// The list of DAGs for the body. May be empty, in which case we should |
| /// emit a builtin call. |
| ListInit *Body; |
| /// The architectural #ifdef guard. |
| std::string Guard; |
| /// Set if the Unavailable bit is 1. This means we don't generate a body, |
| /// just an "unavailable" attribute on a declaration. |
| bool IsUnavailable; |
| /// Is this intrinsic safe for big-endian? or does it need its arguments |
| /// reversing? |
| bool BigEndianSafe; |
| |
| /// The types of return value [0] and parameters [1..]. |
| std::vector<Type> Types; |
| /// The local variables defined. |
| std::map<std::string, Variable> Variables; |
| /// NeededEarly - set if any other intrinsic depends on this intrinsic. |
| bool NeededEarly; |
| /// UseMacro - set if we should implement using a macro or unset for a |
| /// function. |
| bool UseMacro; |
| /// The set of intrinsics that this intrinsic uses/requires. |
| std::set<Intrinsic *> Dependencies; |
| /// The "base type", which is Type('d', OutTS). InBaseType is only |
| /// different if CartesianProductOfTypes = 1 (for vreinterpret). |
| Type BaseType, InBaseType; |
| /// The return variable. |
| Variable RetVar; |
| /// A postfix to apply to every variable. Defaults to "". |
| std::string VariablePostfix; |
| |
| NeonEmitter &Emitter; |
| std::stringstream OS; |
| |
| bool isBigEndianSafe() const { |
| if (BigEndianSafe) |
| return true; |
| |
| for (const auto &T : Types){ |
| if (T.isVector() && T.getNumElements() > 1) |
| return false; |
| } |
| return true; |
| } |
| |
| public: |
| Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS, |
| TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter, |
| StringRef Guard, bool IsUnavailable, bool BigEndianSafe) |
| : R(R), Name(Name.str()), Proto(Proto.str()), OutTS(OutTS), InTS(InTS), |
| CK(CK), Body(Body), Guard(Guard.str()), IsUnavailable(IsUnavailable), |
| BigEndianSafe(BigEndianSafe), NeededEarly(false), UseMacro(false), |
| BaseType(OutTS, 'd'), InBaseType(InTS, 'd'), Emitter(Emitter) { |
| // If this builtin takes an immediate argument, we need to #define it rather |
| // than use a standard declaration, so that SemaChecking can range check |
| // the immediate passed by the user. |
| if (Proto.find('i') != std::string::npos) |
| UseMacro = true; |
| |
| // Pointer arguments need to use macros to avoid hiding aligned attributes |
| // from the pointer type. |
| if (Proto.find('p') != std::string::npos || |
| Proto.find('c') != std::string::npos) |
| UseMacro = true; |
| |
| // It is not permitted to pass or return an __fp16 by value, so intrinsics |
| // taking a scalar float16_t must be implemented as macros. |
| if (OutTS.find('h') != std::string::npos && |
| Proto.find('s') != std::string::npos) |
| UseMacro = true; |
| |
| // Modify the TypeSpec per-argument to get a concrete Type, and create |
| // known variables for each. |
| // Types[0] is the return value. |
| Types.emplace_back(OutTS, Proto[0]); |
| for (unsigned I = 1; I < Proto.size(); ++I) |
| Types.emplace_back(InTS, Proto[I]); |
| } |
| |
| /// Get the Record that this intrinsic is based off. |
| Record *getRecord() const { return R; } |
| /// Get the set of Intrinsics that this intrinsic calls. |
| /// this is the set of immediate dependencies, NOT the |
| /// transitive closure. |
| const std::set<Intrinsic *> &getDependencies() const { return Dependencies; } |
| /// Get the architectural guard string (#ifdef). |
| std::string getGuard() const { return Guard; } |
| /// Get the non-mangled name. |
| std::string getName() const { return Name; } |
| |
| /// Return true if the intrinsic takes an immediate operand. |
| bool hasImmediate() const { |
| return Proto.find('i') != std::string::npos; |
| } |
| |
| /// Return the parameter index of the immediate operand. |
| unsigned getImmediateIdx() const { |
| assert(hasImmediate()); |
| unsigned Idx = Proto.find('i'); |
| assert(Idx > 0 && "Can't return an immediate!"); |
| return Idx - 1; |
| } |
| |
| /// Return true if the intrinsic takes an splat operand. |
| bool hasSplat() const { return Proto.find('a') != std::string::npos; } |
| |
| /// Return the parameter index of the splat operand. |
| unsigned getSplatIdx() const { |
| assert(hasSplat()); |
| unsigned Idx = Proto.find('a'); |
| assert(Idx > 0 && "Can't return a splat!"); |
| return Idx - 1; |
| } |
| |
| unsigned getNumParams() const { return Proto.size() - 1; } |
| Type getReturnType() const { return Types[0]; } |
| Type getParamType(unsigned I) const { return Types[I + 1]; } |
| Type getBaseType() const { return BaseType; } |
| /// Return the raw prototype string. |
| std::string getProto() const { return Proto; } |
| |
| /// Return true if the prototype has a scalar argument. |
| /// This does not return true for the "splat" code ('a'). |
| bool protoHasScalar() const; |
| |
| /// Return the index that parameter PIndex will sit at |
| /// in a generated function call. This is often just PIndex, |
| /// but may not be as things such as multiple-vector operands |
| /// and sret parameters need to be taken into accont. |
| unsigned getGeneratedParamIdx(unsigned PIndex) { |
| unsigned Idx = 0; |
| if (getReturnType().getNumVectors() > 1) |
| // Multiple vectors are passed as sret. |
| ++Idx; |
| |
| for (unsigned I = 0; I < PIndex; ++I) |
| Idx += std::max(1U, getParamType(I).getNumVectors()); |
| |
| return Idx; |
| } |
| |
| bool hasBody() const { return Body && !Body->getValues().empty(); } |
| |
| void setNeededEarly() { NeededEarly = true; } |
| |
| bool operator<(const Intrinsic &Other) const { |
| // Sort lexicographically on a two-tuple (Guard, Name) |
| if (Guard != Other.Guard) |
| return Guard < Other.Guard; |
| return Name < Other.Name; |
| } |
| |
| ClassKind getClassKind(bool UseClassBIfScalar = false) { |
| if (UseClassBIfScalar && !protoHasScalar()) |
| return ClassB; |
| return CK; |
| } |
| |
| /// Return the name, mangled with type information. |
| /// If ForceClassS is true, use ClassS (u32/s32) instead |
| /// of the intrinsic's own type class. |
| std::string getMangledName(bool ForceClassS = false) const; |
| /// Return the type code for a builtin function call. |
| std::string getInstTypeCode(Type T, ClassKind CK) const; |
| /// Return the type string for a BUILTIN() macro in Builtins.def. |
| std::string getBuiltinTypeStr(); |
| |
| /// Generate the intrinsic, returning code. |
| std::string generate(); |
| /// Perform type checking and populate the dependency graph, but |
| /// don't generate code yet. |
| void indexBody(); |
| |
| private: |
| std::string mangleName(std::string Name, ClassKind CK) const; |
| |
| void initVariables(); |
| std::string replaceParamsIn(std::string S); |
| |
| void emitBodyAsBuiltinCall(); |
| |
| void generateImpl(bool ReverseArguments, |
| StringRef NamePrefix, StringRef CallPrefix); |
| void emitReturn(); |
| void emitBody(StringRef CallPrefix); |
| void emitShadowedArgs(); |
| void emitArgumentReversal(); |
| void emitReturnReversal(); |
| void emitReverseVariable(Variable &Dest, Variable &Src); |
| void emitNewLine(); |
| void emitClosingBrace(); |
| void emitOpeningBrace(); |
| void emitPrototype(StringRef NamePrefix); |
| |
| class DagEmitter { |
| Intrinsic &Intr; |
| StringRef CallPrefix; |
| |
| public: |
| DagEmitter(Intrinsic &Intr, StringRef CallPrefix) : |
| Intr(Intr), CallPrefix(CallPrefix) { |
| } |
| std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName); |
| std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI); |
| std::pair<Type, std::string> emitDagSplat(DagInit *DI); |
| std::pair<Type, std::string> emitDagDup(DagInit *DI); |
| std::pair<Type, std::string> emitDagDupTyped(DagInit *DI); |
| std::pair<Type, std::string> emitDagShuffle(DagInit *DI); |
| std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast); |
| std::pair<Type, std::string> emitDagCall(DagInit *DI); |
| std::pair<Type, std::string> emitDagNameReplace(DagInit *DI); |
| std::pair<Type, std::string> emitDagLiteral(DagInit *DI); |
| std::pair<Type, std::string> emitDagOp(DagInit *DI); |
| std::pair<Type, std::string> emitDag(DagInit *DI); |
| }; |
| }; |
| |
| //===----------------------------------------------------------------------===// |
| // NeonEmitter |
| //===----------------------------------------------------------------------===// |
| |
| class NeonEmitter { |
| RecordKeeper &Records; |
| DenseMap<Record *, ClassKind> ClassMap; |
| std::map<std::string, std::deque<Intrinsic>> IntrinsicMap; |
| unsigned UniqueNumber; |
| |
| void createIntrinsic(Record *R, SmallVectorImpl<Intrinsic *> &Out); |
| void genBuiltinsDef(raw_ostream &OS, SmallVectorImpl<Intrinsic *> &Defs); |
| void genOverloadTypeCheckCode(raw_ostream &OS, |
| SmallVectorImpl<Intrinsic *> &Defs); |
| void genIntrinsicRangeCheckCode(raw_ostream &OS, |
| SmallVectorImpl<Intrinsic *> &Defs); |
| |
| public: |
| /// Called by Intrinsic - this attempts to get an intrinsic that takes |
| /// the given types as arguments. |
| Intrinsic &getIntrinsic(StringRef Name, ArrayRef<Type> Types); |
| |
| /// Called by Intrinsic - returns a globally-unique number. |
| unsigned getUniqueNumber() { return UniqueNumber++; } |
| |
| NeonEmitter(RecordKeeper &R) : Records(R), UniqueNumber(0) { |
| Record *SI = R.getClass("SInst"); |
| Record *II = R.getClass("IInst"); |
| Record *WI = R.getClass("WInst"); |
| Record *SOpI = R.getClass("SOpInst"); |
| Record *IOpI = R.getClass("IOpInst"); |
| Record *WOpI = R.getClass("WOpInst"); |
| Record *LOpI = R.getClass("LOpInst"); |
| Record *NoTestOpI = R.getClass("NoTestOpInst"); |
| |
| ClassMap[SI] = ClassS; |
| ClassMap[II] = ClassI; |
| ClassMap[WI] = ClassW; |
| ClassMap[SOpI] = ClassS; |
| ClassMap[IOpI] = ClassI; |
| ClassMap[WOpI] = ClassW; |
| ClassMap[LOpI] = ClassL; |
| ClassMap[NoTestOpI] = ClassNoTest; |
| } |
| |
| // run - Emit arm_neon.h.inc |
| void run(raw_ostream &o); |
| |
| // runFP16 - Emit arm_fp16.h.inc |
| void runFP16(raw_ostream &o); |
| |
| // runHeader - Emit all the __builtin prototypes used in arm_neon.h |
| // and arm_fp16.h |
| void runHeader(raw_ostream &o); |
| |
| // runTests - Emit tests for all the Neon intrinsics. |
| void runTests(raw_ostream &o); |
| }; |
| |
| } // end anonymous namespace |
| |
| //===----------------------------------------------------------------------===// |
| // Type implementation |
| //===----------------------------------------------------------------------===// |
| |
| std::string Type::str() const { |
| if (Void) |
| return "void"; |
| std::string S; |
| |
| if (!Signed && isInteger()) |
| S += "u"; |
| |
| if (Poly) |
| S += "poly"; |
| else if (Float) |
| S += "float"; |
| else |
| S += "int"; |
| |
| S += utostr(ElementBitwidth); |
| if (isVector()) |
| S += "x" + utostr(getNumElements()); |
| if (NumVectors > 1) |
| S += "x" + utostr(NumVectors); |
| S += "_t"; |
| |
| if (Constant) |
| S += " const"; |
| if (Pointer) |
| S += " *"; |
| |
| return S; |
| } |
| |
| std::string Type::builtin_str() const { |
| std::string S; |
| if (isVoid()) |
| return "v"; |
| |
| if (Pointer) |
| // All pointers are void pointers. |
| S += "v"; |
| else if (isInteger()) |
| switch (ElementBitwidth) { |
| case 8: S += "c"; break; |
| case 16: S += "s"; break; |
| case 32: S += "i"; break; |
| case 64: S += "Wi"; break; |
| case 128: S += "LLLi"; break; |
| default: llvm_unreachable("Unhandled case!"); |
| } |
| else |
| switch (ElementBitwidth) { |
| case 16: S += "h"; break; |
| case 32: S += "f"; break; |
| case 64: S += "d"; break; |
| default: llvm_unreachable("Unhandled case!"); |
| } |
| |
| if (isChar() && !Pointer && Signed) |
| // Make chars explicitly signed. |
| S = "S" + S; |
| else if (isInteger() && !Pointer && !Signed) |
| S = "U" + S; |
| |
| // Constant indices are "int", but have the "constant expression" modifier. |
| if (isImmediate()) { |
| assert(isInteger() && isSigned()); |
| S = "I" + S; |
| } |
| |
| if (isScalar()) { |
| if (Constant) S += "C"; |
| if (Pointer) S += "*"; |
| return S; |
| } |
| |
| std::string Ret; |
| for (unsigned I = 0; I < NumVectors; ++I) |
| Ret += "V" + utostr(getNumElements()) + S; |
| |
| return Ret; |
| } |
| |
| unsigned Type::getNeonEnum() const { |
| unsigned Addend; |
| switch (ElementBitwidth) { |
| case 8: Addend = 0; break; |
| case 16: Addend = 1; break; |
| case 32: Addend = 2; break; |
| case 64: Addend = 3; break; |
| case 128: Addend = 4; break; |
| default: llvm_unreachable("Unhandled element bitwidth!"); |
| } |
| |
| unsigned Base = (unsigned)NeonTypeFlags::Int8 + Addend; |
| if (Poly) { |
| // Adjustment needed because Poly32 doesn't exist. |
| if (Addend >= 2) |
| --Addend; |
| Base = (unsigned)NeonTypeFlags::Poly8 + Addend; |
| } |
| if (Float) { |
| assert(Addend != 0 && "Float8 doesn't exist!"); |
| Base = (unsigned)NeonTypeFlags::Float16 + (Addend - 1); |
| } |
| |
| if (Bitwidth == 128) |
| Base |= (unsigned)NeonTypeFlags::QuadFlag; |
| if (isInteger() && !Signed) |
| Base |= (unsigned)NeonTypeFlags::UnsignedFlag; |
| |
| return Base; |
| } |
| |
| Type Type::fromTypedefName(StringRef Name) { |
| Type T; |
| T.Void = false; |
| T.Float = false; |
| T.Poly = false; |
| |
| if (Name.front() == 'u') { |
| T.Signed = false; |
| Name = Name.drop_front(); |
| } else { |
| T.Signed = true; |
| } |
| |
| if (Name.startswith("float")) { |
| T.Float = true; |
| Name = Name.drop_front(5); |
| } else if (Name.startswith("poly")) { |
| T.Poly = true; |
| Name = Name.drop_front(4); |
| } else { |
| assert(Name.startswith("int")); |
| Name = Name.drop_front(3); |
| } |
| |
| unsigned I = 0; |
| for (I = 0; I < Name.size(); ++I) { |
| if (!isdigit(Name[I])) |
| break; |
| } |
| Name.substr(0, I).getAsInteger(10, T.ElementBitwidth); |
| Name = Name.drop_front(I); |
| |
| T.Bitwidth = T.ElementBitwidth; |
| T.NumVectors = 1; |
| |
| if (Name.front() == 'x') { |
| Name = Name.drop_front(); |
| unsigned I = 0; |
| for (I = 0; I < Name.size(); ++I) { |
| if (!isdigit(Name[I])) |
| break; |
| } |
| unsigned NumLanes; |
| Name.substr(0, I).getAsInteger(10, NumLanes); |
| Name = Name.drop_front(I); |
| T.Bitwidth = T.ElementBitwidth * NumLanes; |
| } else { |
| // Was scalar. |
| T.NumVectors = 0; |
| } |
| if (Name.front() == 'x') { |
| Name = Name.drop_front(); |
| unsigned I = 0; |
| for (I = 0; I < Name.size(); ++I) { |
| if (!isdigit(Name[I])) |
| break; |
| } |
| Name.substr(0, I).getAsInteger(10, T.NumVectors); |
| Name = Name.drop_front(I); |
| } |
| |
| assert(Name.startswith("_t") && "Malformed typedef!"); |
| return T; |
| } |
| |
| void Type::applyTypespec(bool &Quad) { |
| std::string S = TS; |
| ScalarForMangling = false; |
| Void = false; |
| Poly = Float = false; |
| ElementBitwidth = ~0U; |
| Signed = true; |
| NumVectors = 1; |
| |
| for (char I : S) { |
| switch (I) { |
| case 'S': |
| ScalarForMangling = true; |
| break; |
| case 'H': |
| NoManglingQ = true; |
| Quad = true; |
| break; |
| case 'Q': |
| Quad = true; |
| break; |
| case 'P': |
| Poly = true; |
| break; |
| case 'U': |
| Signed = false; |
| break; |
| case 'c': |
| ElementBitwidth = 8; |
| break; |
| case 'h': |
| Float = true; |
| LLVM_FALLTHROUGH; |
| case 's': |
| ElementBitwidth = 16; |
| break; |
| case 'f': |
| Float = true; |
| LLVM_FALLTHROUGH; |
| case 'i': |
| ElementBitwidth = 32; |
| break; |
| case 'd': |
| Float = true; |
| LLVM_FALLTHROUGH; |
| case 'l': |
| ElementBitwidth = 64; |
| break; |
| case 'k': |
| ElementBitwidth = 128; |
| // Poly doesn't have a 128x1 type. |
| if (Poly) |
| NumVectors = 0; |
| break; |
| default: |
| llvm_unreachable("Unhandled type code!"); |
| } |
| } |
| assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); |
| |
| Bitwidth = Quad ? 128 : 64; |
| } |
| |
| void Type::applyModifier(char Mod) { |
| bool AppliedQuad = false; |
| applyTypespec(AppliedQuad); |
| |
| switch (Mod) { |
| case 'v': |
| Void = true; |
| break; |
| case 't': |
| if (Poly) { |
| Poly = false; |
| Signed = false; |
| } |
| break; |
| case 'b': |
| Signed = false; |
| Float = false; |
| Poly = false; |
| NumVectors = 0; |
| Bitwidth = ElementBitwidth; |
| break; |
| case '$': |
| Signed = true; |
| Float = false; |
| Poly = false; |
| NumVectors = 0; |
| Bitwidth = ElementBitwidth; |
| break; |
| case 'u': |
| Signed = false; |
| Poly = false; |
| Float = false; |
| break; |
| case 'x': |
| Signed = true; |
| assert(!Poly && "'u' can't be used with poly types!"); |
| Float = false; |
| break; |
| case 'o': |
| Bitwidth = ElementBitwidth = 64; |
| NumVectors = 0; |
| Float = true; |
| break; |
| case 'y': |
| Bitwidth = ElementBitwidth = 32; |
| NumVectors = 0; |
| Float = true; |
| break; |
| case 'Y': |
| Bitwidth = ElementBitwidth = 16; |
| NumVectors = 0; |
| Float = true; |
| break; |
| case 'I': |
| Bitwidth = ElementBitwidth = 32; |
| NumVectors = 0; |
| Float = false; |
| Signed = true; |
| break; |
| case 'L': |
| Bitwidth = ElementBitwidth = 64; |
| NumVectors = 0; |
| Float = false; |
| Signed = true; |
| break; |
| case 'U': |
| Bitwidth = ElementBitwidth = 32; |
| NumVectors = 0; |
| Float = false; |
| Signed = false; |
| break; |
| case 'O': |
| Bitwidth = ElementBitwidth = 64; |
| NumVectors = 0; |
| Float = false; |
| Signed = false; |
| break; |
| case 'f': |
| Float = true; |
| ElementBitwidth = 32; |
| break; |
| case 'F': |
| Float = true; |
| ElementBitwidth = 64; |
| break; |
| case 'H': |
| Float = true; |
| ElementBitwidth = 16; |
| break; |
| case '0': |
| Float = true; |
| if (AppliedQuad) |
| Bitwidth /= 2; |
| ElementBitwidth = 16; |
| break; |
| case '1': |
| Float = true; |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| ElementBitwidth = 16; |
| break; |
| case 'g': |
| if (AppliedQuad) |
| Bitwidth /= 2; |
| break; |
| case 'j': |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| break; |
| case 'w': |
| ElementBitwidth *= 2; |
| Bitwidth *= 2; |
| break; |
| case 'n': |
| ElementBitwidth *= 2; |
| break; |
| case 'i': |
| Float = false; |
| Poly = false; |
| ElementBitwidth = Bitwidth = 32; |
| NumVectors = 0; |
| Signed = true; |
| Immediate = true; |
| break; |
| case 'l': |
| Float = false; |
| Poly = false; |
| ElementBitwidth = Bitwidth = 64; |
| NumVectors = 0; |
| Signed = false; |
| Immediate = true; |
| break; |
| case 'z': |
| ElementBitwidth /= 2; |
| Bitwidth = ElementBitwidth; |
| NumVectors = 0; |
| break; |
| case 'r': |
| ElementBitwidth *= 2; |
| Bitwidth = ElementBitwidth; |
| NumVectors = 0; |
| break; |
| case 's': |
| case 'a': |
| Bitwidth = ElementBitwidth; |
| NumVectors = 0; |
| break; |
| case 'k': |
| Bitwidth *= 2; |
| break; |
| case 'c': |
| Constant = true; |
| LLVM_FALLTHROUGH; |
| case 'p': |
| Pointer = true; |
| Bitwidth = ElementBitwidth; |
| NumVectors = 0; |
| break; |
| case 'h': |
| ElementBitwidth /= 2; |
| break; |
| case 'q': |
| ElementBitwidth /= 2; |
| Bitwidth *= 2; |
| break; |
| case 'e': |
| ElementBitwidth /= 2; |
| Signed = false; |
| break; |
| case 'm': |
| ElementBitwidth /= 2; |
| Bitwidth /= 2; |
| break; |
| case 'd': |
| break; |
| case '2': |
| NumVectors = 2; |
| break; |
| case '3': |
| NumVectors = 3; |
| break; |
| case '4': |
| NumVectors = 4; |
| break; |
| case 'B': |
| NumVectors = 2; |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| break; |
| case 'C': |
| NumVectors = 3; |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| break; |
| case 'D': |
| NumVectors = 4; |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| break; |
| case '7': |
| if (AppliedQuad) |
| Bitwidth /= 2; |
| ElementBitwidth = 8; |
| break; |
| case '8': |
| ElementBitwidth = 8; |
| break; |
| case '9': |
| if (!AppliedQuad) |
| Bitwidth *= 2; |
| ElementBitwidth = 8; |
| break; |
| default: |
| llvm_unreachable("Unhandled character!"); |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Intrinsic implementation |
| //===----------------------------------------------------------------------===// |
| |
| std::string Intrinsic::getInstTypeCode(Type T, ClassKind CK) const { |
| char typeCode = '\0'; |
| bool printNumber = true; |
| |
| if (CK == ClassB) |
| return ""; |
| |
| if (T.isPoly()) |
| typeCode = 'p'; |
| else if (T.isInteger()) |
| typeCode = T.isSigned() ? 's' : 'u'; |
| else |
| typeCode = 'f'; |
| |
| if (CK == ClassI) { |
| switch (typeCode) { |
| default: |
| break; |
| case 's': |
| case 'u': |
| case 'p': |
| typeCode = 'i'; |
| break; |
| } |
| } |
| if (CK == ClassB) { |
| typeCode = '\0'; |
| } |
| |
| std::string S; |
| if (typeCode != '\0') |
| S.push_back(typeCode); |
| if (printNumber) |
| S += utostr(T.getElementSizeInBits()); |
| |
| return S; |
| } |
| |
| static bool isFloatingPointProtoModifier(char Mod) { |
| return Mod == 'F' || Mod == 'f' || Mod == 'H' || Mod == 'Y' || Mod == 'I'; |
| } |
| |
| std::string Intrinsic::getBuiltinTypeStr() { |
| ClassKind LocalCK = getClassKind(true); |
| std::string S; |
| |
| Type RetT = getReturnType(); |
| if ((LocalCK == ClassI || LocalCK == ClassW) && RetT.isScalar() && |
| !RetT.isFloating()) |
| RetT.makeInteger(RetT.getElementSizeInBits(), false); |
| |
| // Since the return value must be one type, return a vector type of the |
| // appropriate width which we will bitcast. An exception is made for |
| // returning structs of 2, 3, or 4 vectors which are returned in a sret-like |
| // fashion, storing them to a pointer arg. |
| if (RetT.getNumVectors() > 1) { |
| S += "vv*"; // void result with void* first argument |
| } else { |
| if (RetT.isPoly()) |
| RetT.makeInteger(RetT.getElementSizeInBits(), false); |
| if (!RetT.isScalar() && !RetT.isSigned()) |
| RetT.makeSigned(); |
| |
| bool ForcedVectorFloatingType = isFloatingPointProtoModifier(Proto[0]); |
| if (LocalCK == ClassB && !RetT.isScalar() && !ForcedVectorFloatingType) |
| // Cast to vector of 8-bit elements. |
| RetT.makeInteger(8, true); |
| |
| S += RetT.builtin_str(); |
| } |
| |
| for (unsigned I = 0; I < getNumParams(); ++I) { |
| Type T = getParamType(I); |
| if (T.isPoly()) |
| T.makeInteger(T.getElementSizeInBits(), false); |
| |
| bool ForcedFloatingType = isFloatingPointProtoModifier(Proto[I + 1]); |
| if (LocalCK == ClassB && !T.isScalar() && !ForcedFloatingType) |
| T.makeInteger(8, true); |
| // Halves always get converted to 8-bit elements. |
| if (T.isHalf() && T.isVector() && !T.isScalarForMangling()) |
| T.makeInteger(8, true); |
| |
| if (LocalCK == ClassI) |
| T.makeSigned(); |
| |
| if (hasImmediate() && getImmediateIdx() == I) |
| T.makeImmediate(32); |
| |
| S += T.builtin_str(); |
| } |
| |
| // Extra constant integer to hold type class enum for this function, e.g. s8 |
| if (LocalCK == ClassB) |
| S += "i"; |
| |
| return S; |
| } |
| |
| std::string Intrinsic::getMangledName(bool ForceClassS) const { |
| // Check if the prototype has a scalar operand with the type of the vector |
| // elements. If not, bitcasting the args will take care of arg checking. |
| // The actual signedness etc. will be taken care of with special enums. |
| ClassKind LocalCK = CK; |
| if (!protoHasScalar()) |
| LocalCK = ClassB; |
| |
| return mangleName(Name, ForceClassS ? ClassS : LocalCK); |
| } |
| |
| std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const { |
| std::string typeCode = getInstTypeCode(BaseType, LocalCK); |
| std::string S = Name; |
| |
| if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" || |
| Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32") |
| return Name; |
| |
| if (!typeCode.empty()) { |
| // If the name ends with _xN (N = 2,3,4), insert the typeCode before _xN. |
| if (Name.size() >= 3 && isdigit(Name.back()) && |
| Name[Name.length() - 2] == 'x' && Name[Name.length() - 3] == '_') |
| S.insert(S.length() - 3, "_" + typeCode); |
| else |
| S += "_" + typeCode; |
| } |
| |
| if (BaseType != InBaseType) { |
| // A reinterpret - out the input base type at the end. |
| S += "_" + getInstTypeCode(InBaseType, LocalCK); |
| } |
| |
| if (LocalCK == ClassB) |
| S += "_v"; |
| |
| // Insert a 'q' before the first '_' character so that it ends up before |
| // _lane or _n on vector-scalar operations. |
| if (BaseType.getSizeInBits() == 128 && !BaseType.noManglingQ()) { |
| size_t Pos = S.find('_'); |
| S.insert(Pos, "q"); |
| } |
| |
| char Suffix = '\0'; |
| if (BaseType.isScalarForMangling()) { |
| switch (BaseType.getElementSizeInBits()) { |
| case 8: Suffix = 'b'; break; |
| case 16: Suffix = 'h'; break; |
| case 32: Suffix = 's'; break; |
| case 64: Suffix = 'd'; break; |
| default: llvm_unreachable("Bad suffix!"); |
| } |
| } |
| if (Suffix != '\0') { |
| size_t Pos = S.find('_'); |
| S.insert(Pos, &Suffix, 1); |
| } |
| |
| return S; |
| } |
| |
| std::string Intrinsic::replaceParamsIn(std::string S) { |
| while (S.find('$') != std::string::npos) { |
| size_t Pos = S.find('$'); |
| size_t End = Pos + 1; |
| while (isalpha(S[End])) |
| ++End; |
| |
| std::string VarName = S.substr(Pos + 1, End - Pos - 1); |
| assert_with_loc(Variables.find(VarName) != Variables.end(), |
| "Variable not defined!"); |
| S.replace(Pos, End - Pos, Variables.find(VarName)->second.getName()); |
| } |
| |
| return S; |
| } |
| |
| void Intrinsic::initVariables() { |
| Variables.clear(); |
| |
| // Modify the TypeSpec per-argument to get a concrete Type, and create |
| // known variables for each. |
| for (unsigned I = 1; I < Proto.size(); ++I) { |
| char NameC = '0' + (I - 1); |
| std::string Name = "p"; |
| Name.push_back(NameC); |
| |
| Variables[Name] = Variable(Types[I], Name + VariablePostfix); |
| } |
| RetVar = Variable(Types[0], "ret" + VariablePostfix); |
| } |
| |
| void Intrinsic::emitPrototype(StringRef NamePrefix) { |
| if (UseMacro) |
| OS << "#define "; |
| else |
| OS << "__ai " << Types[0].str() << " "; |
| |
| OS << NamePrefix.str() << mangleName(Name, ClassS) << "("; |
| |
| for (unsigned I = 0; I < getNumParams(); ++I) { |
| if (I != 0) |
| OS << ", "; |
| |
| char NameC = '0' + I; |
| std::string Name = "p"; |
| Name.push_back(NameC); |
| assert(Variables.find(Name) != Variables.end()); |
| Variable &V = Variables[Name]; |
| |
| if (!UseMacro) |
| OS << V.getType().str() << " "; |
| OS << V.getName(); |
| } |
| |
| OS << ")"; |
| } |
| |
| void Intrinsic::emitOpeningBrace() { |
| if (UseMacro) |
| OS << " __extension__ ({"; |
| else |
| OS << " {"; |
| emitNewLine(); |
| } |
| |
| void Intrinsic::emitClosingBrace() { |
| if (UseMacro) |
| OS << "})"; |
| else |
| OS << "}"; |
| } |
| |
| void Intrinsic::emitNewLine() { |
| if (UseMacro) |
| OS << " \\\n"; |
| else |
| OS << "\n"; |
| } |
| |
| void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) { |
| if (Dest.getType().getNumVectors() > 1) { |
| emitNewLine(); |
| |
| for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) { |
| OS << " " << Dest.getName() << ".val[" << K << "] = " |
| << "__builtin_shufflevector(" |
| << Src.getName() << ".val[" << K << "], " |
| << Src.getName() << ".val[" << K << "]"; |
| for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J) |
| OS << ", " << J; |
| OS << ");"; |
| emitNewLine(); |
| } |
| } else { |
| OS << " " << Dest.getName() |
| << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName(); |
| for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J) |
| OS << ", " << J; |
| OS << ");"; |
| emitNewLine(); |
| } |
| } |
| |
| void Intrinsic::emitArgumentReversal() { |
| if (isBigEndianSafe()) |
| return; |
| |
| // Reverse all vector arguments. |
| for (unsigned I = 0; I < getNumParams(); ++I) { |
| std::string Name = "p" + utostr(I); |
| std::string NewName = "rev" + utostr(I); |
| |
| Variable &V = Variables[Name]; |
| Variable NewV(V.getType(), NewName + VariablePostfix); |
| |
| if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1) |
| continue; |
| |
| OS << " " << NewV.getType().str() << " " << NewV.getName() << ";"; |
| emitReverseVariable(NewV, V); |
| V = NewV; |
| } |
| } |
| |
| void Intrinsic::emitReturnReversal() { |
| if (isBigEndianSafe()) |
| return; |
| if (!getReturnType().isVector() || getReturnType().isVoid() || |
| getReturnType().getNumElements() == 1) |
| return; |
| emitReverseVariable(RetVar, RetVar); |
| } |
| |
| void Intrinsic::emitShadowedArgs() { |
| // Macro arguments are not type-checked like inline function arguments, |
| // so assign them to local temporaries to get the right type checking. |
| if (!UseMacro) |
| return; |
| |
| for (unsigned I = 0; I < getNumParams(); ++I) { |
| // Do not create a temporary for an immediate argument. |
| // That would defeat the whole point of using a macro! |
| if (hasImmediate() && Proto[I+1] == 'i') |
| continue; |
| // Do not create a temporary for pointer arguments. The input |
| // pointer may have an alignment hint. |
| if (getParamType(I).isPointer()) |
| continue; |
| |
| std::string Name = "p" + utostr(I); |
| |
| assert(Variables.find(Name) != Variables.end()); |
| Variable &V = Variables[Name]; |
| |
| std::string NewName = "s" + utostr(I); |
| Variable V2(V.getType(), NewName + VariablePostfix); |
| |
| OS << " " << V2.getType().str() << " " << V2.getName() << " = " |
| << V.getName() << ";"; |
| emitNewLine(); |
| |
| V = V2; |
| } |
| } |
| |
| // We don't check 'a' in this function, because for builtin function the |
| // argument matching to 'a' uses a vector type splatted from a scalar type. |
| bool Intrinsic::protoHasScalar() const { |
| return (Proto.find('s') != std::string::npos || |
| Proto.find('z') != std::string::npos || |
| Proto.find('r') != std::string::npos || |
| Proto.find('b') != std::string::npos || |
| Proto.find('$') != std::string::npos || |
| Proto.find('y') != std::string::npos || |
| Proto.find('o') != std::string::npos); |
| } |
| |
| void Intrinsic::emitBodyAsBuiltinCall() { |
| std::string S; |
| |
| // If this builtin returns a struct 2, 3, or 4 vectors, pass it as an implicit |
| // sret-like argument. |
| bool SRet = getReturnType().getNumVectors() >= 2; |
| |
| StringRef N = Name; |
| if (hasSplat()) { |
| // Call the non-splat builtin: chop off the "_n" suffix from the name. |
| assert(N.endswith("_n")); |
| N = N.drop_back(2); |
| } |
| |
| ClassKind LocalCK = CK; |
| if (!protoHasScalar()) |
| LocalCK = ClassB; |
| |
| if (!getReturnType().isVoid() && !SRet) |
| S += "(" + RetVar.getType().str() + ") "; |
| |
| S += "__builtin_neon_" + mangleName(N, LocalCK) + "("; |
| |
| if (SRet) |
| S += "&" + RetVar.getName() + ", "; |
| |
| for (unsigned I = 0; I < getNumParams(); ++I) { |
| Variable &V = Variables["p" + utostr(I)]; |
| Type T = V.getType(); |
| |
| // Handle multiple-vector values specially, emitting each subvector as an |
| // argument to the builtin. |
| if (T.getNumVectors() > 1) { |
| // Check if an explicit cast is needed. |
| std::string Cast; |
| if (LocalCK == ClassB) { |
| Type T2 = T; |
| T2.makeOneVector(); |
| T2.makeInteger(8, /*Signed=*/true); |
| Cast = "(" + T2.str() + ")"; |
| } |
| |
| for (unsigned J = 0; J < T.getNumVectors(); ++J) |
| S += Cast + V.getName() + ".val[" + utostr(J) + "], "; |
| continue; |
| } |
| |
| std::string Arg; |
| Type CastToType = T; |
| if (hasSplat() && I == getSplatIdx()) { |
| Arg = "(" + BaseType.str() + ") {"; |
| for (unsigned J = 0; J < BaseType.getNumElements(); ++J) { |
| if (J != 0) |
| Arg += ", "; |
| Arg += V.getName(); |
| } |
| Arg += "}"; |
| |
| CastToType = BaseType; |
| } else { |
| Arg = V.getName(); |
| } |
| |
| // Check if an explicit cast is needed. |
| if (CastToType.isVector() && |
| (LocalCK == ClassB || (T.isHalf() && !T.isScalarForMangling()))) { |
| CastToType.makeInteger(8, true); |
| Arg = "(" + CastToType.str() + ")" + Arg; |
| } else if (CastToType.isVector() && LocalCK == ClassI) { |
| CastToType.makeSigned(); |
| Arg = "(" + CastToType.str() + ")" + Arg; |
| } |
| |
| S += Arg + ", "; |
| } |
| |
| // Extra constant integer to hold type class enum for this function, e.g. s8 |
| if (getClassKind(true) == ClassB) { |
| Type ThisTy = getReturnType(); |
| if (Proto[0] == 'v' || isFloatingPointProtoModifier(Proto[0])) |
| ThisTy = getParamType(0); |
| if (ThisTy.isPointer()) |
| ThisTy = getParamType(1); |
| |
| S += utostr(ThisTy.getNeonEnum()); |
| } else { |
| // Remove extraneous ", ". |
| S.pop_back(); |
| S.pop_back(); |
| } |
| S += ");"; |
| |
| std::string RetExpr; |
| if (!SRet && !RetVar.getType().isVoid()) |
| RetExpr = RetVar.getName() + " = "; |
| |
| OS << " " << RetExpr << S; |
| emitNewLine(); |
| } |
| |
| void Intrinsic::emitBody(StringRef CallPrefix) { |
| std::vector<std::string> Lines; |
| |
| assert(RetVar.getType() == Types[0]); |
| // Create a return variable, if we're not void. |
| if (!RetVar.getType().isVoid()) { |
| OS << " " << RetVar.getType().str() << " " << RetVar.getName() << ";"; |
| emitNewLine(); |
| } |
| |
| if (!Body || Body->getValues().empty()) { |
| // Nothing specific to output - must output a builtin. |
| emitBodyAsBuiltinCall(); |
| return; |
| } |
| |
| // We have a list of "things to output". The last should be returned. |
| for (auto *I : Body->getValues()) { |
| if (StringInit *SI = dyn_cast<StringInit>(I)) { |
| Lines.push_back(replaceParamsIn(SI->getAsString())); |
| } else if (DagInit *DI = dyn_cast<DagInit>(I)) { |
| DagEmitter DE(*this, CallPrefix); |
| Lines.push_back(DE.emitDag(DI).second + ";"); |
| } |
| } |
| |
| assert(!Lines.empty() && "Empty def?"); |
| if (!RetVar.getType().isVoid()) |
| Lines.back().insert(0, RetVar.getName() + " = "); |
| |
| for (auto &L : Lines) { |
| OS << " " << L; |
| emitNewLine(); |
| } |
| } |
| |
| void Intrinsic::emitReturn() { |
| if (RetVar.getType().isVoid()) |
| return; |
| if (UseMacro) |
| OS << " " << RetVar.getName() << ";"; |
| else |
| OS << " return " << RetVar.getName() << ";"; |
| emitNewLine(); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) { |
| // At this point we should only be seeing a def. |
| DefInit *DefI = cast<DefInit>(DI->getOperator()); |
| std::string Op = DefI->getAsString(); |
| |
| if (Op == "cast" || Op == "bitcast") |
| return emitDagCast(DI, Op == "bitcast"); |
| if (Op == "shuffle") |
| return emitDagShuffle(DI); |
| if (Op == "dup") |
| return emitDagDup(DI); |
| if (Op == "dup_typed") |
| return emitDagDupTyped(DI); |
| if (Op == "splat") |
| return emitDagSplat(DI); |
| if (Op == "save_temp") |
| return emitDagSaveTemp(DI); |
| if (Op == "op") |
| return emitDagOp(DI); |
| if (Op == "call") |
| return emitDagCall(DI); |
| if (Op == "name_replace") |
| return emitDagNameReplace(DI); |
| if (Op == "literal") |
| return emitDagLiteral(DI); |
| assert_with_loc(false, "Unknown operation!"); |
| return std::make_pair(Type::getVoid(), ""); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) { |
| std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); |
| if (DI->getNumArgs() == 2) { |
| // Unary op. |
| std::pair<Type, std::string> R = |
| emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); |
| return std::make_pair(R.first, Op + R.second); |
| } else { |
| assert(DI->getNumArgs() == 3 && "Can only handle unary and binary ops!"); |
| std::pair<Type, std::string> R1 = |
| emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); |
| std::pair<Type, std::string> R2 = |
| emitDagArg(DI->getArg(2), DI->getArgNameStr(2)); |
| assert_with_loc(R1.first == R2.first, "Argument type mismatch!"); |
| return std::make_pair(R1.first, R1.second + " " + Op + " " + R2.second); |
| } |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCall(DagInit *DI) { |
| std::vector<Type> Types; |
| std::vector<std::string> Values; |
| for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) { |
| std::pair<Type, std::string> R = |
| emitDagArg(DI->getArg(I + 1), DI->getArgNameStr(I + 1)); |
| Types.push_back(R.first); |
| Values.push_back(R.second); |
| } |
| |
| // Look up the called intrinsic. |
| std::string N; |
| if (StringInit *SI = dyn_cast<StringInit>(DI->getArg(0))) |
| N = SI->getAsUnquotedString(); |
| else |
| N = emitDagArg(DI->getArg(0), "").second; |
| Intrinsic &Callee = Intr.Emitter.getIntrinsic(N, Types); |
| |
| // Make sure the callee is known as an early def. |
| Callee.setNeededEarly(); |
| Intr.Dependencies.insert(&Callee); |
| |
| // Now create the call itself. |
| std::string S = ""; |
| if (!Callee.isBigEndianSafe()) |
| S += CallPrefix.str(); |
| S += Callee.getMangledName(true) + "("; |
| for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) { |
| if (I != 0) |
| S += ", "; |
| S += Values[I]; |
| } |
| S += ")"; |
| |
| return std::make_pair(Callee.getReturnType(), S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI, |
| bool IsBitCast){ |
| // (cast MOD* VAL) -> cast VAL to type given by MOD. |
| std::pair<Type, std::string> R = emitDagArg( |
| DI->getArg(DI->getNumArgs() - 1), |
| DI->getArgNameStr(DI->getNumArgs() - 1)); |
| Type castToType = R.first; |
| for (unsigned ArgIdx = 0; ArgIdx < DI->getNumArgs() - 1; ++ArgIdx) { |
| |
| // MOD can take several forms: |
| // 1. $X - take the type of parameter / variable X. |
| // 2. The value "R" - take the type of the return type. |
| // 3. a type string |
| // 4. The value "U" or "S" to switch the signedness. |
| // 5. The value "H" or "D" to half or double the bitwidth. |
| // 6. The value "8" to convert to 8-bit (signed) integer lanes. |
| if (!DI->getArgNameStr(ArgIdx).empty()) { |
| assert_with_loc(Intr.Variables.find(DI->getArgNameStr(ArgIdx)) != |
| Intr.Variables.end(), |
| "Variable not found"); |
| castToType = Intr.Variables[DI->getArgNameStr(ArgIdx)].getType(); |
| } else { |
| StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx)); |
| assert_with_loc(SI, "Expected string type or $Name for cast type"); |
| |
| if (SI->getAsUnquotedString() == "R") { |
| castToType = Intr.getReturnType(); |
| } else if (SI->getAsUnquotedString() == "U") { |
| castToType.makeUnsigned(); |
| } else if (SI->getAsUnquotedString() == "S") { |
| castToType.makeSigned(); |
| } else if (SI->getAsUnquotedString() == "H") { |
| castToType.halveLanes(); |
| } else if (SI->getAsUnquotedString() == "D") { |
| castToType.doubleLanes(); |
| } else if (SI->getAsUnquotedString() == "8") { |
| castToType.makeInteger(8, true); |
| } else { |
| castToType = Type::fromTypedefName(SI->getAsUnquotedString()); |
| assert_with_loc(!castToType.isVoid(), "Unknown typedef"); |
| } |
| } |
| } |
| |
| std::string S; |
| if (IsBitCast) { |
| // Emit a reinterpret cast. The second operand must be an lvalue, so create |
| // a temporary. |
| std::string N = "reint"; |
| unsigned I = 0; |
| while (Intr.Variables.find(N) != Intr.Variables.end()) |
| N = "reint" + utostr(++I); |
| Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix); |
| |
| Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = " |
| << R.second << ";"; |
| Intr.emitNewLine(); |
| |
| S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + ""; |
| } else { |
| // Emit a normal (static) cast. |
| S = "(" + castToType.str() + ")(" + R.second + ")"; |
| } |
| |
| return std::make_pair(castToType, S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){ |
| // See the documentation in arm_neon.td for a description of these operators. |
| class LowHalf : public SetTheory::Operator { |
| public: |
| void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, |
| ArrayRef<SMLoc> Loc) override { |
| SetTheory::RecSet Elts2; |
| ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc); |
| Elts.insert(Elts2.begin(), Elts2.begin() + (Elts2.size() / 2)); |
| } |
| }; |
| |
| class HighHalf : public SetTheory::Operator { |
| public: |
| void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, |
| ArrayRef<SMLoc> Loc) override { |
| SetTheory::RecSet Elts2; |
| ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts2, Loc); |
| Elts.insert(Elts2.begin() + (Elts2.size() / 2), Elts2.end()); |
| } |
| }; |
| |
| class Rev : public SetTheory::Operator { |
| unsigned ElementSize; |
| |
| public: |
| Rev(unsigned ElementSize) : ElementSize(ElementSize) {} |
| |
| void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, |
| ArrayRef<SMLoc> Loc) override { |
| SetTheory::RecSet Elts2; |
| ST.evaluate(Expr->arg_begin() + 1, Expr->arg_end(), Elts2, Loc); |
| |
| int64_t VectorSize = cast<IntInit>(Expr->getArg(0))->getValue(); |
| VectorSize /= ElementSize; |
| |
| std::vector<Record *> Revved; |
| for (unsigned VI = 0; VI < Elts2.size(); VI += VectorSize) { |
| for (int LI = VectorSize - 1; LI >= 0; --LI) { |
| Revved.push_back(Elts2[VI + LI]); |
| } |
| } |
| |
| Elts.insert(Revved.begin(), Revved.end()); |
| } |
| }; |
| |
| class MaskExpander : public SetTheory::Expander { |
| unsigned N; |
| |
| public: |
| MaskExpander(unsigned N) : N(N) {} |
| |
| void expand(SetTheory &ST, Record *R, SetTheory::RecSet &Elts) override { |
| unsigned Addend = 0; |
| if (R->getName() == "mask0") |
| Addend = 0; |
| else if (R->getName() == "mask1") |
| Addend = N; |
| else |
| return; |
| for (unsigned I = 0; I < N; ++I) |
| Elts.insert(R->getRecords().getDef("sv" + utostr(I + Addend))); |
| } |
| }; |
| |
| // (shuffle arg1, arg2, sequence) |
| std::pair<Type, std::string> Arg1 = |
| emitDagArg(DI->getArg(0), DI->getArgNameStr(0)); |
| std::pair<Type, std::string> Arg2 = |
| emitDagArg(DI->getArg(1), DI->getArgNameStr(1)); |
| assert_with_loc(Arg1.first == Arg2.first, |
| "Different types in arguments to shuffle!"); |
| |
| SetTheory ST; |
| SetTheory::RecSet Elts; |
| ST.addOperator("lowhalf", std::make_unique<LowHalf>()); |
| ST.addOperator("highhalf", std::make_unique<HighHalf>()); |
| ST.addOperator("rev", |
| std::make_unique<Rev>(Arg1.first.getElementSizeInBits())); |
| ST.addExpander("MaskExpand", |
| std::make_unique<MaskExpander>(Arg1.first.getNumElements())); |
| ST.evaluate(DI->getArg(2), Elts, None); |
| |
| std::string S = "__builtin_shufflevector(" + Arg1.second + ", " + Arg2.second; |
| for (auto &E : Elts) { |
| StringRef Name = E->getName(); |
| assert_with_loc(Name.startswith("sv"), |
| "Incorrect element kind in shuffle mask!"); |
| S += ", " + Name.drop_front(2).str(); |
| } |
| S += ")"; |
| |
| // Recalculate the return type - the shuffle may have halved or doubled it. |
| Type T(Arg1.first); |
| if (Elts.size() > T.getNumElements()) { |
| assert_with_loc( |
| Elts.size() == T.getNumElements() * 2, |
| "Can only double or half the number of elements in a shuffle!"); |
| T.doubleLanes(); |
| } else if (Elts.size() < T.getNumElements()) { |
| assert_with_loc( |
| Elts.size() == T.getNumElements() / 2, |
| "Can only double or half the number of elements in a shuffle!"); |
| T.halveLanes(); |
| } |
| |
| return std::make_pair(T, S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) { |
| assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument"); |
| std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), |
| DI->getArgNameStr(0)); |
| assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument"); |
| |
| Type T = Intr.getBaseType(); |
| assert_with_loc(T.isVector(), "dup() used but default type is scalar!"); |
| std::string S = "(" + T.str() + ") {"; |
| for (unsigned I = 0; I < T.getNumElements(); ++I) { |
| if (I != 0) |
| S += ", "; |
| S += A.second; |
| } |
| S += "}"; |
| |
| return std::make_pair(T, S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDupTyped(DagInit *DI) { |
| assert_with_loc(DI->getNumArgs() == 2, "dup_typed() expects two arguments"); |
| std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), |
| DI->getArgNameStr(0)); |
| std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), |
| DI->getArgNameStr(1)); |
| assert_with_loc(B.first.isScalar(), |
| "dup_typed() requires a scalar as the second argument"); |
| |
| Type T = A.first; |
| assert_with_loc(T.isVector(), "dup_typed() used but target type is scalar!"); |
| std::string S = "(" + T.str() + ") {"; |
| for (unsigned I = 0; I < T.getNumElements(); ++I) { |
| if (I != 0) |
| S += ", "; |
| S += B.second; |
| } |
| S += "}"; |
| |
| return std::make_pair(T, S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) { |
| assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments"); |
| std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), |
| DI->getArgNameStr(0)); |
| std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), |
| DI->getArgNameStr(1)); |
| |
| assert_with_loc(B.first.isScalar(), |
| "splat() requires a scalar int as the second argument"); |
| |
| std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second; |
| for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) { |
| S += ", " + B.second; |
| } |
| S += ")"; |
| |
| return std::make_pair(Intr.getBaseType(), S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) { |
| assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments"); |
| std::pair<Type, std::string> A = emitDagArg(DI->getArg(1), |
| DI->getArgNameStr(1)); |
| |
| assert_with_loc(!A.first.isVoid(), |
| "Argument to save_temp() must have non-void type!"); |
| |
| std::string N = DI->getArgNameStr(0); |
| assert_with_loc(!N.empty(), |
| "save_temp() expects a name as the first argument"); |
| |
| assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(), |
| "Variable already defined!"); |
| Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix); |
| |
| std::string S = |
| A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second; |
| |
| return std::make_pair(Type::getVoid(), S); |
| } |
| |
| std::pair<Type, std::string> |
| Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) { |
| std::string S = Intr.Name; |
| |
| assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!"); |
| std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); |
| std::string ReplaceWith = cast<StringInit>(DI->getArg(1))->getAsUnquotedString(); |
| |
| size_t Idx = S.find(ToReplace); |
| |
| assert_with_loc(Idx != std::string::npos, "name should contain '" + ToReplace + "'!"); |
| S.replace(Idx, ToReplace.size(), ReplaceWith); |
| |
| return std::make_pair(Type::getVoid(), S); |
| } |
| |
| std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){ |
| std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString(); |
| std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString(); |
| return std::make_pair(Type::fromTypedefName(Ty), Value); |
| } |
| |
| std::pair<Type, std::string> |
| Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) { |
| if (!ArgName.empty()) { |
| assert_with_loc(!Arg->isComplete(), |
| "Arguments must either be DAGs or names, not both!"); |
| assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(), |
| "Variable not defined!"); |
| Variable &V = Intr.Variables[ArgName]; |
| return std::make_pair(V.getType(), V.getName()); |
| } |
| |
| assert(Arg && "Neither ArgName nor Arg?!"); |
| DagInit *DI = dyn_cast<DagInit>(Arg); |
| assert_with_loc(DI, "Arguments must either be DAGs or names!"); |
| |
| return emitDag(DI); |
| } |
| |
| std::string Intrinsic::generate() { |
| // Avoid duplicated code for big and little endian |
| if (isBigEndianSafe()) { |
| generateImpl(false, "", ""); |
| return OS.str(); |
| } |
| // Little endian intrinsics are simple and don't require any argument |
| // swapping. |
| OS << "#ifdef __LITTLE_ENDIAN__\n"; |
| |
| generateImpl(false, "", ""); |
| |
| OS << "#else\n"; |
| |
| // Big endian intrinsics are more complex. The user intended these |
| // intrinsics to operate on a vector "as-if" loaded by (V)LDR, |
| // but we load as-if (V)LD1. So we should swap all arguments and |
| // swap the return value too. |
| // |
| // If we call sub-intrinsics, we should call a version that does |
| // not re-swap the arguments! |
| generateImpl(true, "", "__noswap_"); |
| |
| // If we're needed early, create a non-swapping variant for |
| // big-endian. |
| if (NeededEarly) { |
| generateImpl(false, "__noswap_", "__noswap_"); |
| } |
| OS << "#endif\n\n"; |
| |
| return OS.str(); |
| } |
| |
| void Intrinsic::generateImpl(bool ReverseArguments, |
| StringRef NamePrefix, StringRef CallPrefix) { |
| CurrentRecord = R; |
| |
| // If we call a macro, our local variables may be corrupted due to |
| // lack of proper lexical scoping. So, add a globally unique postfix |
| // to every variable. |
| // |
| // indexBody() should have set up the Dependencies set by now. |
| for (auto *I : Dependencies) |
| if (I->UseMacro) { |
| VariablePostfix = "_" + utostr(Emitter.getUniqueNumber()); |
| break; |
| } |
| |
| initVariables(); |
| |
| emitPrototype(NamePrefix); |
| |
| if (IsUnavailable) { |
| OS << " __attribute__((unavailable));"; |
| } else { |
| emitOpeningBrace(); |
| emitShadowedArgs(); |
| if (ReverseArguments) |
| emitArgumentReversal(); |
| emitBody(CallPrefix); |
| if (ReverseArguments) |
| emitReturnReversal(); |
| emitReturn(); |
| emitClosingBrace(); |
| } |
| OS << "\n"; |
| |
| CurrentRecord = nullptr; |
| } |
| |
| void Intrinsic::indexBody() { |
| CurrentRecord = R; |
| |
| initVariables(); |
| emitBody(""); |
| OS.str(""); |
| |
| CurrentRecord = nullptr; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // NeonEmitter implementation |
| //===----------------------------------------------------------------------===// |
| |
| Intrinsic &NeonEmitter::getIntrinsic(StringRef Name, ArrayRef<Type> Types) { |
| // First, look up the name in the intrinsic map. |
| assert_with_loc(IntrinsicMap.find(Name.str()) != IntrinsicMap.end(), |
| ("Intrinsic '" + Name + "' not found!").str()); |
| auto &V = IntrinsicMap.find(Name.str())->second; |
| std::vector<Intrinsic *> GoodVec; |
| |
| // Create a string to print if we end up failing. |
| std::string ErrMsg = "looking up intrinsic '" + Name.str() + "("; |
| for (unsigned I = 0; I < Types.size(); ++I) { |
| if (I != 0) |
| ErrMsg += ", "; |
| ErrMsg += Types[I].str(); |
| } |
| ErrMsg += ")'\n"; |
| ErrMsg += "Available overloads:\n"; |
| |
| // Now, look through each intrinsic implementation and see if the types are |
| // compatible. |
| for (auto &I : V) { |
| ErrMsg += " - " + I.getReturnType().str() + " " + I.getMangledName(); |
| ErrMsg += "("; |
| for (unsigned A = 0; A < I.getNumParams(); ++A) { |
| if (A != 0) |
| ErrMsg += ", "; |
| ErrMsg += I.getParamType(A).str(); |
| } |
| ErrMsg += ")\n"; |
| |
| if (I.getNumParams() != Types.size()) |
| continue; |
| |
| bool Good = true; |
| for (unsigned Arg = 0; Arg < Types.size(); ++Arg) { |
| if (I.getParamType(Arg) != Types[Arg]) { |
| Good = false; |
| break; |
| } |
| } |
| if (Good) |
| GoodVec.push_back(&I); |
| } |
| |
| assert_with_loc(!GoodVec.empty(), |
| "No compatible intrinsic found - " + ErrMsg); |
| assert_with_loc(GoodVec.size() == 1, "Multiple overloads found - " + ErrMsg); |
| |
| return *GoodVec.front(); |
| } |
| |
| void NeonEmitter::createIntrinsic(Record *R, |
| SmallVectorImpl<Intrinsic *> &Out) { |
| std::string Name = R->getValueAsString("Name"); |
| std::string Proto = R->getValueAsString("Prototype"); |
| std::string Types = R->getValueAsString("Types"); |
| Record *OperationRec = R->getValueAsDef("Operation"); |
| bool CartesianProductOfTypes = R->getValueAsBit("CartesianProductOfTypes"); |
| bool BigEndianSafe = R->getValueAsBit("BigEndianSafe"); |
| std::string Guard = R->getValueAsString("ArchGuard"); |
| bool IsUnavailable = OperationRec->getValueAsBit("Unavailable"); |
| |
| // Set the global current record. This allows assert_with_loc to produce |
| // decent location information even when highly nested. |
| CurrentRecord = R; |
| |
| ListInit *Body = OperationRec->getValueAsListInit("Ops"); |
| |
| std::vector<TypeSpec> TypeSpecs = TypeSpec::fromTypeSpecs(Types); |
| |
| ClassKind CK = ClassNone; |
| if (R->getSuperClasses().size() >= 2) |
| CK = ClassMap[R->getSuperClasses()[1].first]; |
| |
| std::vector<std::pair<TypeSpec, TypeSpec>> NewTypeSpecs; |
| for (auto TS : TypeSpecs) { |
| if (CartesianProductOfTypes) { |
| Type DefaultT(TS, 'd'); |
| for (auto SrcTS : TypeSpecs) { |
| Type DefaultSrcT(SrcTS, 'd'); |
| if (TS == SrcTS || |
| DefaultSrcT.getSizeInBits() != DefaultT.getSizeInBits()) |
| continue; |
| NewTypeSpecs.push_back(std::make_pair(TS, SrcTS)); |
| } |
| } else { |
| NewTypeSpecs.push_back(std::make_pair(TS, TS)); |
| } |
| } |
| |
| llvm::sort(NewTypeSpecs); |
| NewTypeSpecs.erase(std::unique(NewTypeSpecs.begin(), NewTypeSpecs.end()), |
| NewTypeSpecs.end()); |
| auto &Entry = IntrinsicMap[Name]; |
| |
| for (auto &I : NewTypeSpecs) { |
| Entry.emplace_back(R, Name, Proto, I.first, I.second, CK, Body, *this, |
| Guard, IsUnavailable, BigEndianSafe); |
| Out.push_back(&Entry.back()); |
| } |
| |
| CurrentRecord = nullptr; |
| } |
| |
| /// genBuiltinsDef: Generate the BuiltinsARM.def and BuiltinsAArch64.def |
| /// declaration of builtins, checking for unique builtin declarations. |
| void NeonEmitter::genBuiltinsDef(raw_ostream &OS, |
| SmallVectorImpl<Intrinsic *> &Defs) { |
| OS << "#ifdef GET_NEON_BUILTINS\n"; |
| |
| // We only want to emit a builtin once, and we want to emit them in |
| // alphabetical order, so use a std::set. |
| std::set<std::string> Builtins; |
| |
| for (auto *Def : Defs) { |
| if (Def->hasBody()) |
| continue; |
| // Functions with 'a' (the splat code) in the type prototype should not get |
| // their own builtin as they use the non-splat variant. |
| if (Def->hasSplat()) |
| continue; |
| |
| std::string S = "BUILTIN(__builtin_neon_" + Def->getMangledName() + ", \""; |
| |
| S += Def->getBuiltinTypeStr(); |
| S += "\", \"n\")"; |
| |
| Builtins.insert(S); |
| } |
| |
| for (auto &S : Builtins) |
| OS << S << "\n"; |
| OS << "#endif\n\n"; |
| } |
| |
| /// Generate the ARM and AArch64 overloaded type checking code for |
| /// SemaChecking.cpp, checking for unique builtin declarations. |
| void NeonEmitter::genOverloadTypeCheckCode(raw_ostream &OS, |
| SmallVectorImpl<Intrinsic *> &Defs) { |
| OS << "#ifdef GET_NEON_OVERLOAD_CHECK\n"; |
| |
| // We record each overload check line before emitting because subsequent Inst |
| // definitions may extend the number of permitted types (i.e. augment the |
| // Mask). Use std::map to avoid sorting the table by hash number. |
| struct OverloadInfo { |
| uint64_t Mask; |
| int PtrArgNum; |
| bool HasConstPtr; |
| OverloadInfo() : Mask(0ULL), PtrArgNum(0), HasConstPtr(false) {} |
| }; |
| std::map<std::string, OverloadInfo> OverloadMap; |
| |
| for (auto *Def : Defs) { |
| // If the def has a body (that is, it has Operation DAGs), it won't call |
| // __builtin_neon_* so we don't need to generate a definition for it. |
| if (Def->hasBody()) |
| continue; |
| // Functions with 'a' (the splat code) in the type prototype should not get |
| // their own builtin as they use the non-splat variant. |
| if (Def->hasSplat()) |
| continue; |
| // Functions which have a scalar argument cannot be overloaded, no need to |
| // check them if we are emitting the type checking code. |
| if (Def->protoHasScalar()) |
| continue; |
| |
| uint64_t Mask = 0ULL; |
| Type Ty = Def->getReturnType(); |
| if (Def->getProto()[0] == 'v' || |
| isFloatingPointProtoModifier(Def->getProto()[0])) |
| Ty = Def->getParamType(0); |
| if (Ty.isPointer()) |
| Ty = Def->getParamType(1); |
| |
| Mask |= 1ULL << Ty.getNeonEnum(); |
| |
| // Check if the function has a pointer or const pointer argument. |
| std::string Proto = Def->getProto(); |
| int PtrArgNum = -1; |
| bool HasConstPtr = false; |
| for (unsigned I = 0; I < Def->getNumParams(); ++I) { |
| char ArgType = Proto[I + 1]; |
| if (ArgType == 'c') { |
| HasConstPtr = true; |
| PtrArgNum = I; |
| break; |
| } |
| if (ArgType == 'p') { |
| PtrArgNum = I; |
| break; |
| } |
| } |
| // For sret builtins, adjust the pointer argument index. |
| if (PtrArgNum >= 0 && Def->getReturnType().getNumVectors() > 1) |
| PtrArgNum += 1; |
| |
| std::string Name = Def->getName(); |
| // Omit type checking for the pointer arguments of vld1_lane, vld1_dup, |
| // and vst1_lane intrinsics. Using a pointer to the vector element |
| // type with one of those operations causes codegen to select an aligned |
| // load/store instruction. If you want an unaligned operation, |
| // the pointer argument needs to have less alignment than element type, |
| // so just accept any pointer type. |
| if (Name == "vld1_lane" || Name == "vld1_dup" || Name == "vst1_lane") { |
| PtrArgNum = -1; |
| HasConstPtr = false; |
| } |
| |
| if (Mask) { |
| std::string Name = Def->getMangledName(); |
| OverloadMap.insert(std::make_pair(Name, OverloadInfo())); |
| OverloadInfo &OI = OverloadMap[Name]; |
| OI.Mask |= Mask; |
| OI.PtrArgNum |= PtrArgNum; |
| OI.HasConstPtr = HasConstPtr; |
| } |
| } |
| |
| for (auto &I : OverloadMap) { |
| OverloadInfo &OI = I.second; |
| |
| OS << "case NEON::BI__builtin_neon_" << I.first << ": "; |
| OS << "mask = 0x" << Twine::utohexstr(OI.Mask) << "ULL"; |
| if (OI.PtrArgNum >= 0) |
| OS << "; PtrArgNum = " << OI.PtrArgNum; |
| if (OI.HasConstPtr) |
| OS << "; HasConstPtr = true"; |
| OS << "; break;\n"; |
| } |
| OS << "#endif\n\n"; |
| } |
| |
| void NeonEmitter::genIntrinsicRangeCheckCode(raw_ostream &OS, |
| SmallVectorImpl<Intrinsic *> &Defs) { |
| OS << "#ifdef GET_NEON_IMMEDIATE_CHECK\n"; |
| |
| std::set<std::string> Emitted; |
| |
| for (auto *Def : Defs) { |
| if (Def->hasBody()) |
| continue; |
| // Functions with 'a' (the splat code) in the type prototype should not get |
| // their own builtin as they use the non-splat variant. |
| if (Def->hasSplat()) |
| continue; |
| // Functions which do not have an immediate do not need to have range |
| // checking code emitted. |
| if (!Def->hasImmediate()) |
| continue; |
| if (Emitted.find(Def->getMangledName()) != Emitted.end()) |
| continue; |
| |
| std::string LowerBound, UpperBound; |
| |
| Record *R = Def->getRecord(); |
| if (R->getValueAsBit("isVCVT_N")) { |
| // VCVT between floating- and fixed-point values takes an immediate |
| // in the range [1, 32) for f32 or [1, 64) for f64 or [1, 16) for f16. |
| LowerBound = "1"; |
| if (Def->getBaseType().getElementSizeInBits() == 16 || |
| Def->getName().find('h') != std::string::npos) |
| // VCVTh operating on FP16 intrinsics in range [1, 16) |
| UpperBound = "15"; |
| else if (Def->getBaseType().getElementSizeInBits() == 32) |
| UpperBound = "31"; |
| else |
| UpperBound = "63"; |
| } else if (R->getValueAsBit("isScalarShift")) { |
| // Right shifts have an 'r' in the name, left shifts do not. Convert |
| // instructions have the same bounds and right shifts. |
| if (Def->getName().find('r') != std::string::npos || |
| Def->getName().find("cvt") != std::string::npos) |
| LowerBound = "1"; |
| |
| UpperBound = utostr(Def->getReturnType().getElementSizeInBits() - 1); |
| } else if (R->getValueAsBit("isShift")) { |
| // Builtins which are overloaded by type will need to have their upper |
| // bound computed at Sema time based on the type constant. |
| |
| // Right shifts have an 'r' in the name, left shifts do not. |
| if (Def->getName().find('r') != std::string::npos) |
| LowerBound = "1"; |
| UpperBound = "RFT(TV, true)"; |
| } else if (Def->getClassKind(true) == ClassB) { |
| // ClassB intrinsics have a type (and hence lane number) that is only |
| // known at runtime. |
| if (R->getValueAsBit("isLaneQ")) |
| UpperBound = "RFT(TV, false, true)"; |
| else |
| UpperBound = "RFT(TV, false, false)"; |
| } else { |
| // The immediate generally refers to a lane in the preceding argument. |
| assert(Def->getImmediateIdx() > 0); |
| Type T = Def->getParamType(Def->getImmediateIdx() - 1); |
| UpperBound = utostr(T.getNumElements() - 1); |
| } |
| |
| // Calculate the index of the immediate that should be range checked. |
| unsigned Idx = Def->getNumParams(); |
| if (Def->hasImmediate()) |
| Idx = Def->getGeneratedParamIdx(Def->getImmediateIdx()); |
| |
| OS << "case NEON::BI__builtin_neon_" << Def->getMangledName() << ": " |
| << "i = " << Idx << ";"; |
| if (!LowerBound.empty()) |
| OS << " l = " << LowerBound << ";"; |
| if (!UpperBound.empty()) |
| OS << " u = " << UpperBound << ";"; |
| OS << " break;\n"; |
| |
| Emitted.insert(Def->getMangledName()); |
| } |
| |
| OS << "#endif\n\n"; |
| } |
| |
| /// runHeader - Emit a file with sections defining: |
| /// 1. the NEON section of BuiltinsARM.def and BuiltinsAArch64.def. |
| /// 2. the SemaChecking code for the type overload checking. |
| /// 3. the SemaChecking code for validation of intrinsic immediate arguments. |
| void NeonEmitter::runHeader(raw_ostream &OS) { |
| std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| |
| SmallVector<Intrinsic *, 128> Defs; |
| for (auto *R : RV) |
| createIntrinsic(R, Defs); |
| |
| // Generate shared BuiltinsXXX.def |
| genBuiltinsDef(OS, Defs); |
| |
| // Generate ARM overloaded type checking code for SemaChecking.cpp |
| genOverloadTypeCheckCode(OS, Defs); |
| |
| // Generate ARM range checking code for shift/lane immediates. |
| genIntrinsicRangeCheckCode(OS, Defs); |
| } |
| |
| /// run - Read the records in arm_neon.td and output arm_neon.h. arm_neon.h |
| /// is comprised of type definitions and function declarations. |
| void NeonEmitter::run(raw_ostream &OS) { |
| OS << "/*===---- arm_neon.h - ARM Neon intrinsics " |
| "------------------------------" |
| "---===\n" |
| " *\n" |
| " * Permission is hereby granted, free of charge, to any person " |
| "obtaining " |
| "a copy\n" |
| " * of this software and associated documentation files (the " |
| "\"Software\")," |
| " to deal\n" |
| " * in the Software without restriction, including without limitation " |
| "the " |
| "rights\n" |
| " * to use, copy, modify, merge, publish, distribute, sublicense, " |
| "and/or sell\n" |
| " * copies of the Software, and to permit persons to whom the Software " |
| "is\n" |
| " * furnished to do so, subject to the following conditions:\n" |
| " *\n" |
| " * The above copyright notice and this permission notice shall be " |
| "included in\n" |
| " * all copies or substantial portions of the Software.\n" |
| " *\n" |
| " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, " |
| "EXPRESS OR\n" |
| " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF " |
| "MERCHANTABILITY,\n" |
| " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT " |
| "SHALL THE\n" |
| " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " |
| "OTHER\n" |
| " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, " |
| "ARISING FROM,\n" |
| " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER " |
| "DEALINGS IN\n" |
| " * THE SOFTWARE.\n" |
| " *\n" |
| " *===-----------------------------------------------------------------" |
| "---" |
| "---===\n" |
| " */\n\n"; |
| |
| OS << "#ifndef __ARM_NEON_H\n"; |
| OS << "#define __ARM_NEON_H\n\n"; |
| |
| OS << "#if !defined(__ARM_NEON)\n"; |
| OS << "#error \"NEON support not enabled\"\n"; |
| OS << "#endif\n\n"; |
| |
| OS << "#include <stdint.h>\n\n"; |
| |
| // Emit NEON-specific scalar typedefs. |
| OS << "typedef float float32_t;\n"; |
| OS << "typedef __fp16 float16_t;\n"; |
| |
| OS << "#ifdef __aarch64__\n"; |
| OS << "typedef double float64_t;\n"; |
| OS << "#endif\n\n"; |
| |
| // For now, signedness of polynomial types depends on target |
| OS << "#ifdef __aarch64__\n"; |
| OS << "typedef uint8_t poly8_t;\n"; |
| OS << "typedef uint16_t poly16_t;\n"; |
| OS << "typedef uint64_t poly64_t;\n"; |
| OS << "typedef __uint128_t poly128_t;\n"; |
| OS << "#else\n"; |
| OS << "typedef int8_t poly8_t;\n"; |
| OS << "typedef int16_t poly16_t;\n"; |
| OS << "#endif\n"; |
| |
| // Emit Neon vector typedefs. |
| std::string TypedefTypes( |
| "cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl"); |
| std::vector<TypeSpec> TDTypeVec = TypeSpec::fromTypeSpecs(TypedefTypes); |
| |
| // Emit vector typedefs. |
| bool InIfdef = false; |
| for (auto &TS : TDTypeVec) { |
| bool IsA64 = false; |
| Type T(TS, 'd'); |
| if (T.isDouble() || (T.isPoly() && T.isLong())) |
| IsA64 = true; |
| |
| if (InIfdef && !IsA64) { |
| OS << "#endif\n"; |
| InIfdef = false; |
| } |
| if (!InIfdef && IsA64) { |
| OS << "#ifdef __aarch64__\n"; |
| InIfdef = true; |
| } |
| |
| if (T.isPoly()) |
| OS << "typedef __attribute__((neon_polyvector_type("; |
| else |
| OS << "typedef __attribute__((neon_vector_type("; |
| |
| Type T2 = T; |
| T2.makeScalar(); |
| OS << T.getNumElements() << "))) "; |
| OS << T2.str(); |
| OS << " " << T.str() << ";\n"; |
| } |
| if (InIfdef) |
| OS << "#endif\n"; |
| OS << "\n"; |
| |
| // Emit struct typedefs. |
| InIfdef = false; |
| for (unsigned NumMembers = 2; NumMembers <= 4; ++NumMembers) { |
| for (auto &TS : TDTypeVec) { |
| bool IsA64 = false; |
| Type T(TS, 'd'); |
| if (T.isDouble() || (T.isPoly() && T.isLong())) |
| IsA64 = true; |
| |
| if (InIfdef && !IsA64) { |
| OS << "#endif\n"; |
| InIfdef = false; |
| } |
| if (!InIfdef && IsA64) { |
| OS << "#ifdef __aarch64__\n"; |
| InIfdef = true; |
| } |
| |
| char M = '2' + (NumMembers - 2); |
| Type VT(TS, M); |
| OS << "typedef struct " << VT.str() << " {\n"; |
| OS << " " << T.str() << " val"; |
| OS << "[" << NumMembers << "]"; |
| OS << ";\n} "; |
| OS << VT.str() << ";\n"; |
| OS << "\n"; |
| } |
| } |
| if (InIfdef) |
| OS << "#endif\n"; |
| OS << "\n"; |
| |
| OS << "#define __ai static __inline__ __attribute__((__always_inline__, " |
| "__nodebug__))\n\n"; |
| |
| SmallVector<Intrinsic *, 128> Defs; |
| std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| for (auto *R : RV) |
| createIntrinsic(R, Defs); |
| |
| for (auto *I : Defs) |
| I->indexBody(); |
| |
| llvm::stable_sort(Defs, llvm::deref<std::less<>>()); |
| |
| // Only emit a def when its requirements have been met. |
| // FIXME: This loop could be made faster, but it's fast enough for now. |
| bool MadeProgress = true; |
| std::string InGuard; |
| while (!Defs.empty() && MadeProgress) { |
| MadeProgress = false; |
| |
| for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin(); |
| I != Defs.end(); /*No step*/) { |
| bool DependenciesSatisfied = true; |
| for (auto *II : (*I)->getDependencies()) { |
| if (llvm::is_contained(Defs, II)) |
| DependenciesSatisfied = false; |
| } |
| if (!DependenciesSatisfied) { |
| // Try the next one. |
| ++I; |
| continue; |
| } |
| |
| // Emit #endif/#if pair if needed. |
| if ((*I)->getGuard() != InGuard) { |
| if (!InGuard.empty()) |
| OS << "#endif\n"; |
| InGuard = (*I)->getGuard(); |
| if (!InGuard.empty()) |
| OS << "#if " << InGuard << "\n"; |
| } |
| |
| // Actually generate the intrinsic code. |
| OS << (*I)->generate(); |
| |
| MadeProgress = true; |
| I = Defs.erase(I); |
| } |
| } |
| assert(Defs.empty() && "Some requirements were not satisfied!"); |
| if (!InGuard.empty()) |
| OS << "#endif\n"; |
| |
| OS << "\n"; |
| OS << "#undef __ai\n\n"; |
| OS << "#endif /* __ARM_NEON_H */\n"; |
| } |
| |
| /// run - Read the records in arm_fp16.td and output arm_fp16.h. arm_fp16.h |
| /// is comprised of type definitions and function declarations. |
| void NeonEmitter::runFP16(raw_ostream &OS) { |
| OS << "/*===---- arm_fp16.h - ARM FP16 intrinsics " |
| "------------------------------" |
| "---===\n" |
| " *\n" |
| " * Permission is hereby granted, free of charge, to any person " |
| "obtaining a copy\n" |
| " * of this software and associated documentation files (the " |
| "\"Software\"), to deal\n" |
| " * in the Software without restriction, including without limitation " |
| "the rights\n" |
| " * to use, copy, modify, merge, publish, distribute, sublicense, " |
| "and/or sell\n" |
| " * copies of the Software, and to permit persons to whom the Software " |
| "is\n" |
| " * furnished to do so, subject to the following conditions:\n" |
| " *\n" |
| " * The above copyright notice and this permission notice shall be " |
| "included in\n" |
| " * all copies or substantial portions of the Software.\n" |
| " *\n" |
| " * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, " |
| "EXPRESS OR\n" |
| " * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF " |
| "MERCHANTABILITY,\n" |
| " * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT " |
| "SHALL THE\n" |
| " * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " |
| "OTHER\n" |
| " * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, " |
| "ARISING FROM,\n" |
| " * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER " |
| "DEALINGS IN\n" |
| " * THE SOFTWARE.\n" |
| " *\n" |
| " *===-----------------------------------------------------------------" |
| "---" |
| "---===\n" |
| " */\n\n"; |
| |
| OS << "#ifndef __ARM_FP16_H\n"; |
| OS << "#define __ARM_FP16_H\n\n"; |
| |
| OS << "#include <stdint.h>\n\n"; |
| |
| OS << "typedef __fp16 float16_t;\n"; |
| |
| OS << "#define __ai static __inline__ __attribute__((__always_inline__, " |
| "__nodebug__))\n\n"; |
| |
| SmallVector<Intrinsic *, 128> Defs; |
| std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| for (auto *R : RV) |
| createIntrinsic(R, Defs); |
| |
| for (auto *I : Defs) |
| I->indexBody(); |
| |
| llvm::stable_sort(Defs, llvm::deref<std::less<>>()); |
| |
| // Only emit a def when its requirements have been met. |
| // FIXME: This loop could be made faster, but it's fast enough for now. |
| bool MadeProgress = true; |
| std::string InGuard; |
| while (!Defs.empty() && MadeProgress) { |
| MadeProgress = false; |
| |
| for (SmallVector<Intrinsic *, 128>::iterator I = Defs.begin(); |
| I != Defs.end(); /*No step*/) { |
| bool DependenciesSatisfied = true; |
| for (auto *II : (*I)->getDependencies()) { |
| if (llvm::is_contained(Defs, II)) |
| DependenciesSatisfied = false; |
| } |
| if (!DependenciesSatisfied) { |
| // Try the next one. |
| ++I; |
| continue; |
| } |
| |
| // Emit #endif/#if pair if needed. |
| if ((*I)->getGuard() != InGuard) { |
| if (!InGuard.empty()) |
| OS << "#endif\n"; |
| InGuard = (*I)->getGuard(); |
| if (!InGuard.empty()) |
| OS << "#if " << InGuard << "\n"; |
| } |
| |
| // Actually generate the intrinsic code. |
| OS << (*I)->generate(); |
| |
| MadeProgress = true; |
| I = Defs.erase(I); |
| } |
| } |
| assert(Defs.empty() && "Some requirements were not satisfied!"); |
| if (!InGuard.empty()) |
| OS << "#endif\n"; |
| |
| OS << "\n"; |
| OS << "#undef __ai\n\n"; |
| OS << "#endif /* __ARM_FP16_H */\n"; |
| } |
| |
| void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) { |
| NeonEmitter(Records).run(OS); |
| } |
| |
| void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) { |
| NeonEmitter(Records).runFP16(OS); |
| } |
| |
| void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) { |
| NeonEmitter(Records).runHeader(OS); |
| } |
| |
| void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) { |
| llvm_unreachable("Neon test generation no longer implemented!"); |
| } |