blob: 9bfac2ac9167ef7d2abd836efec3444df73baf38 [file] [log] [blame]
//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// Populates the VFABI attribute with the scalar-to-vector mappings
// from the TargetLibraryInfo.
#include "llvm/Transforms/Utils/InjectTLIMappings.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/VFABIDemangler.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
using namespace llvm;
#define DEBUG_TYPE "inject-tli-mappings"
"Number of calls in which the mappings have been injected.");
"Number of function declarations that have been added.");
"Number of `@llvm.compiler.used` operands that have been added.");
/// A helper function that adds the vector variant declaration for vectorizing
/// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
/// mapping, TLI provides a VABI prefix, which contains all information required
/// to create vector function declaration.
static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
const VecDesc *VD) {
Module *M = CI.getModule();
FunctionType *ScalarFTy = CI.getFunctionType();
assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");
const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(
VD->getVectorFunctionABIVariantString(), ScalarFTy);
assert(Info && "Failed to demangle vector variant");
assert(Info->Shape.VF == VF && "Mangled name does not match VF");
const StringRef VFName = VD->getVectorFnName();
FunctionType *VectorFTy = VFABI::createFunctionType(*Info, ScalarFTy);
Function *VecFunc =
Function::Create(VectorFTy, Function::ExternalLinkage, VFName, M);
LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
<< "` of type " << *VectorFTy << "\n");
// Make function declaration (without a body) "sticky" in the IR by
// listing it in the @llvm.compiler.used intrinsic.
assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "
"only on declarations.");
appendToCompilerUsed(*M, {VecFunc});
LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
<< "` to `@llvm.compiler.used`.\n");
static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
// This is needed to make sure we don't query the TLI for calls to
// bitcast of function pointers, like `%call = call i32 (i32*, ...)
// bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
// as such calls make the `isFunctionVectorizable` raise an
// exception.
if (CI.isNoBuiltin() || !CI.getCalledFunction())
StringRef ScalarName = CI.getCalledFunction()->getName();
// Nothing to be done if the TLI thinks the function is not
// vectorizable.
if (!TLI.isFunctionVectorizable(ScalarName))
SmallVector<std::string, 8> Mappings;
VFABI::getVectorVariantNames(CI, Mappings);
Module *M = CI.getModule();
const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
const VecDesc *VD = TLI.getVectorMappingInfo(ScalarName, VF, Predicate);
if (VD && !VD->getVectorFnName().empty()) {
std::string MangledName = VD->getVectorFunctionABIVariantString();
if (!OriginalSetOfMappings.count(MangledName)) {
Function *VariantF = M->getFunction(VD->getVectorFnName());
if (!VariantF)
addVariantDeclaration(CI, VF, VD);
// All VFs in the TLI are powers of 2.
ElementCount WidestFixedVF, WidestScalableVF;
TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
for (bool Predicated : {false, true}) {
for (ElementCount VF = ElementCount::getFixed(2);
ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)
AddVariantDecl(VF, Predicated);
for (ElementCount VF = ElementCount::getScalable(2);
ElementCount::isKnownLE(VF, WidestScalableVF); VF *= 2)
AddVariantDecl(VF, Predicated);
VFABI::setVectorVariantNames(&CI, Mappings);
static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
for (auto &I : instructions(F))
if (auto CI = dyn_cast<CallInst>(&I))
addMappingsFromTLI(TLI, *CI);
// Even if the pass adds IR attributes, the analyses are preserved.
return false;
// New pass manager implementation.
PreservedAnalyses InjectTLIMappings::run(Function &F,
FunctionAnalysisManager &AM) {
const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
runImpl(TLI, F);
// Even if the pass adds IR attributes, the analyses are preserved.
return PreservedAnalyses::all();