blob: 19e2a6a27020d7cb10dab6ce05f8f428b39882c1 [file] [log] [blame]
//===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===-------------------------------------------------------------===//
#include "AMDGPUAsanInstrumentation.h"
#define DEBUG_TYPE "amdgpu-asan-instrumentation"
using namespace llvm;
namespace llvm {
namespace AMDGPU {
static uint64_t getRedzoneSizeForScale(int AsanScale) {
// Redzone used for stack and globals is at least 32 bytes.
// For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
return std::max(32U, 1U << AsanScale);
}
static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) {
return getRedzoneSizeForScale(AsanScale);
}
uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) {
constexpr uint64_t kMaxRZ = 1 << 18;
const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale);
uint64_t RZ = 0;
if (SizeInBytes <= MinRZ / 2) {
// Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
// at least 32 bytes, optimize when SizeInBytes is less than or equal to
// half of MinRZ.
RZ = MinRZ - SizeInBytes;
} else {
// Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ);
// Round up to multiple of MinRZ.
if (SizeInBytes % MinRZ)
RZ += MinRZ - (SizeInBytes % MinRZ);
}
assert((RZ + SizeInBytes) % MinRZ == 0);
return RZ;
}
static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
size_t Res = llvm::countr_zero(TypeSize / 8);
return Res;
}
static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
Value *Cond, bool Recover) {
Value *ReportCond = Cond;
if (!Recover) {
auto *Ballot =
IRB.CreateIntrinsic(Intrinsic::amdgcn_ballot, IRB.getInt64Ty(), {Cond});
ReportCond = IRB.CreateIsNotNull(Ballot);
}
auto *Trm = SplitBlockAndInsertIfThen(
ReportCond, &*IRB.GetInsertPoint(), false,
MDBuilder(M.getContext()).createUnlikelyBranchWeights());
Trm->getParent()->setName("asan.report");
if (Recover)
return Trm;
Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
IRB.SetInsertPoint(Trm);
return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {});
}
static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
Value *AddrLong, Value *ShadowValue,
uint32_t TypeStoreSize, int AsanScale) {
uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale;
// Addr & (Granularity - 1)
Value *LastAccessedByte =
IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));
// (Addr & (Granularity - 1)) + size - 1
if (TypeStoreSize / 8 > 1)
LastAccessedByte = IRB.CreateAdd(
LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1));
// (uint8_t) ((Addr & (Granularity-1)) + size - 1)
LastAccessedByte =
IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false);
// ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);
}
static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB,
Type *IntptrTy, Instruction *InsertBefore,
Value *Addr, bool IsWrite,
size_t AccessSizeIndex,
Value *SizeArgument, bool Recover) {
IRB.SetInsertPoint(InsertBefore);
CallInst *Call = nullptr;
SmallString<128> kAsanReportErrorTemplate{"__asan_report_"};
SmallString<64> TypeStr{IsWrite ? "store" : "load"};
SmallString<64> EndingStr{Recover ? "_noabort" : ""};
SmallString<128> AsanErrorCallbackSizedString;
raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString);
AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n"
<< EndingStr;
SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
AttributeList AL2;
FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction(
AsanErrorCallbackSizedOS.str(),
FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);
SmallVector<Type *, 2> Args1{1, IntptrTy};
AttributeList AL1;
SmallString<128> AsanErrorCallbackString;
raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString);
AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr
<< (1ULL << AccessSizeIndex) << EndingStr;
FunctionCallee AsanErrorCallback = M.getOrInsertFunction(
AsanErrorCallbackOS.str(),
FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);
if (SizeArgument) {
Call = IRB.CreateCall(AsanErrorCallbackSized, {Addr, SizeArgument});
} else {
Call = IRB.CreateCall(AsanErrorCallback, Addr);
}
Call->setCannotMerge();
return Call;
}
static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
Value *Shadow, int AsanScale, uint32_t AsanOffset) {
// Shadow >> scale
Shadow = IRB.CreateLShr(Shadow, AsanScale);
if (AsanOffset == 0)
return Shadow;
// (Shadow >> scale) | offset
Value *ShadowBase = ConstantInt::get(IntptrTy, AsanOffset);
return IRB.CreateAdd(Shadow, ShadowBase);
}
static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB,
Instruction *OrigIns,
Instruction *InsertBefore, Value *Addr,
Align Alignment, uint32_t TypeStoreSize,
bool IsWrite, Value *SizeArgument,
bool UseCalls, bool Recover, int AsanScale,
int AsanOffset) {
Type *AddrTy = Addr->getType();
Type *IntptrTy = M.getDataLayout().getIntPtrType(
M.getContext(), AddrTy->getPointerAddressSpace());
IRB.SetInsertPoint(InsertBefore);
size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize);
Type *ShadowTy = IntegerType::get(M.getContext(),
std::max(8U, TypeStoreSize >> AsanScale));
Type *ShadowPtrTy = PointerType::get(M.getContext(), 0);
Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
Value *ShadowPtr =
memToShadow(M, IRB, IntptrTy, AddrLong, AsanScale, AsanOffset);
const uint64_t ShadowAlign =
std::max<uint64_t>(Alignment.value() >> AsanScale, 1);
Value *ShadowValue = IRB.CreateAlignedLoad(
ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign));
Value *Cmp = IRB.CreateIsNotNull(ShadowValue);
auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue,
TypeStoreSize, AsanScale);
Cmp = IRB.CreateAnd(Cmp, Cmp2);
Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cmp, Recover);
Instruction *Crash =
generateCrashCode(M, IRB, IntptrTy, CrashTerm, AddrLong, IsWrite,
AccessSizeIndex, SizeArgument, Recover);
Crash->setDebugLoc(OrigIns->getDebugLoc());
}
void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns,
Instruction *InsertBefore, Value *Addr, Align Alignment,
TypeSize TypeStoreSize, bool IsWrite,
Value *SizeArgument, bool UseCalls, bool Recover,
int AsanScale, int AsanOffset) {
if (!TypeStoreSize.isScalable()) {
unsigned Granularity = 1 << AsanScale;
const auto FixedSize = TypeStoreSize.getFixedValue();
switch (FixedSize) {
case 8:
case 16:
case 32:
case 64:
case 128:
if (Alignment.value() >= Granularity ||
Alignment.value() >= FixedSize / 8)
return instrumentAddressImpl(
M, IRB, OrigIns, InsertBefore, Addr, Alignment, FixedSize, IsWrite,
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
}
}
// Instrument unusual size or unusual alignment.
IRB.SetInsertPoint(InsertBefore);
Type *AddrTy = Addr->getType();
Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy);
Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize);
Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3));
Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
Value *SizeMinusOne = IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, -1));
Value *LastByte =
IRB.CreateIntToPtr(IRB.CreateAdd(AddrLong, SizeMinusOne), AddrTy);
instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, {}, 8, IsWrite,
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, LastByte, {}, 8, IsWrite,
SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
}
void getInterestingMemoryOperands(
Module &M, Instruction *I,
SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
const DataLayout &DL = M.getDataLayout();
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
LI->getType(), LI->getAlign());
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
SI->getValueOperand()->getType(), SI->getAlign());
} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
RMW->getValOperand()->getType(), std::nullopt);
} else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
XCHG->getCompareOperand()->getType(),
std::nullopt);
} else if (auto *CI = dyn_cast<CallInst>(I)) {
switch (CI->getIntrinsicID()) {
case Intrinsic::masked_load:
case Intrinsic::masked_store:
case Intrinsic::masked_gather:
case Intrinsic::masked_scatter: {
bool IsWrite = CI->getType()->isVoidTy();
// Masked store has an initial operand for the value.
unsigned OpOffset = IsWrite ? 1 : 0;
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
MaybeAlign Alignment = Align(1);
// Otherwise no alignment guarantees. We probably got Undef.
if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
Alignment = Op->getMaybeAlignValue();
Value *Mask = CI->getOperand(2 + OpOffset);
Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);
break;
}
case Intrinsic::masked_expandload:
case Intrinsic::masked_compressstore: {
bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
unsigned OpOffset = IsWrite ? 1 : 0;
auto *BasePtr = CI->getOperand(OpOffset);
MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
IRBuilder<> IB(I);
Value *Mask = CI->getOperand(1 + OpOffset);
Type *IntptrTy = M.getDataLayout().getIntPtrType(
M.getContext(), BasePtr->getType()->getPointerAddressSpace());
// Use the popcount of Mask as the effective vector length.
Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty));
Value *ExtMask = IB.CreateZExt(Mask, ExtTy);
Value *EVL = IB.CreateAddReduce(ExtMask);
Value *TrueMask = ConstantInt::get(Mask->getType(), 1);
Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask,
EVL);
break;
}
case Intrinsic::vp_load:
case Intrinsic::vp_store:
case Intrinsic::experimental_vp_strided_load:
case Intrinsic::experimental_vp_strided_store: {
auto *VPI = cast<VPIntrinsic>(CI);
unsigned IID = CI->getIntrinsicID();
bool IsWrite = CI->getType()->isVoidTy();
unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(DL);
Value *Stride = nullptr;
if (IID == Intrinsic::experimental_vp_strided_store ||
IID == Intrinsic::experimental_vp_strided_load) {
Stride = VPI->getOperand(PtrOpNo + 1);
// Use the pointer alignment as the element alignment if the stride is a
// mutiple of the pointer alignment. Otherwise, the element alignment
// should be Align(1).
unsigned PointerAlign = Alignment.valueOrOne().value();
if (!isa<ConstantInt>(Stride) ||
cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
Alignment = Align(1);
}
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
VPI->getMaskParam(), VPI->getVectorLengthParam(),
Stride);
break;
}
case Intrinsic::vp_gather:
case Intrinsic::vp_scatter: {
auto *VPI = cast<VPIntrinsic>(CI);
unsigned IID = CI->getIntrinsicID();
bool IsWrite = IID == Intrinsic::vp_scatter;
unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
MaybeAlign Alignment = VPI->getPointerAlignment();
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
VPI->getMaskParam(),
VPI->getVectorLengthParam());
break;
}
case Intrinsic::amdgcn_raw_buffer_load:
case Intrinsic::amdgcn_raw_ptr_buffer_load:
case Intrinsic::amdgcn_raw_buffer_load_format:
case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
case Intrinsic::amdgcn_raw_tbuffer_load:
case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
case Intrinsic::amdgcn_struct_buffer_load:
case Intrinsic::amdgcn_struct_ptr_buffer_load:
case Intrinsic::amdgcn_struct_buffer_load_format:
case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
case Intrinsic::amdgcn_struct_tbuffer_load:
case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
case Intrinsic::amdgcn_s_buffer_load:
case Intrinsic::amdgcn_global_load_tr_b64:
case Intrinsic::amdgcn_global_load_tr_b128: {
unsigned PtrOpNo = 0;
bool IsWrite = false;
Type *Ty = CI->getType();
Value *Ptr = CI->getArgOperand(PtrOpNo);
MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
break;
}
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
case Intrinsic::amdgcn_raw_buffer_store:
case Intrinsic::amdgcn_raw_ptr_buffer_store:
case Intrinsic::amdgcn_raw_buffer_store_format:
case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
case Intrinsic::amdgcn_struct_buffer_store:
case Intrinsic::amdgcn_struct_ptr_buffer_store:
case Intrinsic::amdgcn_struct_buffer_store_format:
case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
unsigned PtrOpNo = 1;
bool IsWrite = true;
Value *Ptr = CI->getArgOperand(PtrOpNo);
Type *Ty = Ptr->getType();
MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
break;
}
default:
for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
if (Type *Ty = CI->getParamByRefType(ArgNo)) {
Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
} else if (Type *Ty = CI->getParamByValType(ArgNo)) {
Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
}
}
}
}
}
} // end namespace AMDGPU
} // end namespace llvm