Google Git
Sign in
llvm/llvm-project/24a5231188bf92e2ce9fe38e29e4cb75bab51414/./clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
blob: 3c48b8b67d9ee0c00fa49b26d8a619ae7023ddb0 [file]
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit Builtin calls as CIR or a function call to be
// later resolved.
//
//===----------------------------------------------------------------------===//
#include "CIRGenCall.h"
#include "CIRGenFunction.h"
#include "CIRGenModule.h"
#include "CIRGenValue.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/Value.h"
#include "mlir/Support/LLVM.h"
#include "clang/AST/DeclBase.h"
#include "clang/AST/Expr.h"
#include "clang/AST/GlobalDecl.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/OperatorKinds.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/Support/ErrorHandling.h"
using namespace clang;
using namespace clang::CIRGen;
using namespace llvm;
static bool shouldEmitBuiltinAsIR(unsigned builtinID,
const Builtin::Context &bi,
const CIRGenFunction &cgf) {
if (!cgf.cgm.getLangOpts().MathErrno &&
cgf.curFPFeatures.getExceptionMode() ==
LangOptions::FPExceptionModeKind::FPE_Ignore &&
!cgf.cgm.getTargetCIRGenInfo().supportsLibCall()) {
switch (builtinID) {
default:
return false;
case Builtin::BIlogbf:
case Builtin::BI__builtin_logbf:
case Builtin::BIlogb:
case Builtin::BI__builtin_logb:
case Builtin::BIscalbnf:
case Builtin::BI__builtin_scalbnf:
case Builtin::BIscalbn:
case Builtin::BI__builtin_scalbn:
return true;
}
}
return false;
}
static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,
const CallExpr *e, mlir::Operation *calleeValue) {
CIRGenCallee callee = CIRGenCallee::forDirect(calleeValue, GlobalDecl(fd));
return cgf.emitCall(e->getCallee()->getType(), callee, e, ReturnValueSlot());
}
template <typename Op, typename... Args>
static mlir::Value createBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
mlir::Value arg, Args... args) {
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(e->getSourceRange());
auto op = Op::create(builder, loc, arg, args...);
mlir::Value result = op.getResult();
mlir::Type resultTy = cgf.convertType(e->getType());
if (resultTy != result.getType())
result = builder.createIntCast(result, resultTy);
return result;
}
template <typename Op, typename... Args>
static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
Args... args) {
mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
return RValue::get(createBuiltinBitOp<Op>(cgf, e, arg, args...));
}
/// Emit a clz/ctz bit op with optional fallback for __builtin_c[lt]zg.
/// When a fallback is present, the result is the fallback value if the input is
/// zero, otherwise the bit count.
template <typename Op>
static RValue emitBuiltinBitOpWithFallback(CIRGenFunction &cgf,
const CallExpr *e) {
bool hasFallback = e->getNumArgs() > 1;
bool poisonZero = hasFallback || cgf.getTarget().isCLZForZeroUndef();
if (!hasFallback) {
assert(!cir::MissingFeatures::builtinCheckKind());
return emitBuiltinBitOp<Op>(cgf, e, poisonZero);
}
assert(!cir::MissingFeatures::builtinBitCountExpr());
mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
mlir::Value result = createBuiltinBitOp<Op>(cgf, e, arg, poisonZero);
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(e->getSourceRange());
mlir::Value zero = builder.getNullValue(arg.getType(), loc);
mlir::Value isZero =
builder.createCompare(loc, cir::CmpOpKind::eq, arg, zero);
mlir::Value fallbackValue = cgf.emitScalarExpr(e->getArg(1));
return RValue::get(builder.createSelect(loc, isZero, fallbackValue, result));
}
/// Emit the conversions required to turn the given value into an
/// integer of the given size.
static mlir::Value emitToInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
cir::IntType intType) {
v = cgf.emitToMemory(v, t);
if (mlir::isa<cir::PointerType>(v.getType()))
return cgf.getBuilder().createPtrToInt(v, intType);
assert(v.getType() == intType);
return v;
}
static mlir::Value emitFromInt(CIRGenFunction &cgf, mlir::Value v, QualType t,
mlir::Type resultType) {
v = cgf.emitFromMemory(v, t);
if (mlir::isa<cir::PointerType>(resultType))
return cgf.getBuilder().createIntToPtr(v, resultType);
assert(v.getType() == resultType);
return v;
}
static mlir::Value emitSignBit(mlir::Location loc, CIRGenFunction &cgf,
mlir::Value val) {
assert(!::cir::MissingFeatures::isPPC_FP128Ty());
cir::SignBitOp returnValue = cgf.getBuilder().createSignBit(loc, val);
return returnValue->getResult(0);
}
static Address checkAtomicAlignment(CIRGenFunction &cgf, const CallExpr *e) {
ASTContext &astContext = cgf.getContext();
Address ptr = cgf.emitPointerWithAlignment(e->getArg(0));
unsigned bytes =
mlir::isa<cir::PointerType>(ptr.getElementType())
? astContext.getTypeSizeInChars(astContext.VoidPtrTy).getQuantity()
: cgf.cgm.getDataLayout().getTypeSizeInBits(ptr.getElementType()) /
cgf.cgm.getASTContext().getCharWidth();
unsigned align = ptr.getAlignment().getQuantity();
if (align % bytes != 0) {
DiagnosticsEngine &diags = cgf.cgm.getDiags();
diags.Report(e->getBeginLoc(), diag::warn_sync_op_misaligned);
// Force address to be at least naturally-aligned.
return ptr.withAlignment(CharUnits::fromQuantity(bytes));
}
return ptr;
}
/// Utility to insert an atomic instruction based on Intrinsic::ID
/// and the expression node.
static mlir::Value makeBinaryAtomicValue(
CIRGenFunction &cgf, cir::AtomicFetchKind kind, const CallExpr *expr,
mlir::Type *originalArgType = nullptr,
mlir::Value *emittedArgValue = nullptr,
cir::MemOrder ordering = cir::MemOrder::SequentiallyConsistent) {
QualType type = expr->getType();
QualType ptrType = expr->getArg(0)->getType();
assert(ptrType->isPointerType());
assert(
cgf.getContext().hasSameUnqualifiedType(type, ptrType->getPointeeType()));
assert(cgf.getContext().hasSameUnqualifiedType(type,
expr->getArg(1)->getType()));
Address destAddr = checkAtomicAlignment(cgf, expr);
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Value val = cgf.emitScalarExpr(expr->getArg(1));
mlir::Type valueType = val.getType();
mlir::Value destValue = destAddr.emitRawPointer();
if (ptrType->getPointeeType()->isPointerType()) {
// Pointer to pointer
// `cir.atomic.fetch` expects a pointer to an integer type, so we cast
// ptr<ptr<T>> to ptr<intPtrSize>
cir::IntType ptrSizeInt =
builder.getSIntNTy(cgf.getContext().getTypeSize(ptrType));
destValue =
builder.createBitcast(destValue, builder.getPointerTo(ptrSizeInt));
val = emitToInt(cgf, val, type, ptrSizeInt);
} else {
// Pointer to integer type
cir::IntType intType =
ptrType->getPointeeType()->isUnsignedIntegerType()
? builder.getUIntNTy(cgf.getContext().getTypeSize(type))
: builder.getSIntNTy(cgf.getContext().getTypeSize(type));
val = emitToInt(cgf, val, type, intType);
}
// This output argument is needed for post atomic fetch operations
// that calculate the result of the operation as return value of
// <binop>_and_fetch builtins. The `AtomicFetch` operation only updates the
// memory location and returns the old value.
if (emittedArgValue) {
*emittedArgValue = val;
assert(originalArgType != nullptr &&
"originalArgType must be provided when emittedArgValue is set");
*originalArgType = valueType;
}
auto rmwi = cir::AtomicFetchOp::create(
builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind,
ordering, cir::SyncScopeKind::System, false, /* is volatile */
true); /* fetch first */
return rmwi->getResult(0);
}
static RValue emitBinaryAtomic(CIRGenFunction &cgf,
cir::AtomicFetchKind atomicOpkind,
const CallExpr *e) {
return RValue::get(makeBinaryAtomicValue(cgf, atomicOpkind, e));
}
template <typename BinOp>
static RValue emitBinaryAtomicPost(CIRGenFunction &cgf,
cir::AtomicFetchKind atomicOpkind,
const CallExpr *e, bool invert = false) {
mlir::Value emittedArgValue;
mlir::Type originalArgType;
clang::QualType typ = e->getType();
mlir::Value result = makeBinaryAtomicValue(
cgf, atomicOpkind, e, &originalArgType, &emittedArgValue);
clang::CIRGen::CIRGenBuilderTy &builder = cgf.getBuilder();
result = BinOp::create(builder, result.getLoc(), result, emittedArgValue);
if (invert)
result = builder.createNot(result);
result = emitFromInt(cgf, result, typ, originalArgType);
return RValue::get(result);
}
static void emitAtomicFenceOp(CIRGenFunction &cgf, const CallExpr *expr,
cir::SyncScopeKind syncScope) {
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) {
cir::AtomicFenceOp::create(
builder, loc, memOrder,
cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(), syncScope));
};
cgf.emitAtomicExprWithMemOrder(expr->getArg(0), /*isStore*/ false,
/*isLoad*/ false, /*isFence*/ true,
emitAtomicOpCallBackFn);
}
// Emit a runtime call to bool __atomic_is_lock_free(size_t size, void *ptr).
// For the __c11 builtin the pointer is null, since an _Atomic object is always
// suitably aligned.
static RValue emitAtomicIsLockFree(CIRGenFunction &cgf, const CallExpr *e,
unsigned builtinID) {
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(e->getExprLoc());
mlir::Type sizeTy = cgf.convertType(cgf.getContext().getSizeType());
mlir::Value size = cgf.emitScalarExpr(e->getArg(0));
mlir::Value ptr;
if (builtinID == Builtin::BI__atomic_is_lock_free)
ptr = builder.createBitcast(cgf.emitScalarExpr(e->getArg(1)),
builder.getVoidPtrTy());
else
ptr = builder.getNullPtr(builder.getVoidPtrTy(), loc);
cir::FuncOp func = cgf.cgm.createRuntimeFunction(
cir::FuncType::get({sizeTy, builder.getVoidPtrTy()}, builder.getBoolTy()),
"__atomic_is_lock_free");
return RValue::get(
builder.createCallOp(loc, func, mlir::ValueRange{size, ptr}).getResult());
}
namespace {
struct WidthAndSignedness {
unsigned width;
bool isSigned;
};
} // namespace
static WidthAndSignedness
getIntegerWidthAndSignedness(const clang::ASTContext &astContext,
const clang::QualType type) {
assert(type->isIntegerType() && "Given type is not an integer.");
unsigned width = type->isBooleanType() ? 1
: type->isBitIntType() ? astContext.getIntWidth(type)
: astContext.getTypeInfo(type).Width;
bool isSigned = type->isSignedIntegerType();
return {width, isSigned};
}
/// Create a checked overflow arithmetic op and return its result and overflow
/// flag.
template <typename OpTy>
static std::pair<mlir::Value, mlir::Value>
emitOverflowOp(CIRGenBuilderTy &builder, mlir::Location loc,
mlir::Type resultTy, mlir::Value lhs, mlir::Value rhs) {
auto op = OpTy::create(builder, loc, resultTy, lhs, rhs);
return {op.getResult(), op.getOverflow()};
}
// Given one or more integer types, this function produces an integer type that
// encompasses them: any value in one of the given types could be expressed in
// the encompassing type.
static struct WidthAndSignedness
EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> types) {
assert(types.size() > 0 && "Empty list of types.");
// If any of the given types is signed, we must return a signed type.
bool isSigned = llvm::any_of(types, [](const auto &t) { return t.isSigned; });
// The encompassing type must have a width greater than or equal to the width
// of the specified types. Additionally, if the encompassing type is signed,
// its width must be strictly greater than the width of any unsigned types
// given.
unsigned width = 0;
for (const auto &type : types)
width = std::max(width, type.width + (isSigned && !type.isSigned));
return {width, isSigned};
}
RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) {
mlir::Value input = emitScalarExpr(e->getArg(0));
mlir::Value amount = emitScalarExpr(e->getArg(1));
// TODO(cir): MSVC flavor bit rotate builtins use different types for input
// and amount, but cir.rotate requires them to have the same type. Cast amount
// to the type of input when necessary.
assert(!cir::MissingFeatures::msvcBuiltins());
auto r = cir::RotateOp::create(builder, getLoc(e->getSourceRange()), input,
amount, isRotateLeft);
return RValue::get(r);
}
template <class Operation>
static RValue emitUnaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
const CallExpr &e) {
mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(cgf, &e);
assert(!cir::MissingFeatures::fpConstraints());
auto call =
Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);
return RValue::get(call->getResult(0));
}
template <class Operation>
static RValue emitUnaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));
auto call =
Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);
return RValue::get(call->getResult(0));
}
template <typename Op>
static RValue emitUnaryMaybeConstrainedFPToIntBuiltin(CIRGenFunction &cgf,
const CallExpr &e) {
mlir::Type resultType = cgf.convertType(e.getType());
mlir::Value src = cgf.emitScalarExpr(e.getArg(0));
assert(!cir::MissingFeatures::fpConstraints());
auto call = Op::create(cgf.getBuilder(), src.getLoc(), resultType, src);
return RValue::get(call->getResult(0));
}
template <typename Op>
static RValue emitBinaryFPBuiltin(CIRGenFunction &cgf, const CallExpr &e) {
mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
mlir::Location loc = cgf.getLoc(e.getExprLoc());
mlir::Type ty = cgf.convertType(e.getType());
auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1);
return RValue::get(call->getResult(0));
}
template <typename Op>
static mlir::Value emitBinaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
const CallExpr &e) {
mlir::Value arg0 = cgf.emitScalarExpr(e.getArg(0));
mlir::Value arg1 = cgf.emitScalarExpr(e.getArg(1));
mlir::Location loc = cgf.getLoc(e.getExprLoc());
mlir::Type ty = cgf.convertType(e.getType());
assert(!cir::MissingFeatures::fpConstraints());
auto call = Op::create(cgf.getBuilder(), loc, ty, arg0, arg1);
return call->getResult(0);
}
static RValue errorBuiltinNYI(CIRGenFunction &cgf, const CallExpr *e,
unsigned builtinID) {
if (cgf.getContext().BuiltinInfo.isLibFunction(builtinID)) {
cgf.cgm.errorNYI(
e->getSourceRange(),
std::string("unimplemented X86 library function builtin call: ") +
cgf.getContext().BuiltinInfo.getName(builtinID));
} else {
cgf.cgm.errorNYI(e->getSourceRange(),
std::string("unimplemented X86 builtin call: ") +
cgf.getContext().BuiltinInfo.getName(builtinID));
}
return cgf.getUndefRValue(e->getType());
}
static RValue emitBuiltinAlloca(CIRGenFunction &cgf, const CallExpr *e,
unsigned builtinID) {
assert(builtinID == Builtin::BI__builtin_alloca ||
builtinID == Builtin::BI__builtin_alloca_uninitialized ||
builtinID == Builtin::BIalloca || builtinID == Builtin::BI_alloca);
// Get alloca size input
mlir::Value size = cgf.emitScalarExpr(e->getArg(0));
// The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
const TargetInfo &ti = cgf.getContext().getTargetInfo();
const CharUnits suitableAlignmentInBytes =
cgf.getContext().toCharUnitsFromBits(ti.getSuitableAlign());
// Emit the alloca op with type `u8 *` to match the semantics of
// `llvm.alloca`. We later bitcast the type to `void *` to match the
// semantics of C/C++
// FIXME(cir): It may make sense to allow AllocaOp of type `u8` to return a
// pointer of type `void *`. This will require a change to the allocaOp
// verifier.
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Value allocaAddr = builder.createAlloca(
cgf.getLoc(e->getSourceRange()), builder.getUInt8PtrTy(),
builder.getUInt8Ty(), "bi_alloca", suitableAlignmentInBytes, size);
// Initialize the allocated buffer if required.
if (builtinID != Builtin::BI__builtin_alloca_uninitialized) {
// Initialize the alloca with the given size and alignment according to
// the lang opts. Only the trivial non-initialization is supported for
// now.
switch (cgf.getLangOpts().getTrivialAutoVarInit()) {
case LangOptions::TrivialAutoVarInitKind::Uninitialized:
// Nothing to initialize.
break;
case LangOptions::TrivialAutoVarInitKind::Zero:
case LangOptions::TrivialAutoVarInitKind::Pattern:
cgf.cgm.errorNYI("trivial auto var init");
break;
}
}
// An alloca will always return a pointer to the alloca (stack) address
// space. This address space need not be the same as the AST / Language
// default (e.g. in C / C++ auto vars are in the generic address space). At
// the AST level this is handled within CreateTempAlloca et al., but for the
// builtin / dynamic alloca we have to handle it here.
if (!cir::isMatchingAddressSpace(
cgf.getCIRAllocaAddressSpace(),
e->getType()->getPointeeType().getAddressSpace())) {
cgf.cgm.errorNYI(e->getSourceRange(),
"Address Space Cast for builtin alloca");
}
// Bitcast the alloca to the expected type.
return RValue::get(builder.createBitcast(
allocaAddr, builder.getVoidPtrTy(cgf.getCIRAllocaAddressSpace())));
}
static bool shouldCIREmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
unsigned builtinID) {
std::optional<bool> errnoOverriden;
// ErrnoOverriden is true if math-errno is overriden via the
// '#pragma float_control(precise, on)'. This pragma disables fast-math,
// which implies math-errno.
if (e->hasStoredFPFeatures()) {
FPOptionsOverride op = e->getFPFeatures();
if (op.hasMathErrnoOverride())
errnoOverriden = op.getMathErrnoOverride();
}
// True if 'attribute__((optnone))' is used. This attribute overrides
// fast-math which implies math-errno.
bool optNone =
cgf.curFuncDecl && cgf.curFuncDecl->hasAttr<OptimizeNoneAttr>();
bool isOptimizationEnabled = cgf.cgm.getCodeGenOpts().OptimizationLevel != 0;
bool generateFPMathIntrinsics =
cgf.getContext().BuiltinInfo.shouldGenerateFPMathIntrinsic(
builtinID, cgf.cgm.getTriple(), errnoOverriden,
cgf.getLangOpts().MathErrno, optNone, isOptimizationEnabled);
return generateFPMathIntrinsics;
}
static RValue tryEmitFPMathIntrinsic(CIRGenFunction &cgf, const CallExpr *e,
unsigned builtinID) {
assert(!cir::MissingFeatures::fastMathFlags());
switch (builtinID) {
case Builtin::BIacos:
case Builtin::BIacosf:
case Builtin::BIacosl:
case Builtin::BI__builtin_acos:
case Builtin::BI__builtin_acosf:
case Builtin::BI__builtin_acosf16:
case Builtin::BI__builtin_acosl:
case Builtin::BI__builtin_acosf128:
case Builtin::BI__builtin_elementwise_acos:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ACosOp>(cgf, *e);
case Builtin::BIasin:
case Builtin::BIasinf:
case Builtin::BIasinl:
case Builtin::BI__builtin_asin:
case Builtin::BI__builtin_asinf:
case Builtin::BI__builtin_asinf16:
case Builtin::BI__builtin_asinl:
case Builtin::BI__builtin_asinf128:
case Builtin::BI__builtin_elementwise_asin:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ASinOp>(cgf, *e);
case Builtin::BIatan:
case Builtin::BIatanf:
case Builtin::BIatanl:
case Builtin::BI__builtin_atan:
case Builtin::BI__builtin_atanf:
case Builtin::BI__builtin_atanf16:
case Builtin::BI__builtin_atanl:
case Builtin::BI__builtin_atanf128:
case Builtin::BI__builtin_elementwise_atan:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ATanOp>(cgf, *e);
case Builtin::BIatan2:
case Builtin::BIatan2f:
case Builtin::BIatan2l:
case Builtin::BI__builtin_atan2:
case Builtin::BI__builtin_atan2f:
case Builtin::BI__builtin_atan2f16:
case Builtin::BI__builtin_atan2l:
case Builtin::BI__builtin_atan2f128:
case Builtin::BI__builtin_elementwise_atan2:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::ATan2Op>(cgf, *e));
case Builtin::BIceil:
case Builtin::BIceilf:
case Builtin::BIceill:
case Builtin::BI__builtin_ceil:
case Builtin::BI__builtin_ceilf:
case Builtin::BI__builtin_ceilf16:
case Builtin::BI__builtin_ceill:
case Builtin::BI__builtin_ceilf128:
case Builtin::BI__builtin_elementwise_ceil:
return emitUnaryMaybeConstrainedFPBuiltin<cir::CeilOp>(cgf, *e);
case Builtin::BIcopysign:
case Builtin::BIcopysignf:
case Builtin::BIcopysignl:
case Builtin::BI__builtin_copysign:
case Builtin::BI__builtin_copysignf:
case Builtin::BI__builtin_copysignf16:
case Builtin::BI__builtin_copysignl:
case Builtin::BI__builtin_copysignf128:
return emitBinaryFPBuiltin<cir::CopysignOp>(cgf, *e);
case Builtin::BIcos:
case Builtin::BIcosf:
case Builtin::BIcosl:
case Builtin::BI__builtin_cos:
case Builtin::BI__builtin_cosf:
case Builtin::BI__builtin_cosf16:
case Builtin::BI__builtin_cosl:
case Builtin::BI__builtin_cosf128:
return emitUnaryMaybeConstrainedFPBuiltin<cir::CosOp>(cgf, *e);
case Builtin::BI__builtin_elementwise_cos:
case Builtin::BIcosh:
case Builtin::BIcoshf:
case Builtin::BIcoshl:
case Builtin::BI__builtin_cosh:
case Builtin::BI__builtin_coshf:
case Builtin::BI__builtin_coshf16:
case Builtin::BI__builtin_coshl:
case Builtin::BI__builtin_coshf128:
case Builtin::BI__builtin_elementwise_cosh:
return RValue::getIgnored();
case Builtin::BIexp:
case Builtin::BIexpf:
case Builtin::BIexpl:
case Builtin::BI__builtin_exp:
case Builtin::BI__builtin_expf:
case Builtin::BI__builtin_expf16:
case Builtin::BI__builtin_expl:
case Builtin::BI__builtin_expf128:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ExpOp>(cgf, *e);
case Builtin::BI__builtin_elementwise_exp:
return RValue::getIgnored();
case Builtin::BIexp2:
case Builtin::BIexp2f:
case Builtin::BIexp2l:
case Builtin::BI__builtin_exp2:
case Builtin::BI__builtin_exp2f:
case Builtin::BI__builtin_exp2f16:
case Builtin::BI__builtin_exp2l:
case Builtin::BI__builtin_exp2f128:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp2Op>(cgf, *e);
case Builtin::BI__builtin_elementwise_exp2:
case Builtin::BI__builtin_exp10:
case Builtin::BI__builtin_exp10f:
case Builtin::BI__builtin_exp10f16:
case Builtin::BI__builtin_exp10l:
case Builtin::BI__builtin_exp10f128:
case Builtin::BI__builtin_elementwise_exp10:
return RValue::getIgnored();
case Builtin::BIfabs:
case Builtin::BIfabsf:
case Builtin::BIfabsl:
case Builtin::BI__builtin_fabs:
case Builtin::BI__builtin_fabsf:
case Builtin::BI__builtin_fabsf16:
case Builtin::BI__builtin_fabsl:
case Builtin::BI__builtin_fabsf128:
return emitUnaryMaybeConstrainedFPBuiltin<cir::FAbsOp>(cgf, *e);
case Builtin::BIfloor:
case Builtin::BIfloorf:
case Builtin::BIfloorl:
case Builtin::BI__builtin_floor:
case Builtin::BI__builtin_floorf:
case Builtin::BI__builtin_floorf16:
case Builtin::BI__builtin_floorl:
case Builtin::BI__builtin_floorf128:
return emitUnaryMaybeConstrainedFPBuiltin<cir::FloorOp>(cgf, *e);
case Builtin::BI__builtin_elementwise_floor:
case Builtin::BIfma:
case Builtin::BIfmaf:
case Builtin::BIfmal:
case Builtin::BI__builtin_fma:
case Builtin::BI__builtin_fmaf:
case Builtin::BI__builtin_fmaf16:
case Builtin::BI__builtin_fmal:
case Builtin::BI__builtin_fmaf128:
case Builtin::BI__builtin_elementwise_fma:
return RValue::getIgnored();
case Builtin::BIfmax:
case Builtin::BIfmaxf:
case Builtin::BIfmaxl:
case Builtin::BI__builtin_fmax:
case Builtin::BI__builtin_fmaxf:
case Builtin::BI__builtin_fmaxf16:
case Builtin::BI__builtin_fmaxl:
case Builtin::BI__builtin_fmaxf128:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::FMaxNumOp>(cgf, *e));
case Builtin::BIfmin:
case Builtin::BIfminf:
case Builtin::BIfminl:
case Builtin::BI__builtin_fmin:
case Builtin::BI__builtin_fminf:
case Builtin::BI__builtin_fminf16:
case Builtin::BI__builtin_fminl:
case Builtin::BI__builtin_fminf128:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::FMinNumOp>(cgf, *e));
case Builtin::BIfmaximum_num:
case Builtin::BIfmaximum_numf:
case Builtin::BIfmaximum_numl:
case Builtin::BI__builtin_fmaximum_num:
case Builtin::BI__builtin_fmaximum_numf:
case Builtin::BI__builtin_fmaximum_numf16:
case Builtin::BI__builtin_fmaximum_numl:
case Builtin::BI__builtin_fmaximum_numf128:
case Builtin::BIfminimum_num:
case Builtin::BIfminimum_numf:
case Builtin::BIfminimum_numl:
case Builtin::BI__builtin_fminimum_num:
case Builtin::BI__builtin_fminimum_numf:
case Builtin::BI__builtin_fminimum_numf16:
case Builtin::BI__builtin_fminimum_numl:
case Builtin::BI__builtin_fminimum_numf128:
return RValue::getIgnored();
case Builtin::BIfmod:
case Builtin::BIfmodf:
case Builtin::BIfmodl:
case Builtin::BI__builtin_fmod:
case Builtin::BI__builtin_fmodf:
case Builtin::BI__builtin_fmodf16:
case Builtin::BI__builtin_fmodl:
case Builtin::BI__builtin_fmodf128:
case Builtin::BI__builtin_elementwise_fmod:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::FModOp>(cgf, *e));
case Builtin::BIlog:
case Builtin::BIlogf:
case Builtin::BIlogl:
case Builtin::BI__builtin_log:
case Builtin::BI__builtin_logf:
case Builtin::BI__builtin_logf16:
case Builtin::BI__builtin_logl:
case Builtin::BI__builtin_logf128:
case Builtin::BI__builtin_elementwise_log:
return emitUnaryMaybeConstrainedFPBuiltin<cir::LogOp>(cgf, *e);
case Builtin::BIlog10:
case Builtin::BIlog10f:
case Builtin::BIlog10l:
case Builtin::BI__builtin_log10:
case Builtin::BI__builtin_log10f:
case Builtin::BI__builtin_log10f16:
case Builtin::BI__builtin_log10l:
case Builtin::BI__builtin_log10f128:
case Builtin::BI__builtin_elementwise_log10:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Log10Op>(cgf, *e);
case Builtin::BIlog2:
case Builtin::BIlog2f:
case Builtin::BIlog2l:
case Builtin::BI__builtin_log2:
case Builtin::BI__builtin_log2f:
case Builtin::BI__builtin_log2f16:
case Builtin::BI__builtin_log2l:
case Builtin::BI__builtin_log2f128:
case Builtin::BI__builtin_elementwise_log2:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Log2Op>(cgf, *e);
case Builtin::BInearbyint:
case Builtin::BInearbyintf:
case Builtin::BInearbyintl:
case Builtin::BI__builtin_nearbyint:
case Builtin::BI__builtin_nearbyintf:
case Builtin::BI__builtin_nearbyintl:
case Builtin::BI__builtin_nearbyintf128:
case Builtin::BI__builtin_elementwise_nearbyint:
return emitUnaryMaybeConstrainedFPBuiltin<cir::NearbyintOp>(cgf, *e);
case Builtin::BIpow:
case Builtin::BIpowf:
case Builtin::BIpowl:
case Builtin::BI__builtin_pow:
case Builtin::BI__builtin_powf:
case Builtin::BI__builtin_powf16:
case Builtin::BI__builtin_powl:
case Builtin::BI__builtin_powf128:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::PowOp>(cgf, *e));
case Builtin::BI__builtin_elementwise_pow:
return RValue::getIgnored();
case Builtin::BIrint:
case Builtin::BIrintf:
case Builtin::BIrintl:
case Builtin::BI__builtin_rint:
case Builtin::BI__builtin_rintf:
case Builtin::BI__builtin_rintf16:
case Builtin::BI__builtin_rintl:
case Builtin::BI__builtin_rintf128:
case Builtin::BI__builtin_elementwise_rint:
return emitUnaryMaybeConstrainedFPBuiltin<cir::RintOp>(cgf, *e);
case Builtin::BIround:
case Builtin::BIroundf:
case Builtin::BIroundl:
case Builtin::BI__builtin_round:
case Builtin::BI__builtin_roundf:
case Builtin::BI__builtin_roundf16:
case Builtin::BI__builtin_roundl:
case Builtin::BI__builtin_roundf128:
case Builtin::BI__builtin_elementwise_round:
return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundOp>(cgf, *e);
case Builtin::BIroundeven:
case Builtin::BIroundevenf:
case Builtin::BIroundevenl:
case Builtin::BI__builtin_roundeven:
case Builtin::BI__builtin_roundevenf:
case Builtin::BI__builtin_roundevenf16:
case Builtin::BI__builtin_roundevenl:
case Builtin::BI__builtin_roundevenf128:
case Builtin::BI__builtin_elementwise_roundeven:
return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundEvenOp>(cgf, *e);
case Builtin::BIsin:
case Builtin::BIsinf:
case Builtin::BIsinl:
case Builtin::BI__builtin_sin:
case Builtin::BI__builtin_sinf:
case Builtin::BI__builtin_sinf16:
case Builtin::BI__builtin_sinl:
case Builtin::BI__builtin_sinf128:
case Builtin::BI__builtin_elementwise_sin:
return emitUnaryMaybeConstrainedFPBuiltin<cir::SinOp>(cgf, *e);
case Builtin::BIsinh:
case Builtin::BIsinhf:
case Builtin::BIsinhl:
case Builtin::BI__builtin_sinh:
case Builtin::BI__builtin_sinhf:
case Builtin::BI__builtin_sinhf16:
case Builtin::BI__builtin_sinhl:
case Builtin::BI__builtin_sinhf128:
case Builtin::BI__builtin_elementwise_sinh:
case Builtin::BI__builtin_sincospi:
case Builtin::BI__builtin_sincospif:
case Builtin::BI__builtin_sincospil:
case Builtin::BIsincos:
case Builtin::BIsincosf:
case Builtin::BIsincosl:
case Builtin::BI__builtin_sincos:
case Builtin::BI__builtin_sincosf:
case Builtin::BI__builtin_sincosf16:
case Builtin::BI__builtin_sincosl:
case Builtin::BI__builtin_sincosf128:
return RValue::getIgnored();
case Builtin::BIsqrt:
case Builtin::BIsqrtf:
case Builtin::BIsqrtl:
case Builtin::BI__builtin_sqrt:
case Builtin::BI__builtin_sqrtf:
case Builtin::BI__builtin_sqrtf16:
case Builtin::BI__builtin_sqrtl:
case Builtin::BI__builtin_sqrtf128:
case Builtin::BI__builtin_elementwise_sqrt:
return emitUnaryMaybeConstrainedFPBuiltin<cir::SqrtOp>(cgf, *e);
case Builtin::BItan:
case Builtin::BItanf:
case Builtin::BItanl:
case Builtin::BI__builtin_tan:
case Builtin::BI__builtin_tanf:
case Builtin::BI__builtin_tanf16:
case Builtin::BI__builtin_tanl:
case Builtin::BI__builtin_tanf128:
case Builtin::BI__builtin_elementwise_tan:
return emitUnaryMaybeConstrainedFPBuiltin<cir::TanOp>(cgf, *e);
case Builtin::BItanh:
case Builtin::BItanhf:
case Builtin::BItanhl:
case Builtin::BI__builtin_tanh:
case Builtin::BI__builtin_tanhf:
case Builtin::BI__builtin_tanhf16:
case Builtin::BI__builtin_tanhl:
case Builtin::BI__builtin_tanhf128:
case Builtin::BI__builtin_elementwise_tanh:
return RValue::getIgnored();
case Builtin::BItrunc:
case Builtin::BItruncf:
case Builtin::BItruncl:
case Builtin::BI__builtin_trunc:
case Builtin::BI__builtin_truncf:
case Builtin::BI__builtin_truncf16:
case Builtin::BI__builtin_truncl:
case Builtin::BI__builtin_truncf128:
case Builtin::BI__builtin_elementwise_trunc:
return emitUnaryMaybeConstrainedFPBuiltin<cir::TruncOp>(cgf, *e);
case Builtin::BIlround:
case Builtin::BIlroundf:
case Builtin::BIlroundl:
case Builtin::BI__builtin_lround:
case Builtin::BI__builtin_lroundf:
case Builtin::BI__builtin_lroundl:
case Builtin::BI__builtin_lroundf128:
return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LroundOp>(cgf, *e);
case Builtin::BIllround:
case Builtin::BIllroundf:
case Builtin::BIllroundl:
case Builtin::BI__builtin_llround:
case Builtin::BI__builtin_llroundf:
case Builtin::BI__builtin_llroundl:
case Builtin::BI__builtin_llroundf128:
return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlroundOp>(cgf, *e);
case Builtin::BIlrint:
case Builtin::BIlrintf:
case Builtin::BIlrintl:
case Builtin::BI__builtin_lrint:
case Builtin::BI__builtin_lrintf:
case Builtin::BI__builtin_lrintl:
case Builtin::BI__builtin_lrintf128:
return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LrintOp>(cgf, *e);
case Builtin::BIllrint:
case Builtin::BIllrintf:
case Builtin::BIllrintl:
case Builtin::BI__builtin_llrint:
case Builtin::BI__builtin_llrintf:
case Builtin::BI__builtin_llrintl:
case Builtin::BI__builtin_llrintf128:
return emitUnaryMaybeConstrainedFPToIntBuiltin<cir::LlrintOp>(cgf, *e);
case Builtin::BI__builtin_ldexp:
case Builtin::BI__builtin_ldexpf:
case Builtin::BI__builtin_ldexpl:
case Builtin::BI__builtin_ldexpf16:
case Builtin::BI__builtin_ldexpf128:
case Builtin::BI__builtin_elementwise_ldexp:
default:
break;
}
return RValue::getIgnored();
}
// FIXME: Remove cgf parameter when all descriptor kinds are implemented
static mlir::Type
decodeFixedType(CIRGenFunction &cgf,
ArrayRef<llvm::Intrinsic::IITDescriptor> &infos,
mlir::MLIRContext *context) {
using namespace llvm::Intrinsic;
IITDescriptor descriptor = infos.front();
infos = infos.slice(1);
switch (descriptor.Kind) {
case IITDescriptor::Void:
return cir::VoidType::get(context);
case IITDescriptor::Half:
return cir::FP16Type::get(context);
case IITDescriptor::BFloat:
return cir::BF16Type::get(context);
case IITDescriptor::Float:
return cir::SingleType::get(context);
case IITDescriptor::Double:
return cir::DoubleType::get(context);
case IITDescriptor::Quad:
return cir::FP128Type::get(context);
// If the intrinsic expects unsigned integers, the signedness is corrected in
// correctIntegerSignedness()
case IITDescriptor::Integer:
return cir::IntType::get(context, descriptor.IntegerWidth,
/*isSigned=*/true);
case IITDescriptor::Vector: {
mlir::Type elementType = decodeFixedType(cgf, infos, context);
unsigned numElements = descriptor.VectorWidth.getFixedValue();
return cir::VectorType::get(elementType, numElements);
}
case IITDescriptor::Pointer: {
mlir::Builder builder(context);
auto addrSpace = cir::TargetAddressSpaceAttr::get(
context, descriptor.PointerAddressSpace);
return cir::PointerType::get(cir::VoidType::get(context), addrSpace);
}
default:
cgf.cgm.errorNYI("Unimplemented intrinsic type descriptor");
return cir::VoidType::get(context);
}
}
/// Helper function to correct integer signedness for intrinsic arguments and
/// return type. IIT always returns signed integers, but the actual intrinsic
/// may expect unsigned integers based on the AST FunctionDecl parameter types.
static mlir::Type correctIntegerSignedness(mlir::Type iitType, QualType astType,
mlir::MLIRContext *context) {
auto intTy = dyn_cast<cir::IntType>(iitType);
if (!intTy)
return iitType;
if (astType->isUnsignedIntegerType())
return cir::IntType::get(context, intTy.getWidth(), /*isSigned=*/false);
return iitType;
}
static mlir::Value getCorrectedPtr(mlir::Value argValue, mlir::Type expectedTy,
CIRGenBuilderTy &builder) {
auto ptrType = mlir::cast<cir::PointerType>(argValue.getType());
auto expectedPtrType = mlir::cast<cir::PointerType>(expectedTy);
assert(ptrType != expectedPtrType && "types should not match");
if (ptrType.getAddrSpace() != expectedPtrType.getAddrSpace()) {
assert(!cir::MissingFeatures::addressSpace() &&
"address space handling not yet implemented");
auto newPtrType = cir::PointerType::get(ptrType.getPointee(),
expectedPtrType.getAddrSpace());
return builder.createAddrSpaceCast(argValue, newPtrType);
}
return builder.createBitcast(argValue, expectedTy);
}
static cir::FuncType getIntrinsicType(CIRGenFunction &cgf,
mlir::MLIRContext *context,
llvm::Intrinsic::ID id) {
using namespace llvm::Intrinsic;
SmallVector<IITDescriptor, 8> table;
auto [tableRef, _, isVarArg] = getIntrinsicInfoTableEntries(id, table);
mlir::Type resultTy = decodeFixedType(cgf, tableRef, context);
SmallVector<mlir::Type, 8> argTypes;
while (!tableRef.empty())
argTypes.push_back(decodeFixedType(cgf, tableRef, context));
// CIR convention: no explicit void return type
if (isa<cir::VoidType>(resultTy))
return cir::FuncType::get(context, argTypes, /*optionalReturnType=*/nullptr,
isVarArg);
return cir::FuncType::get(context, argTypes, resultTy, isVarArg);
}
RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
const CallExpr *e,
ReturnValueSlot returnValue) {
mlir::Location loc = getLoc(e->getSourceRange());
// See if we can constant fold this builtin. If so, don't emit it at all.
// TODO: Extend this handling to all builtin calls that we can constant-fold.
// Do not constant-fold immediate (target-specific) builtins; their ASTs can
// trigger the constant evaluator in cases it cannot safely handle.
// Skip EvaluateAsRValue for those.
Expr::EvalResult result;
if (e->isPRValue() && !getContext().BuiltinInfo.isImmediate(builtinID) &&
e->EvaluateAsRValue(result, cgm.getASTContext()) &&
!result.hasSideEffects()) {
if (result.Val.isInt()) {
QualType type = e->getType();
if (type->isBooleanType())
return RValue::get(
builder.getBool(result.Val.getInt().getBoolValue(), loc));
return RValue::get(builder.getConstInt(loc, result.Val.getInt()));
}
if (result.Val.isFloat()) {
// Note: we are using result type of CallExpr to determine the type of
// the constant. Classic codegen uses the result value to determine the
// type. We feel it should be Ok to use expression type because it is
// hard to imagine a builtin function evaluates to a value that
// over/underflows its own defined type.
mlir::Type type = convertType(e->getType());
return RValue::get(builder.getConstFP(loc, type, result.Val.getFloat()));
}
}
const FunctionDecl *fd = gd.getDecl()->getAsFunction();
assert(!cir::MissingFeatures::builtinCallF128());
// If the builtin has been declared explicitly with an assembler label,
// disable the specialized emitting below. Ideally we should communicate the
// rename in IR, or at least avoid generating the intrinsic calls that are
// likely to get lowered to the renamed library functions.
unsigned builtinIDIfNoAsmLabel = fd->hasAttr<AsmLabelAttr>() ? 0 : builtinID;
bool generateFPMathIntrinsics =
shouldCIREmitFPMathIntrinsic(*this, e, builtinID);
if (generateFPMathIntrinsics) {
// Try to match the builtinID with a floating point math builtin.
RValue rv = tryEmitFPMathIntrinsic(*this, e, builtinIDIfNoAsmLabel);
// Return the result directly if a math intrinsic was generated.
if (!rv.isIgnored()) {
return rv;
}
}
assert(!cir::MissingFeatures::builtinCall());
switch (builtinIDIfNoAsmLabel) {
default:
break;
// C stdarg builtins.
case Builtin::BI__builtin_stdarg_start:
case Builtin::BI__builtin_va_start:
case Builtin::BI__va_start: {
mlir::Value vaList = builtinID == Builtin::BI__va_start
? emitScalarExpr(e->getArg(0))
: emitVAListRef(e->getArg(0)).getPointer();
emitVAStart(vaList);
return {};
}
case Builtin::BI__builtin_va_end:
emitVAEnd(emitVAListRef(e->getArg(0)).getPointer());
return {};
case Builtin::BI__builtin_va_copy: {
mlir::Value dstPtr = emitVAListRef(e->getArg(0)).getPointer();
mlir::Value srcPtr = emitVAListRef(e->getArg(1)).getPointer();
cir::VACopyOp::create(builder, dstPtr.getLoc(), dstPtr, srcPtr);
return {};
}
case Builtin::BIabs:
case Builtin::BIlabs:
case Builtin::BIllabs:
case Builtin::BI__builtin_abs:
case Builtin::BI__builtin_labs:
case Builtin::BI__builtin_llabs: {
bool sanitizeOverflow = sanOpts.has(SanitizerKind::SignedIntegerOverflow);
mlir::Value arg = emitScalarExpr(e->getArg(0));
mlir::Value result;
switch (getLangOpts().getSignedOverflowBehavior()) {
case LangOptions::SOB_Defined:
result = cir::AbsOp::create(builder, loc, arg.getType(), arg,
/*minIsPoison=*/false);
break;
case LangOptions::SOB_Undefined:
if (!sanitizeOverflow) {
result = cir::AbsOp::create(builder, loc, arg.getType(), arg,
/*minIsPoison=*/true);
break;
}
[[fallthrough]];
case LangOptions::SOB_Trapping:
cgm.errorNYI(e->getSourceRange(), "abs with overflow handling");
return RValue::get(nullptr);
}
return RValue::get(result);
}
case Builtin::BI__assume:
case Builtin::BI__builtin_assume: {
if (e->getArg(0)->HasSideEffects(getContext()))
return RValue::get(nullptr);
mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0));
cir::AssumeOp::create(builder, loc, argValue, cir::AssumeBundleKind::None,
mlir::ValueRange{});
return RValue::get(nullptr);
}
case Builtin::BI__builtin_assume_separate_storage: {
mlir::Value value0 = emitScalarExpr(e->getArg(0));
mlir::Value value1 = emitScalarExpr(e->getArg(1));
mlir::Value cond = builder.getBool(true, loc);
cir::AssumeOp::create(builder, loc, cond,
cir::AssumeBundleKind::SeparateStorage,
mlir::ValueRange{value0, value1});
return RValue::get(nullptr);
}
case Builtin::BI__builtin_assume_dereferenceable: {
mlir::Value ptrValue = emitScalarExpr(e->getArg(0));
mlir::Value sizeValue = emitScalarExpr(e->getArg(1));
// The `dereferenceable` operand bundle expects a pointer-sized unsigned
// integer; widen/narrow as needed.
mlir::Type uintPtrTy = convertType(getContext().getUIntPtrType());
if (sizeValue.getType() != uintPtrTy)
sizeValue = builder.createIntCast(sizeValue, uintPtrTy);
mlir::Value cond = builder.getBool(true, loc);
cir::AssumeOp::create(builder, loc, cond,
cir::AssumeBundleKind::Dereferenceable,
mlir::ValueRange{ptrValue, sizeValue});
return RValue::get(nullptr);
}
case Builtin::BI__builtin_assume_aligned: {
const Expr *ptrExpr = e->getArg(0);
mlir::Value ptrValue = emitScalarExpr(ptrExpr);
mlir::Value offsetValue =
(e->getNumArgs() > 2) ? emitScalarExpr(e->getArg(2)) : nullptr;
std::optional<llvm::APSInt> alignment =
e->getArg(1)->getIntegerConstantExpr(getContext());
assert(alignment.has_value() &&
"the second argument to __builtin_assume_aligned must be an "
"integral constant expression");
mlir::Value result =
emitAlignmentAssumption(ptrValue, ptrExpr, ptrExpr->getExprLoc(),
alignment->getSExtValue(), offsetValue);
return RValue::get(result);
}
case Builtin::BI__builtin_complex: {
mlir::Value real = emitScalarExpr(e->getArg(0));
mlir::Value imag = emitScalarExpr(e->getArg(1));
mlir::Value complex = builder.createComplexCreate(loc, real, imag);
return RValue::getComplex(complex);
}
case Builtin::BI__builtin_creal:
case Builtin::BI__builtin_crealf:
case Builtin::BI__builtin_creall:
case Builtin::BIcreal:
case Builtin::BIcrealf:
case Builtin::BIcreall: {
mlir::Value complex = emitComplexExpr(e->getArg(0));
mlir::Value real = builder.createComplexReal(loc, complex);
return RValue::get(real);
}
case Builtin::BI__builtin_cimag:
case Builtin::BI__builtin_cimagf:
case Builtin::BI__builtin_cimagl:
case Builtin::BIcimag:
case Builtin::BIcimagf:
case Builtin::BIcimagl: {
mlir::Value complex = emitComplexExpr(e->getArg(0));
mlir::Value imag = builder.createComplexImag(loc, complex);
return RValue::get(imag);
}
case Builtin::BI__builtin_conj:
case Builtin::BI__builtin_conjf:
case Builtin::BI__builtin_conjl:
case Builtin::BIconj:
case Builtin::BIconjf:
case Builtin::BIconjl: {
mlir::Value complex = emitComplexExpr(e->getArg(0));
mlir::Value conj = builder.createComplexConj(loc, complex);
return RValue::getComplex(conj);
}
case Builtin::BI__builtin_clrsb:
case Builtin::BI__builtin_clrsbl:
case Builtin::BI__builtin_clrsbll:
return emitBuiltinBitOp<cir::BitClrsbOp>(*this, e);
case Builtin::BI__builtin_ctzs:
case Builtin::BI__builtin_ctz:
case Builtin::BI__builtin_ctzl:
case Builtin::BI__builtin_ctzll:
assert(!cir::MissingFeatures::builtinCheckKind());
return emitBuiltinBitOp<cir::BitCtzOp>(*this, e,
getTarget().isCLZForZeroUndef());
case Builtin::BI__builtin_ctzg:
return emitBuiltinBitOpWithFallback<cir::BitCtzOp>(*this, e);
case Builtin::BI__builtin_clzs:
case Builtin::BI__builtin_clz:
case Builtin::BI__builtin_clzl:
case Builtin::BI__builtin_clzll:
assert(!cir::MissingFeatures::builtinCheckKind());
return emitBuiltinBitOp<cir::BitClzOp>(*this, e,
getTarget().isCLZForZeroUndef());
case Builtin::BI__builtin_clzg:
return emitBuiltinBitOpWithFallback<cir::BitClzOp>(*this, e);
case Builtin::BI__builtin_elementwise_ctzg:
cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_ctzg");
return RValue::get(nullptr);
case Builtin::BI__builtin_elementwise_clzg:
cgm.errorNYI(e->getSourceRange(), "__builtin_elementwise_clzg");
return RValue::get(nullptr);
case Builtin::BI__builtin_ffs:
case Builtin::BI__builtin_ffsl:
case Builtin::BI__builtin_ffsll:
return emitBuiltinBitOp<cir::BitFfsOp>(*this, e);
case Builtin::BI__builtin_parity:
case Builtin::BI__builtin_parityl:
case Builtin::BI__builtin_parityll:
return emitBuiltinBitOp<cir::BitParityOp>(*this, e);
case Builtin::BI__lzcnt16:
case Builtin::BI__lzcnt:
case Builtin::BI__lzcnt64:
return emitBuiltinBitOp<cir::BitClzOp>(*this, e);
case Builtin::BI__popcnt16:
case Builtin::BI__popcnt:
case Builtin::BI__popcnt64:
case Builtin::BI__builtin_popcount:
case Builtin::BI__builtin_popcountl:
case Builtin::BI__builtin_popcountll:
case Builtin::BI__builtin_popcountg:
return emitBuiltinBitOp<cir::BitPopcountOp>(*this, e);
// Always return the argument of __builtin_unpredictable. LLVM does not
// have an intrinsic corresponding to this builtin. Metadata for this
// builtin should be added directly to instructions such as branches or
// switches that use it.
case Builtin::BI__builtin_unpredictable: {
return RValue::get(emitScalarExpr(e->getArg(0)));
}
case Builtin::BI__builtin_expect:
case Builtin::BI__builtin_expect_with_probability: {
mlir::Value argValue = emitScalarExpr(e->getArg(0));
if (cgm.getCodeGenOpts().OptimizationLevel == 0)
return RValue::get(argValue);
mlir::Value expectedValue = emitScalarExpr(e->getArg(1));
mlir::FloatAttr probAttr;
if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {
llvm::APFloat probability(0.0);
const Expr *probArg = e->getArg(2);
[[maybe_unused]] bool evalSucceeded =
probArg->EvaluateAsFloat(probability, cgm.getASTContext());
assert(evalSucceeded &&
"probability should be able to evaluate as float");
bool loseInfo = false; // ignored
probability.convert(llvm::APFloat::IEEEdouble(),
llvm::RoundingMode::Dynamic, &loseInfo);
probAttr = mlir::FloatAttr::get(mlir::Float64Type::get(&getMLIRContext()),
probability);
}
auto result = cir::ExpectOp::create(builder, loc, argValue.getType(),
argValue, expectedValue, probAttr);
return RValue::get(result);
}
case Builtin::BI__builtin_bswapg: {
mlir::Value arg = emitScalarExpr(e->getArg(0));
// CIR models bool as cir.bool rather than an integer, so peel it off
// before the cast below. Like classic codegen's i1 case, it byte-swaps
// to itself.
if (mlir::isa<cir::BoolType>(arg.getType()))
return RValue::get(arg);
auto argTy = mlir::cast<cir::IntType>(arg.getType());
// A single bit or a single byte byte-swaps to itself.
if (argTy.getWidth() == 1 || argTy.getWidth() == 8)
return RValue::get(arg);
assert(argTy.getWidth() % 16 == 0 &&
"__builtin_bswapg requires a single byte or a multiple of 16 bits");
// cir.byte_swap requires an unsigned operand. Reinterpret a signed
// argument as unsigned of the same width; createBuiltinBitOp casts the
// swapped result back to the builtin's (possibly signed) return type.
if (argTy.isSigned())
arg = builder.createIntCast(arg, builder.getUIntNTy(argTy.getWidth()));
return RValue::get(createBuiltinBitOp<cir::ByteSwapOp>(*this, e, arg));
}
case Builtin::BI__builtin_bswap16:
case Builtin::BI__builtin_bswap32:
case Builtin::BI__builtin_bswap64:
case Builtin::BI_byteswap_ushort:
case Builtin::BI_byteswap_ulong:
case Builtin::BI_byteswap_uint64: {
mlir::Value arg = emitScalarExpr(e->getArg(0));
return RValue::get(cir::ByteSwapOp::create(builder, loc, arg));
}
case Builtin::BI__builtin_bitreverse8:
case Builtin::BI__builtin_bitreverse16:
case Builtin::BI__builtin_bitreverse32:
case Builtin::BI__builtin_bitreverse64: {
mlir::Value arg = emitScalarExpr(e->getArg(0));
return RValue::get(cir::BitReverseOp::create(builder, loc, arg));
}
case Builtin::BI__builtin_rotateleft8:
case Builtin::BI__builtin_rotateleft16:
case Builtin::BI__builtin_rotateleft32:
case Builtin::BI__builtin_rotateleft64:
return emitRotate(e, /*isRotateLeft=*/true);
case Builtin::BI__builtin_rotateright8:
case Builtin::BI__builtin_rotateright16:
case Builtin::BI__builtin_rotateright32:
case Builtin::BI__builtin_rotateright64:
return emitRotate(e, /*isRotateLeft=*/false);
case Builtin::BI__builtin_coro_id:
case Builtin::BI__builtin_coro_promise:
case Builtin::BI__builtin_coro_resume:
case Builtin::BI__builtin_coro_noop:
case Builtin::BI__builtin_coro_destroy:
case Builtin::BI__builtin_coro_done:
case Builtin::BI__builtin_coro_alloc:
case Builtin::BI__builtin_coro_begin:
case Builtin::BI__builtin_coro_end:
case Builtin::BI__builtin_coro_suspend:
case Builtin::BI__builtin_coro_align:
cgm.errorNYI(e->getSourceRange(), "BI__builtin_coro_id like NYI");
return getUndefRValue(e->getType());
case Builtin::BI__builtin_coro_frame: {
return emitCoroutineFrame();
}
case Builtin::BI__builtin_coro_free:
return RValue::get(emitCoroFreeBuiltin(e).getResult());
case Builtin::BI__builtin_coro_size: {
GlobalDecl gd{fd};
mlir::Type ty = cgm.getTypes().getFunctionType(
cgm.getTypes().arrangeGlobalDeclaration(gd));
const auto *nd = cast<NamedDecl>(gd.getDecl());
cir::FuncOp fnOp =
cgm.getOrCreateCIRFunction(nd->getName(), ty, gd, /*ForVTable=*/false);
fnOp.setBuiltin(true);
return emitCall(e->getCallee()->getType(), CIRGenCallee::forDirect(fnOp), e,
returnValue);
}
case Builtin::BI__builtin_constant_p: {
mlir::Type resultType = convertType(e->getType());
const Expr *arg = e->getArg(0);
QualType argType = arg->getType();
// FIXME: The allowance for Obj-C pointers and block pointers is historical
// and likely a mistake.
if (!argType->isIntegralOrEnumerationType() && !argType->isFloatingType() &&
!argType->isObjCObjectPointerType() && !argType->isBlockPointerType()) {
// Per the GCC documentation, only numeric constants are recognized after
// inlining.
return RValue::get(
builder.getConstInt(getLoc(e->getSourceRange()),
mlir::cast<cir::IntType>(resultType), 0));
}
if (arg->HasSideEffects(getContext())) {
// The argument is unevaluated, so be conservative if it might have
// side-effects.
return RValue::get(
builder.getConstInt(getLoc(e->getSourceRange()),
mlir::cast<cir::IntType>(resultType), 0));
}
mlir::Value argValue = emitScalarExpr(arg);
if (argType->isObjCObjectPointerType()) {
cgm.errorNYI(e->getSourceRange(),
"__builtin_constant_p: Obj-C object pointer");
return {};
}
argValue = builder.createBitcast(argValue, convertType(argType));
mlir::Value result = cir::IsConstantOp::create(
builder, getLoc(e->getSourceRange()), argValue);
// IsConstantOp returns a bool, but __builtin_constant_p returns an int.
result = builder.createBoolToInt(result, resultType);
return RValue::get(result);
}
case Builtin::BI__builtin_dynamic_object_size:
case Builtin::BI__builtin_object_size: {
unsigned type =
e->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
auto resType = mlir::cast<cir::IntType>(convertType(e->getType()));
// We pass this builtin onto the optimizer so that it can figure out the
// object size in more complex cases.
bool isDynamic = builtinID == Builtin::BI__builtin_dynamic_object_size;
return RValue::get(emitBuiltinObjectSize(e->getArg(0), type, resType,
/*EmittedE=*/nullptr, isDynamic));
}
case Builtin::BI__builtin_prefetch: {
auto evaluateOperandAsInt = [&](const Expr *arg) {
Expr::EvalResult res;
[[maybe_unused]] bool evalSucceed =
arg->EvaluateAsInt(res, cgm.getASTContext());
assert(evalSucceed && "expression should be able to evaluate as int");
return res.Val.getInt().getZExtValue();
};
bool isWrite = false;
if (e->getNumArgs() > 1)
isWrite = evaluateOperandAsInt(e->getArg(1));
int locality = 3;
if (e->getNumArgs() > 2)
locality = evaluateOperandAsInt(e->getArg(2));
mlir::Value address = emitScalarExpr(e->getArg(0));
cir::PrefetchOp::create(builder, loc, address, locality, isWrite);
return RValue::get(nullptr);
}
case Builtin::BI__builtin_readcyclecounter:
case Builtin::BI__builtin_readsteadycounter:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin___clear_cache: {
mlir::Value begin =
builder.createPtrBitcast(emitScalarExpr(e->getArg(0)), cgm.voidTy);
mlir::Value end =
builder.createPtrBitcast(emitScalarExpr(e->getArg(1)), cgm.voidTy);
cir::ClearCacheOp::create(builder, getLoc(e->getSourceRange()), begin, end);
return RValue::get(nullptr);
}
case Builtin::BI__builtin_trap:
emitTrap(loc, /*createNewBlock=*/true);
return RValue::getIgnored();
case Builtin::BI__builtin_verbose_trap:
assert(!cir::MissingFeatures::generateDebugInfo());
emitTrap(loc, /*createNewBlock=*/true);
return RValue::getIgnored();
case Builtin::BI__debugbreak:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_unreachable:
emitUnreachable(e->getExprLoc(), /*createNewBlock=*/true);
return RValue::getIgnored();
case Builtin::BI__builtin_powi:
case Builtin::BI__builtin_powif:
case Builtin::BI__builtin_powil: {
mlir::Value src0 = emitScalarExpr(e->getArg(0));
mlir::Value src1 = emitScalarExpr(e->getArg(1));
return RValue::get(builder.emitIntrinsicCallOp(
getLoc(e->getExprLoc()), "powi", src0.getType(),
mlir::ValueRange{src0, src1}));
}
case Builtin::BI__builtin_frexpl:
case Builtin::BI__builtin_frexp:
case Builtin::BI__builtin_frexpf:
case Builtin::BI__builtin_frexpf128:
case Builtin::BI__builtin_frexpf16: {
mlir::Value val = emitScalarExpr(e->getArg(0));
mlir::Value ptr = emitScalarExpr(e->getArg(1));
mlir::Type fpTy = val.getType();
QualType intQualTy = e->getArg(1)->getType()->getPointeeType();
mlir::Type intTy = convertType(intQualTy);
mlir::Location callLoc = getLoc(e->getExprLoc());
auto frexpOp = cir::FrexpOp::create(builder, callLoc, fpTy, intTy, val);
LValue lv = makeNaturalAlignAddrLValue(ptr, intQualTy);
emitStoreOfScalar(frexpOp.getExp(), lv, /*isInit=*/false);
return RValue::get(frexpOp.getResult());
}
case Builtin::BImodf:
case Builtin::BImodff:
case Builtin::BImodfl:
case Builtin::BI__builtin_modf:
case Builtin::BI__builtin_modff:
case Builtin::BI__builtin_modfl: {
mlir::Value val = emitScalarExpr(e->getArg(0));
mlir::Value ptr = emitScalarExpr(e->getArg(1));
mlir::Type fpTy = val.getType();
mlir::Location callLoc = getLoc(e->getExprLoc());
auto modfOp = cir::ModfOp::create(builder, callLoc, fpTy, fpTy, val);
QualType destPtrTy = e->getArg(1)->getType()->getPointeeType();
LValue lv = makeNaturalAlignAddrLValue(ptr, destPtrTy);
emitStoreOfScalar(modfOp.getIntegral(), lv, /*isInit=*/false);
return RValue::get(modfOp.getFractional());
}
case Builtin::BI__builtin_isgreater:
case Builtin::BI__builtin_isgreaterequal:
case Builtin::BI__builtin_isless:
case Builtin::BI__builtin_islessequal:
case Builtin::BI__builtin_islessgreater:
case Builtin::BI__builtin_isunordered: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value lhs = emitScalarExpr(e->getArg(0));
mlir::Value rhs = emitScalarExpr(e->getArg(1));
mlir::Location loc = getLoc(e->getBeginLoc());
mlir::Type intTy = convertType(e->getType());
mlir::Value cmpResult;
switch (builtinID) {
case Builtin::BI__builtin_isgreater:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::gt, lhs, rhs);
break;
case Builtin::BI__builtin_isgreaterequal:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::ge, lhs, rhs);
break;
case Builtin::BI__builtin_isless:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::lt, lhs, rhs);
break;
case Builtin::BI__builtin_islessequal:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::le, lhs, rhs);
break;
case Builtin::BI__builtin_islessgreater:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::one, lhs, rhs);
break;
case Builtin::BI__builtin_isunordered:
cmpResult = builder.createCompare(loc, cir::CmpOpKind::uno, lhs, rhs);
break;
default:
llvm_unreachable("Unknown ordered comparison");
}
return RValue::get(builder.createBoolToInt(cmpResult, intTy));
}
// From https://clang.llvm.org/docs/LanguageExtensions.html#builtin-isfpclass
//
// The `__builtin_isfpclass()` builtin is a generalization of functions
// isnan, isinf, isfinite and some others defined by the C standard. It tests
// if the floating-point value, specified by the first argument, falls into
// any of data classes, specified by the second argument.
case Builtin::BI__builtin_isnan: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
assert(!cir::MissingFeatures::fpConstraints());
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Nan),
convertType(e->getType())));
}
case Builtin::BI__builtin_issignaling: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::SignalingNaN),
convertType(e->getType())));
}
case Builtin::BI__builtin_isinf: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
assert(!cir::MissingFeatures::fpConstraints());
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Infinity),
convertType(e->getType())));
}
case Builtin::BIfinite:
case Builtin::BI__finite:
case Builtin::BIfinitef:
case Builtin::BI__finitef:
case Builtin::BIfinitel:
case Builtin::BI__finitel:
case Builtin::BI__builtin_isfinite: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
assert(!cir::MissingFeatures::fpConstraints());
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Finite),
convertType(e->getType())));
}
case Builtin::BI__builtin_isnormal: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Normal),
convertType(e->getType())));
}
case Builtin::BI__builtin_issubnormal: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Subnormal),
convertType(e->getType())));
}
case Builtin::BI__builtin_iszero: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
mlir::Location loc = getLoc(e->getBeginLoc());
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest::Zero),
convertType(e->getType())));
}
case Builtin::BI__builtin_isfpclass: {
Expr::EvalResult result;
if (!e->getArg(1)->EvaluateAsInt(result, cgm.getASTContext()))
break;
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Value v = emitScalarExpr(e->getArg(0));
uint64_t test = result.Val.getInt().getLimitedValue();
mlir::Location loc = getLoc(e->getBeginLoc());
//
return RValue::get(builder.createBoolToInt(
builder.createIsFPClass(loc, v, cir::FPClassTest(test)),
convertType(e->getType())));
}
case Builtin::BI__builtin_nondeterministic_value: {
mlir::Type ty = convertType(e->getArg(0)->getType());
mlir::Value result =
cir::ConstantOp::create(builder, loc, ty, cir::PoisonAttr::get(ty));
result = cir::FreezeOp::create(builder, loc, result);
return RValue::get(result);
}
case Builtin::BI__builtin_elementwise_abs: {
mlir::Type cirTy = convertType(e->getArg(0)->getType());
bool isIntTy = cir::isIntOrVectorOfIntType(cirTy);
if (!isIntTy)
return emitUnaryFPBuiltin<cir::FAbsOp>(*this, *e);
mlir::Value arg = emitScalarExpr(e->getArg(0));
mlir::Value result = cir::AbsOp::create(builder, getLoc(e->getExprLoc()),
arg.getType(), arg, false);
return RValue::get(result);
}
case Builtin::BI__builtin_elementwise_acos:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ACosOp>(*this, *e);
case Builtin::BI__builtin_elementwise_asin:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ASinOp>(*this, *e);
case Builtin::BI__builtin_elementwise_atan:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ATanOp>(*this, *e);
case Builtin::BI__builtin_elementwise_atan2:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::ATan2Op>(*this, *e));
case Builtin::BI__builtin_elementwise_exp:
return emitUnaryMaybeConstrainedFPBuiltin<cir::ExpOp>(*this, *e);
case Builtin::BI__builtin_elementwise_exp2:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Exp2Op>(*this, *e);
case Builtin::BI__builtin_elementwise_log:
return emitUnaryMaybeConstrainedFPBuiltin<cir::LogOp>(*this, *e);
case Builtin::BI__builtin_elementwise_log2:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Log2Op>(*this, *e);
case Builtin::BI__builtin_elementwise_log10:
return emitUnaryMaybeConstrainedFPBuiltin<cir::Log10Op>(*this, *e);
case Builtin::BI__builtin_elementwise_cos:
return emitUnaryMaybeConstrainedFPBuiltin<cir::CosOp>(*this, *e);
case Builtin::BI__builtin_elementwise_floor:
return emitUnaryMaybeConstrainedFPBuiltin<cir::FloorOp>(*this, *e);
case Builtin::BI__builtin_elementwise_round:
return emitUnaryMaybeConstrainedFPBuiltin<cir::RoundOp>(*this, *e);
case Builtin::BI__builtin_elementwise_rint:
return emitUnaryMaybeConstrainedFPBuiltin<cir::RintOp>(*this, *e);
case Builtin::BI__builtin_elementwise_nearbyint:
return emitUnaryMaybeConstrainedFPBuiltin<cir::NearbyintOp>(*this, *e);
case Builtin::BI__builtin_elementwise_sin:
return emitUnaryMaybeConstrainedFPBuiltin<cir::SinOp>(*this, *e);
case Builtin::BI__builtin_elementwise_sqrt:
return emitUnaryMaybeConstrainedFPBuiltin<cir::SqrtOp>(*this, *e);
case Builtin::BI__builtin_elementwise_tan:
return emitUnaryMaybeConstrainedFPBuiltin<cir::TanOp>(*this, *e);
case Builtin::BI__builtin_elementwise_trunc:
return emitUnaryMaybeConstrainedFPBuiltin<cir::TruncOp>(*this, *e);
case Builtin::BI__builtin_elementwise_fmod:
return RValue::get(
emitBinaryMaybeConstrainedFPBuiltin<cir::FModOp>(*this, *e));
case Builtin::BI__builtin_elementwise_ceil:
case Builtin::BI__builtin_elementwise_exp10:
case Builtin::BI__builtin_elementwise_ldexp:
case Builtin::BI__builtin_elementwise_pow:
case Builtin::BI__builtin_elementwise_bitreverse:
case Builtin::BI__builtin_elementwise_cosh:
case Builtin::BI__builtin_elementwise_popcount:
case Builtin::BI__builtin_elementwise_roundeven:
case Builtin::BI__builtin_elementwise_sinh:
case Builtin::BI__builtin_elementwise_tanh:
case Builtin::BI__builtin_elementwise_canonicalize:
case Builtin::BI__builtin_elementwise_copysign:
case Builtin::BI__builtin_elementwise_fma:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_elementwise_fshl: {
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Value a = emitScalarExpr(e->getArg(0));
mlir::Value b = emitScalarExpr(e->getArg(1));
mlir::Value c = emitScalarExpr(e->getArg(2));
return RValue::get(builder.emitIntrinsicCallOp(loc, "fshl", a.getType(),
mlir::ValueRange{a, b, c}));
}
case Builtin::BI__builtin_elementwise_fshr: {
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Value a = emitScalarExpr(e->getArg(0));
mlir::Value b = emitScalarExpr(e->getArg(1));
mlir::Value c = emitScalarExpr(e->getArg(2));
return RValue::get(builder.emitIntrinsicCallOp(loc, "fshr", a.getType(),
mlir::ValueRange{a, b, c}));
}
case Builtin::BI__builtin_elementwise_add_sat:
case Builtin::BI__builtin_elementwise_sub_sat: {
// cir.add/cir.sub do not model i1 arithmetic, so a bool-element
// saturating add/sub is not representable through the saturated op.
// Bail before emitScalarExpr: an ext-vector-of-bool operand would
// otherwise hit the NYI bool-vector load, which returns a null value
// and would crash op0.getType().
QualType argTy = e->getArg(0)->getType();
if (argTy->isBooleanType() || argTy->isExtVectorBoolType()) {
cgm.errorNYI(e->getSourceRange(),
"saturating add/sub on a boolean operand");
return RValue::get(nullptr);
}
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Value op0 = emitScalarExpr(e->getArg(0));
mlir::Value op1 = emitScalarExpr(e->getArg(1));
assert(cir::isIntOrVectorOfIntType(op0.getType()) &&
"elementwise saturating add/sub requires integer operands");
mlir::Value val =
builtinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat
? builder.createAdd(loc, op0, op1, cir::OverflowBehavior::Saturated)
: builder.createSub(loc, op0, op1,
cir::OverflowBehavior::Saturated);
return RValue::get(val);
}
case Builtin::BI__builtin_elementwise_max:
case Builtin::BI__builtin_elementwise_min:
case Builtin::BI__builtin_elementwise_maxnum:
case Builtin::BI__builtin_elementwise_minnum:
case Builtin::BI__builtin_elementwise_maximum:
case Builtin::BI__builtin_elementwise_minimum:
case Builtin::BI__builtin_elementwise_maximumnum:
case Builtin::BI__builtin_elementwise_minimumnum:
case Builtin::BI__builtin_reduce_max:
case Builtin::BI__builtin_reduce_min:
case Builtin::BI__builtin_reduce_add:
case Builtin::BI__builtin_reduce_mul:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_reduce_xor:
return emitBuiltinWithOneOverloadedType<1>(
e, "vector.reduce.xor",
cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
.getElementType());
case Builtin::BI__builtin_reduce_or:
return emitBuiltinWithOneOverloadedType<1>(
e, "vector.reduce.or",
cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
.getElementType());
case Builtin::BI__builtin_reduce_and:
return emitBuiltinWithOneOverloadedType<1>(
e, "vector.reduce.and",
cast<cir::VectorType>(convertType(e->getArg(0)->getType()))
.getElementType());
case Builtin::BI__builtin_reduce_assoc_fadd:
case Builtin::BI__builtin_reduce_in_order_fadd:
case Builtin::BI__builtin_reduce_maximum:
case Builtin::BI__builtin_reduce_minimum:
case Builtin::BI__builtin_matrix_transpose:
case Builtin::BI__builtin_matrix_column_major_load:
case Builtin::BI__builtin_matrix_column_major_store:
case Builtin::BI__builtin_masked_load:
case Builtin::BI__builtin_masked_expand_load:
case Builtin::BI__builtin_masked_gather:
case Builtin::BI__builtin_masked_store:
case Builtin::BI__builtin_masked_compress_store:
case Builtin::BI__builtin_masked_scatter:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_isinf_sign: {
CIRGenFunction::CIRGenFPOptionsRAII FPOptsRAII(*this, e);
mlir::Location loc = getLoc(e->getBeginLoc());
mlir::Value arg = emitScalarExpr(e->getArg(0));
mlir::Value isInf =
builder.createIsFPClass(loc, arg, cir::FPClassTest::Infinity);
mlir::Value isNeg = emitSignBit(loc, *this, arg);
mlir::Type intTy = convertType(e->getType());
cir::ConstantOp zero = builder.getNullValue(intTy, loc);
cir::ConstantOp one = builder.getConstant(loc, cir::IntAttr::get(intTy, 1));
cir::ConstantOp negativeOne =
builder.getConstant(loc, cir::IntAttr::get(intTy, -1));
mlir::Value signResult = builder.createSelect(loc, isNeg, negativeOne, one);
mlir::Value result = builder.createSelect(loc, isInf, signResult, zero);
return RValue::get(result);
}
case Builtin::BI__builtin_flt_rounds: {
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Type resultType = convertType(e->getType());
mlir::Value result =
builder.emitIntrinsicCallOp(loc, "get.rounding", resultType);
if (result.getType() != resultType)
result =
builder.createCast(loc, cir::CastKind::integral, result, resultType);
return RValue::get(result);
}
case Builtin::BI__builtin_set_flt_rounds: {
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Value v = emitScalarExpr(e->getArg(0));
builder.emitIntrinsicCallOp(loc, "set.rounding", builder.getVoidTy(),
mlir::ValueRange{v});
return RValue::get(nullptr);
}
case Builtin::BI__builtin_fpclassify: {
CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
mlir::Location loc = getLoc(e->getBeginLoc());
mlir::Value value = emitScalarExpr(e->getArg(5));
mlir::Type resultTy = convertType(e->getType());
// if isZero then
// result = FP_ZERO
// elseif isNan then
// result = FP_NAN
// elseif isInfinity then
// result = FP_INFINITE
// elseif isNormal then
// result = FP_NORMAL
// else
// result = FP_SUBNORMAL
auto isZero =
cir::IsFPClassOp::create(builder, loc, value, cir::FPClassTest::Zero);
mlir::Value result =
cir::TernaryOp::create(
builder, loc, isZero,
/*thenBuilder=*/
[&](mlir::OpBuilder &opBuilder, mlir::Location location) {
mlir::Value zeroLiteral = emitScalarExpr(e->getArg(4));
cir::YieldOp::create(opBuilder, location, zeroLiteral);
},
/*elseBuilder=*/
[&](mlir::OpBuilder &opBuilder, mlir::Location location) {
auto isNan = cir::IsFPClassOp::create(opBuilder, location, value,
cir::FPClassTest::Nan);
mlir::Value nanResult =
cir::TernaryOp::create(
opBuilder, location, isNan,
/*thenBuilder=*/
[&](mlir::OpBuilder &opBuilder, mlir::Location location) {
mlir::Value nanLiteral = emitScalarExpr(e->getArg(0));
cir::YieldOp::create(opBuilder, location, nanLiteral);
},
/*elseBuilder=*/
[&](mlir::OpBuilder &opBuilder, mlir::Location location) {
auto isInfinity = cir::IsFPClassOp::create(
opBuilder, location, value,
cir::FPClassTest::Infinity);
mlir::Value infResult =
cir::TernaryOp::create(
opBuilder, location, isInfinity,
/*thenBuilder=*/
[&](mlir::OpBuilder &opBuilder,
mlir::Location location) {
mlir::Value infinityLiteral =
emitScalarExpr(e->getArg(1));
cir::YieldOp::create(opBuilder, location,
infinityLiteral);
},
/*elseBuilder=*/
[&](mlir::OpBuilder &opBuilder,
mlir::Location location) {
auto isNormal = cir::IsFPClassOp::create(
opBuilder, location, value,
cir::FPClassTest::Normal);
mlir::Value fpNormal =
emitScalarExpr(e->getArg(2));
mlir::Value fpSubnormal =
emitScalarExpr(e->getArg(3));
mlir::Value returnValue =
cir::SelectOp::create(
opBuilder, location, resultTy,
isNormal, fpNormal, fpSubnormal);
cir::YieldOp::create(opBuilder, location,
returnValue);
})
.getResult();
cir::YieldOp::create(opBuilder, location, infResult);
})
.getResult();
cir::YieldOp::create(opBuilder, location, nanResult);
})
.getResult();
return RValue::get(result);
}
case Builtin::BIalloca:
case Builtin::BI_alloca:
case Builtin::BI__builtin_alloca_uninitialized:
case Builtin::BI__builtin_alloca:
return emitBuiltinAlloca(*this, e, builtinID);
case Builtin::BI__builtin_alloca_with_align_uninitialized:
case Builtin::BI__builtin_alloca_with_align:
case Builtin::BI__builtin_infer_alloc_token:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BIbzero:
case Builtin::BI__builtin_bzero: {
mlir::Location loc = getLoc(e->getSourceRange());
Address destPtr = emitPointerWithAlignment(e->getArg(0));
Address destPtrCast = destPtr.withElementType(builder, cgm.voidTy);
mlir::Value size = emitScalarExpr(e->getArg(1));
mlir::Value zero = builder.getNullValue(builder.getUInt8Ty(), loc);
assert(!cir::MissingFeatures::sanitizers());
builder.createMemSet(loc, destPtrCast, zero, size);
assert(!cir::MissingFeatures::generateDebugInfo());
return RValue::getIgnored();
}
case Builtin::BIbcopy:
case Builtin::BI__builtin_bcopy: {
Address src = emitPointerWithAlignment(e->getArg(0));
Address dest = emitPointerWithAlignment(e->getArg(1));
mlir::Value sizeVal = emitScalarExpr(e->getArg(2));
emitNonNullArgCheck(RValue::get(src.getPointer()), e->getArg(0)->getType(),
e->getArg(0)->getExprLoc(), fd, 0);
emitNonNullArgCheck(RValue::get(dest.getPointer()), e->getArg(1)->getType(),
e->getArg(1)->getExprLoc(), fd, 0);
builder.createMemMove(getLoc(e->getSourceRange()), dest.getPointer(),
src.getPointer(), sizeVal);
return RValue::get(nullptr);
}
case Builtin::BI__builtin_char_memchr:
case Builtin::BI__builtin_memchr: {
Address srcPtr = emitPointerWithAlignment(e->getArg(0));
mlir::Value src =
builder.createBitcast(srcPtr.getPointer(), builder.getVoidPtrTy());
mlir::Value pattern = emitScalarExpr(e->getArg(1));
mlir::Value len = emitScalarExpr(e->getArg(2));
mlir::Value res = cir::MemChrOp::create(builder, getLoc(e->getExprLoc()),
src, pattern, len);
return RValue::get(res);
}
case Builtin::BImemcpy:
case Builtin::BI__builtin_memcpy:
case Builtin::BImempcpy:
case Builtin::BI__builtin_mempcpy:
case Builtin::BI__builtin_memcpy_inline:
case Builtin::BI__builtin___memcpy_chk:
case Builtin::BI__builtin_objc_memmove_collectable:
case Builtin::BI__builtin___memmove_chk:
case Builtin::BI__builtin_trivially_relocate:
case Builtin::BImemmove:
case Builtin::BI__builtin_memmove:
case Builtin::BImemset:
case Builtin::BI__builtin_memset:
case Builtin::BI__builtin_memset_inline:
case Builtin::BI__builtin___memset_chk:
case Builtin::BI__builtin_wmemchr:
case Builtin::BI__builtin_wmemcmp:
break; // Handled as library calls below.
case Builtin::BI__builtin_dwarf_cfa:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_return_address: {
llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());
return RValue::get(cir::ReturnAddrOp::create(
builder, getLoc(e->getExprLoc()),
builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));
}
case Builtin::BI_ReturnAddress: {
return RValue::get(cir::ReturnAddrOp::create(
builder, getLoc(e->getExprLoc()),
builder.getConstInt(loc, builder.getUInt32Ty(), 0)));
}
case Builtin::BI__builtin_frame_address: {
llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());
mlir::Location loc = getLoc(e->getExprLoc());
mlir::Value addr = cir::FrameAddrOp::create(
builder, loc, allocaInt8PtrTy,
builder.getConstAPInt(loc, builder.getUInt32Ty(), level));
return RValue::get(
builder.createCast(loc, cir::CastKind::bitcast, addr, voidPtrTy));
}
case Builtin::BI__builtin_extract_return_addr:
case Builtin::BI__builtin_frob_return_addr:
case Builtin::BI__builtin_dwarf_sp_column:
case Builtin::BI__builtin_init_dwarf_reg_size_table:
case Builtin::BI__builtin_eh_return:
case Builtin::BI__builtin_unwind_init:
case Builtin::BI__builtin_extend_pointer:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_setjmp: {
Address buf = emitPointerWithAlignment(e->getArg(0));
mlir::Location loc = getLoc(e->getExprLoc());
cir::PointerType voidPtrTy = builder.getVoidPtrTy();
cir::PointerType ppTy = builder.getPointerTo(voidPtrTy);
Address castBuf = buf.withElementType(builder, voidPtrTy);
assert(!cir::MissingFeatures::emitCheckedInBoundsGEP());
if (getTarget().getTriple().isSystemZ()) {
cgm.errorNYI(e->getExprLoc(), "setjmp on SystemZ");
return {};
}
mlir::Value frameAddress =
cir::FrameAddrOp::create(builder, loc, voidPtrTy,
mlir::ValueRange{builder.getUInt32(0, loc)})
.getResult();
builder.createStore(loc, frameAddress, castBuf);
mlir::Value stacksave =
cir::StackSaveOp::create(builder, loc, voidPtrTy).getResult();
cir::PtrStrideOp stackSaveSlot = cir::PtrStrideOp::create(
builder, loc, ppTy, castBuf.getPointer(), builder.getSInt32(2, loc));
llvm::TypeSize voidPtrTySize =
cgm.getDataLayout().getTypeAllocSize(voidPtrTy);
CharUnits slotAlign = castBuf.getAlignment().alignmentAtOffset(
CharUnits().fromQuantity(2 * voidPtrTySize));
Address slotAddr = Address(stackSaveSlot, voidPtrTy, slotAlign);
builder.createStore(loc, stacksave, slotAddr);
auto op = cir::EhSetjmpOp::create(builder, loc, castBuf.getPointer());
return RValue::get(op);
}
case Builtin::BI__builtin_longjmp: {
mlir::Value buf = emitScalarExpr(e->getArg(0));
mlir::Location loc = getLoc(e->getExprLoc());
cir::EhLongjmpOp::create(builder, loc, buf);
cir::UnreachableOp::create(builder, loc);
return RValue::get(nullptr);
}
case Builtin::BI__builtin_launder: {
const Expr *arg = e->getArg(0);
QualType argTy = arg->getType()->getPointeeType();
mlir::Value ptr = emitScalarExpr(arg);
if (cgm.getCodeGenOpts().StrictVTablePointers &&
argTy.requiresBuiltinLaunder(cgm.getASTContext())) {
mlir::Location loc = getLoc(e->getExprLoc());
ptr = cir::LaunderOp::create(builder, loc, ptr).getResult();
}
return RValue::get(ptr);
}
case Builtin::BI__sync_fetch_and_add:
case Builtin::BI__sync_fetch_and_sub:
case Builtin::BI__sync_fetch_and_or:
case Builtin::BI__sync_fetch_and_and:
case Builtin::BI__sync_fetch_and_xor:
case Builtin::BI__sync_fetch_and_nand:
case Builtin::BI__sync_add_and_fetch:
case Builtin::BI__sync_sub_and_fetch:
case Builtin::BI__sync_and_and_fetch:
case Builtin::BI__sync_or_and_fetch:
case Builtin::BI__sync_xor_and_fetch:
case Builtin::BI__sync_nand_and_fetch:
case Builtin::BI__sync_val_compare_and_swap:
case Builtin::BI__sync_bool_compare_and_swap:
case Builtin::BI__sync_lock_test_and_set:
case Builtin::BI__sync_lock_release:
case Builtin::BI__sync_swap:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__sync_fetch_and_add_1:
case Builtin::BI__sync_fetch_and_add_2:
case Builtin::BI__sync_fetch_and_add_4:
case Builtin::BI__sync_fetch_and_add_8:
case Builtin::BI__sync_fetch_and_add_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::Add, e);
case Builtin::BI__sync_fetch_and_sub_1:
case Builtin::BI__sync_fetch_and_sub_2:
case Builtin::BI__sync_fetch_and_sub_4:
case Builtin::BI__sync_fetch_and_sub_8:
case Builtin::BI__sync_fetch_and_sub_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::Sub, e);
case Builtin::BI__sync_fetch_and_or_1:
case Builtin::BI__sync_fetch_and_or_2:
case Builtin::BI__sync_fetch_and_or_4:
case Builtin::BI__sync_fetch_and_or_8:
case Builtin::BI__sync_fetch_and_or_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::Or, e);
case Builtin::BI__sync_fetch_and_and_1:
case Builtin::BI__sync_fetch_and_and_2:
case Builtin::BI__sync_fetch_and_and_4:
case Builtin::BI__sync_fetch_and_and_8:
case Builtin::BI__sync_fetch_and_and_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::And, e);
case Builtin::BI__sync_fetch_and_xor_1:
case Builtin::BI__sync_fetch_and_xor_2:
case Builtin::BI__sync_fetch_and_xor_4:
case Builtin::BI__sync_fetch_and_xor_8:
case Builtin::BI__sync_fetch_and_xor_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::Xor, e);
case Builtin::BI__sync_fetch_and_nand_1:
case Builtin::BI__sync_fetch_and_nand_2:
case Builtin::BI__sync_fetch_and_nand_4:
case Builtin::BI__sync_fetch_and_nand_8:
case Builtin::BI__sync_fetch_and_nand_16:
return emitBinaryAtomic(*this, cir::AtomicFetchKind::Nand, e);
case Builtin::BI__sync_fetch_and_min:
case Builtin::BI__sync_fetch_and_max:
case Builtin::BI__sync_fetch_and_umin:
case Builtin::BI__sync_fetch_and_umax:
return errorBuiltinNYI(*this, e, builtinID);
return getUndefRValue(e->getType());
case Builtin::BI__sync_add_and_fetch_1:
case Builtin::BI__sync_add_and_fetch_2:
case Builtin::BI__sync_add_and_fetch_4:
case Builtin::BI__sync_add_and_fetch_8:
case Builtin::BI__sync_add_and_fetch_16:
return emitBinaryAtomicPost<cir::AddOp>(*this, cir::AtomicFetchKind::Add,
e);
case Builtin::BI__sync_sub_and_fetch_1:
case Builtin::BI__sync_sub_and_fetch_2:
case Builtin::BI__sync_sub_and_fetch_4:
case Builtin::BI__sync_sub_and_fetch_8:
case Builtin::BI__sync_sub_and_fetch_16:
return emitBinaryAtomicPost<cir::SubOp>(*this, cir::AtomicFetchKind::Sub,
e);
case Builtin::BI__sync_and_and_fetch_1:
case Builtin::BI__sync_and_and_fetch_2:
case Builtin::BI__sync_and_and_fetch_4:
case Builtin::BI__sync_and_and_fetch_8:
case Builtin::BI__sync_and_and_fetch_16:
return emitBinaryAtomicPost<cir::AndOp>(*this, cir::AtomicFetchKind::And,
e);
case Builtin::BI__sync_or_and_fetch_1:
case Builtin::BI__sync_or_and_fetch_2:
case Builtin::BI__sync_or_and_fetch_4:
case Builtin::BI__sync_or_and_fetch_8:
case Builtin::BI__sync_or_and_fetch_16:
return emitBinaryAtomicPost<cir::OrOp>(*this, cir::AtomicFetchKind::Or, e);
case Builtin::BI__sync_xor_and_fetch_1:
case Builtin::BI__sync_xor_and_fetch_2:
case Builtin::BI__sync_xor_and_fetch_4:
case Builtin::BI__sync_xor_and_fetch_8:
case Builtin::BI__sync_xor_and_fetch_16:
return emitBinaryAtomicPost<cir::XorOp>(*this, cir::AtomicFetchKind::Xor,
e);
case Builtin::BI__sync_nand_and_fetch_1:
case Builtin::BI__sync_nand_and_fetch_2:
case Builtin::BI__sync_nand_and_fetch_4:
case Builtin::BI__sync_nand_and_fetch_8:
case Builtin::BI__sync_nand_and_fetch_16:
return emitBinaryAtomicPost<cir::AndOp>(*this, cir::AtomicFetchKind::Nand,
e, /*invert=*/true);
case Builtin::BI__sync_val_compare_and_swap_1:
case Builtin::BI__sync_val_compare_and_swap_2:
case Builtin::BI__sync_val_compare_and_swap_4:
case Builtin::BI__sync_val_compare_and_swap_8:
case Builtin::BI__sync_val_compare_and_swap_16:
case Builtin::BI__sync_bool_compare_and_swap_1:
case Builtin::BI__sync_bool_compare_and_swap_2:
case Builtin::BI__sync_bool_compare_and_swap_4:
case Builtin::BI__sync_bool_compare_and_swap_8:
case Builtin::BI__sync_bool_compare_and_swap_16:
case Builtin::BI__sync_swap_1:
case Builtin::BI__sync_swap_2:
case Builtin::BI__sync_swap_4:
case Builtin::BI__sync_swap_8:
case Builtin::BI__sync_swap_16:
case Builtin::BI__sync_lock_test_and_set_1:
case Builtin::BI__sync_lock_test_and_set_2:
case Builtin::BI__sync_lock_test_and_set_4:
case Builtin::BI__sync_lock_test_and_set_8:
case Builtin::BI__sync_lock_test_and_set_16:
case Builtin::BI__sync_lock_release_1:
case Builtin::BI__sync_lock_release_2:
case Builtin::BI__sync_lock_release_4:
case Builtin::BI__sync_lock_release_8:
case Builtin::BI__sync_lock_release_16:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__sync_synchronize: {
// We assume this is supposed to correspond to a C++0x-style
// sequentially-consistent fence (i.e. this is only usable for
// synchronization, not device I/O or anything like that). This intrinsic
// is really badly designed in the sense that in theory, there isn't
// any way to safely use it... but in practice, it mostly works
// to use it with non-atomic loads and stores to get acquire/release
// semantics.
cir::AtomicFenceOp::create(
builder, getLoc(e->getSourceRange()),
cir::MemOrder::SequentiallyConsistent,
cir::SyncScopeKindAttr::get(&getMLIRContext(),
cir::SyncScopeKind::System));
return RValue::get(nullptr);
}
case Builtin::BI__builtin_nontemporal_load: {
Address addr = emitPointerWithAlignment(e->getArg(0));
LValue lv = makeAddrLValue(addr, e->getType(),
LValueBaseInfo(AlignmentSource::Type));
lv.setNontemporal(true);
mlir::Value val = emitLoadOfScalar(lv, e->getExprLoc());
return RValue::get(val);
}
case Builtin::BI__builtin_nontemporal_store: {
mlir::Value val = emitScalarExpr(e->getArg(0));
Address addr = emitPointerWithAlignment(e->getArg(1));
val = emitToMemory(val, e->getArg(0)->getType());
LValue lv = makeAddrLValue(addr, e->getArg(0)->getType(),
LValueBaseInfo(AlignmentSource::Type));
lv.setNontemporal(true);
emitStoreOfScalar(val, lv, /*isInit=*/false);
return RValue::get(nullptr);
}
case Builtin::BI__c11_atomic_is_lock_free:
case Builtin::BI__atomic_is_lock_free:
return emitAtomicIsLockFree(*this, e, builtinID);
case Builtin::BI__atomic_test_and_set:
case Builtin::BI__atomic_clear:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__atomic_thread_fence:
case Builtin::BI__c11_atomic_thread_fence: {
emitAtomicFenceOp(*this, e, cir::SyncScopeKind::System);
return RValue::get(nullptr);
}
case Builtin::BI__atomic_signal_fence:
case Builtin::BI__c11_atomic_signal_fence: {
emitAtomicFenceOp(*this, e, cir::SyncScopeKind::SingleThread);
return RValue::get(nullptr);
}
case Builtin::BI__scoped_atomic_thread_fence:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_signbit:
case Builtin::BI__builtin_signbitf:
case Builtin::BI__builtin_signbitl: {
CIRGenFunction::CIRGenFPOptionsRAII fPOptsRAII(*this, e);
mlir::Location loc = getLoc(e->getBeginLoc());
mlir::Value value = emitScalarExpr(e->getArg(0));
mlir::Operation *signBitOp = cir::SignBitOp::create(builder, loc, value);
mlir::Value result = builder.createBoolToInt(signBitOp->getResult(0),
convertType(e->getType()));
return RValue::get(result);
}
case Builtin::BI__warn_memset_zero_len:
case Builtin::BI__annotation:
case Builtin::BI__builtin_annotation:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_addcb:
case Builtin::BI__builtin_addcs:
case Builtin::BI__builtin_addc:
case Builtin::BI__builtin_addcl:
case Builtin::BI__builtin_addcll:
case Builtin::BI__builtin_subcb:
case Builtin::BI__builtin_subcs:
case Builtin::BI__builtin_subc:
case Builtin::BI__builtin_subcl:
case Builtin::BI__builtin_subcll: {
// Multiprecision add/sub-with-carry. Lower as two chained checked
// add/sub overflow ops, matching classic CodeGen:
// sum1, carry1 = x +/- y
// result, carry2 = sum1 +/- carryin
// *carryout = carry1 | carry2
// All operands and the result share the builtin's integer type, so no
// encompassing-type widening is needed.
mlir::Value x = emitScalarExpr(e->getArg(0));
mlir::Value y = emitScalarExpr(e->getArg(1));
mlir::Value carryin = emitScalarExpr(e->getArg(2));
Address carryOutPtr = emitPointerWithAlignment(e->getArg(3));
mlir::Location loc = getLoc(e->getSourceRange());
mlir::Type resultTy = convertType(e->getType());
static constexpr unsigned addcBuiltins[] = {
Builtin::BI__builtin_addcb, Builtin::BI__builtin_addcs,
Builtin::BI__builtin_addc, Builtin::BI__builtin_addcl,
Builtin::BI__builtin_addcll};
bool isAdd = llvm::is_contained(addcBuiltins, builtinID);
mlir::Value sum1, carry1, sum2, carry2;
if (isAdd) {
std::tie(sum1, carry1) =
emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultTy, x, y);
std::tie(sum2, carry2) = emitOverflowOp<cir::AddOverflowOp>(
builder, loc, resultTy, sum1, carryin);
} else {
std::tie(sum1, carry1) =
emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultTy, x, y);
std::tie(sum2, carry2) = emitOverflowOp<cir::SubOverflowOp>(
builder, loc, resultTy, sum1, carryin);
}
// Combine the two carry bits, then widen to the result integer type.
mlir::Value carryOut = builder.createBoolToInt(
builder.createOr(loc, carry1, carry2), resultTy);
builder.createStore(loc, carryOut, carryOutPtr);
return RValue::get(sum2);
}
case Builtin::BI__builtin_add_overflow:
case Builtin::BI__builtin_sub_overflow:
case Builtin::BI__builtin_mul_overflow: {
const clang::Expr *leftArg = e->getArg(0);
const clang::Expr *rightArg = e->getArg(1);
const clang::Expr *resultArg = e->getArg(2);
clang::QualType resultQTy =
resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
WidthAndSignedness leftInfo =
getIntegerWidthAndSignedness(cgm.getASTContext(), leftArg->getType());
WidthAndSignedness rightInfo =
getIntegerWidthAndSignedness(cgm.getASTContext(), rightArg->getType());
WidthAndSignedness resultInfo =
getIntegerWidthAndSignedness(cgm.getASTContext(), resultQTy);
// Note we compute the encompassing type with the consideration to the
// result type, so later in LLVM lowering we don't get redundant integral
// extension casts.
WidthAndSignedness encompassingInfo =
EncompassingIntegerType({leftInfo, rightInfo, resultInfo});
auto encompassingCIRTy = cir::IntType::get(
&getMLIRContext(), encompassingInfo.width, encompassingInfo.isSigned);
mlir::Type resultCIRTy = cgm.convertType(resultQTy);
mlir::Value x = emitScalarExpr(leftArg);
mlir::Value y = emitScalarExpr(rightArg);
Address resultPtr = emitPointerWithAlignment(resultArg);
// Extend each operand to the encompassing type, if necessary.
if (x.getType() != encompassingCIRTy) {
x = builder.createCast(mlir::isa<cir::BoolType>(x.getType())
? cir::CastKind::bool_to_int
: cir::CastKind::integral,
x, encompassingCIRTy);
}
if (y.getType() != encompassingCIRTy) {
y = builder.createCast(mlir::isa<cir::BoolType>(y.getType())
? cir::CastKind::bool_to_int
: cir::CastKind::integral,
y, encompassingCIRTy);
}
// Perform the operation on the extended values.
mlir::Location loc = getLoc(e->getSourceRange());
mlir::Value result, overflow;
switch (builtinID) {
default:
llvm_unreachable("Unknown overflow builtin id.");
case Builtin::BI__builtin_add_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
case Builtin::BI__builtin_sub_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
case Builtin::BI__builtin_mul_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
}
// Here is a slight difference from the original clang CodeGen:
// - In the original clang CodeGen, the checked arithmetic result is
// first computed as a value of the encompassing type, and then it is
// truncated to the actual result type with a second overflow checking.
// - In CIRGen, the checked arithmetic operation directly produce the
// checked arithmetic result in its expected type, which may be a
// `cir.bool`.
//
// So we don't need a truncation and a second overflow checking here.
// Finally, store the result using the pointer.
bool isVolatile =
resultArg->getType()->getPointeeType().isVolatileQualified();
builder.createStore(loc, result, resultPtr, isVolatile);
return RValue::get(overflow);
}
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
case Builtin::BI__builtin_usub_overflow:
case Builtin::BI__builtin_usubl_overflow:
case Builtin::BI__builtin_usubll_overflow:
case Builtin::BI__builtin_umul_overflow:
case Builtin::BI__builtin_umull_overflow:
case Builtin::BI__builtin_umulll_overflow:
case Builtin::BI__builtin_sadd_overflow:
case Builtin::BI__builtin_saddl_overflow:
case Builtin::BI__builtin_saddll_overflow:
case Builtin::BI__builtin_ssub_overflow:
case Builtin::BI__builtin_ssubl_overflow:
case Builtin::BI__builtin_ssubll_overflow:
case Builtin::BI__builtin_smul_overflow:
case Builtin::BI__builtin_smull_overflow:
case Builtin::BI__builtin_smulll_overflow: {
// Scalarize our inputs.
mlir::Value x = emitScalarExpr(e->getArg(0));
mlir::Value y = emitScalarExpr(e->getArg(1));
const clang::Expr *resultArg = e->getArg(2);
Address resultPtr = emitPointerWithAlignment(resultArg);
clang::QualType resultQTy =
resultArg->getType()->castAs<clang::PointerType>()->getPointeeType();
auto resultCIRTy = mlir::cast<cir::IntType>(cgm.convertType(resultQTy));
// Create the appropriate overflow-checked arithmetic operation.
mlir::Location loc = getLoc(e->getSourceRange());
mlir::Value result, overflow;
switch (builtinID) {
default:
llvm_unreachable("Unknown overflow builtin id.");
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
case Builtin::BI__builtin_sadd_overflow:
case Builtin::BI__builtin_saddl_overflow:
case Builtin::BI__builtin_saddll_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::AddOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
case Builtin::BI__builtin_usub_overflow:
case Builtin::BI__builtin_usubl_overflow:
case Builtin::BI__builtin_usubll_overflow:
case Builtin::BI__builtin_ssub_overflow:
case Builtin::BI__builtin_ssubl_overflow:
case Builtin::BI__builtin_ssubll_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::SubOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
case Builtin::BI__builtin_umul_overflow:
case Builtin::BI__builtin_umull_overflow:
case Builtin::BI__builtin_umulll_overflow:
case Builtin::BI__builtin_smul_overflow:
case Builtin::BI__builtin_smull_overflow:
case Builtin::BI__builtin_smulll_overflow:
std::tie(result, overflow) =
emitOverflowOp<cir::MulOverflowOp>(builder, loc, resultCIRTy, x, y);
break;
}
bool isVolatile =
resultArg->getType()->getPointeeType().isVolatileQualified();
builder.createStore(loc, emitToMemory(result, resultQTy), resultPtr,
isVolatile);
return RValue::get(overflow);
}
case Builtin::BIaddressof:
case Builtin::BI__addressof:
case Builtin::BI__builtin_addressof:
return RValue::get(emitLValue(e->getArg(0)).getPointer());
case Builtin::BI__builtin_function_start:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_operator_new:
return emitNewOrDeleteBuiltinCall(
e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_New);
case Builtin::BI__builtin_operator_delete:
emitNewOrDeleteBuiltinCall(
e->getCallee()->getType()->castAs<FunctionProtoType>(), e, OO_Delete);
return RValue::get(nullptr);
case Builtin::BI__builtin_is_aligned:
case Builtin::BI__builtin_align_up:
case Builtin::BI__builtin_align_down:
case Builtin::BI__noop:
case Builtin::BI__builtin_call_with_static_chain:
case Builtin::BI_InterlockedExchange8:
case Builtin::BI_InterlockedExchange16:
case Builtin::BI_InterlockedExchange:
case Builtin::BI_InterlockedExchangePointer:
case Builtin::BI_InterlockedCompareExchangePointer:
case Builtin::BI_InterlockedCompareExchangePointer_nf:
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
case Builtin::BI_InterlockedCompareExchange64:
case Builtin::BI_InterlockedIncrement16:
case Builtin::BI_InterlockedIncrement:
case Builtin::BI_InterlockedDecrement16:
case Builtin::BI_InterlockedDecrement:
case Builtin::BI_InterlockedAnd8:
case Builtin::BI_InterlockedAnd16:
case Builtin::BI_InterlockedAnd:
case Builtin::BI_InterlockedExchangeAdd8:
case Builtin::BI_InterlockedExchangeAdd16:
case Builtin::BI_InterlockedExchangeAdd:
case Builtin::BI_InterlockedExchangeSub8:
case Builtin::BI_InterlockedExchangeSub16:
case Builtin::BI_InterlockedExchangeSub:
case Builtin::BI_InterlockedOr8:
case Builtin::BI_InterlockedOr16:
case Builtin::BI_InterlockedOr:
case Builtin::BI_InterlockedXor8:
case Builtin::BI_InterlockedXor16:
case Builtin::BI_InterlockedXor:
case Builtin::BI_bittest64:
case Builtin::BI_bittest:
case Builtin::BI_bittestandcomplement64:
case Builtin::BI_bittestandcomplement:
case Builtin::BI_bittestandreset64:
case Builtin::BI_bittestandreset:
case Builtin::BI_bittestandset64:
case Builtin::BI_bittestandset:
case Builtin::BI_interlockedbittestandreset:
case Builtin::BI_interlockedbittestandreset64:
case Builtin::BI_interlockedbittestandreset64_acq:
case Builtin::BI_interlockedbittestandreset64_rel:
case Builtin::BI_interlockedbittestandreset64_nf:
case Builtin::BI_interlockedbittestandset64:
case Builtin::BI_interlockedbittestandset64_acq:
case Builtin::BI_interlockedbittestandset64_rel:
case Builtin::BI_interlockedbittestandset64_nf:
case Builtin::BI_interlockedbittestandset:
case Builtin::BI_interlockedbittestandset_acq:
case Builtin::BI_interlockedbittestandset_rel:
case Builtin::BI_interlockedbittestandset_nf:
case Builtin::BI_interlockedbittestandreset_acq:
case Builtin::BI_interlockedbittestandreset_rel:
case Builtin::BI_interlockedbittestandreset_nf:
case Builtin::BI__iso_volatile_load8:
case Builtin::BI__iso_volatile_load16:
case Builtin::BI__iso_volatile_load32:
case Builtin::BI__iso_volatile_load64:
case Builtin::BI__iso_volatile_store8:
case Builtin::BI__iso_volatile_store16:
case Builtin::BI__iso_volatile_store32:
case Builtin::BI__iso_volatile_store64:
case Builtin::BI__builtin_ptrauth_sign_constant:
case Builtin::BI__builtin_ptrauth_auth:
case Builtin::BI__builtin_ptrauth_auth_and_resign:
case Builtin::BI__builtin_ptrauth_blend_discriminator:
case Builtin::BI__builtin_ptrauth_sign_generic_data:
case Builtin::BI__builtin_ptrauth_sign_unauthenticated:
case Builtin::BI__builtin_ptrauth_strip:
case Builtin::BI__builtin_get_vtable_pointer:
case Builtin::BI__exception_code:
case Builtin::BI_exception_code:
case Builtin::BI__exception_info:
case Builtin::BI_exception_info:
case Builtin::BI__abnormal_termination:
case Builtin::BI_abnormal_termination:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI_setjmpex:
case Builtin::BI_setjmp:
if (getTarget().getTriple().isOSMSVCRT()) {
cgm.errorNYI(e->getSourceRange(), "setjmp/setjmpex on MSVCRT");
return getUndefRValue(e->getType());
}
// Else break and this will be handled as a library call.
break;
case Builtin::BImove:
case Builtin::BImove_if_noexcept:
case Builtin::BIforward:
case Builtin::BIforward_like:
case Builtin::BIas_const:
return RValue::get(emitLValue(e->getArg(0)).getPointer());
case Builtin::BI__GetExceptionInfo:
case Builtin::BI__fastfail:
case Builtin::BIread_pipe:
case Builtin::BIwrite_pipe:
case Builtin::BIreserve_read_pipe:
case Builtin::BIreserve_write_pipe:
case Builtin::BIwork_group_reserve_read_pipe:
case Builtin::BIwork_group_reserve_write_pipe:
case Builtin::BIsub_group_reserve_read_pipe:
case Builtin::BIsub_group_reserve_write_pipe:
case Builtin::BIcommit_read_pipe:
case Builtin::BIcommit_write_pipe:
case Builtin::BIwork_group_commit_read_pipe:
case Builtin::BIwork_group_commit_write_pipe:
case Builtin::BIsub_group_commit_read_pipe:
case Builtin::BIsub_group_commit_write_pipe:
case Builtin::BIget_pipe_num_packets:
case Builtin::BIget_pipe_max_packets:
case Builtin::BIto_global:
case Builtin::BIto_local:
case Builtin::BIto_private:
case Builtin::BIenqueue_kernel:
case Builtin::BIget_kernel_work_group_size:
case Builtin::BIget_kernel_preferred_work_group_size_multiple:
case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
case Builtin::BIget_kernel_sub_group_count_for_ndrange:
case Builtin::BI__builtin_store_half:
case Builtin::BI__builtin_store_halff:
case Builtin::BI__builtin_load_half:
case Builtin::BI__builtin_load_halff:
return errorBuiltinNYI(*this, e, builtinID);
case Builtin::BI__builtin_printf:
case Builtin::BIprintf:
if (getTarget().getTriple().isNVPTX() ||
getTarget().getTriple().isAMDGCN() ||
(getTarget().getTriple().isSPIRV() &&
getTarget().getTriple().getVendor() == llvm::Triple::AMD)) {
if (getTarget().getTriple().isNVPTX())
return RValue::get(emitNVPTXDevicePrintfCallExpr(e));
if ((getTarget().getTriple().isAMDGCN() ||
getTarget().getTriple().isSPIRV()) &&
getLangOpts().HIP)
return errorBuiltinNYI(*this, e, builtinID);
}
break;
case Builtin::BI__builtin_canonicalize:
case Builtin::BI__builtin_canonicalizef:
case Builtin::BI__builtin_canonicalizef16:
case Builtin::BI__builtin_canonicalizel:
case Builtin::BI__builtin_thread_pointer:
case Builtin::BI__builtin_os_log_format:
case Builtin::BI__xray_customevent:
case Builtin::BI__xray_typedevent:
case Builtin::BI__builtin_ms_va_start:
case Builtin::BI__builtin_ms_va_end:
case Builtin::BI__builtin_ms_va_copy:
case Builtin::BI__builtin_get_device_side_mangled_name:
return errorBuiltinNYI(*this, e, builtinID);
}
// If this is an alias for a lib function (e.g. __builtin_sin), emit
// the call using the normal call path, but using the unmangled
// version of the function name.
if (!shouldEmitBuiltinAsIR(builtinID, getContext().BuiltinInfo, *this) &&
getContext().BuiltinInfo.isLibFunction(builtinID))
return emitLibraryCall(*this, fd, e,
cgm.getBuiltinLibFunction(fd, builtinID));
// If this is a predefined lib function (e.g. malloc), emit the call
// using exactly the normal call path.
if (getContext().BuiltinInfo.isPredefinedLibFunction(builtinID))
return emitLibraryCall(*this, fd, e,
emitScalarExpr(e->getCallee()).getDefiningOp());
// See if we have a target specific intrinsic.
std::string name = getContext().BuiltinInfo.getName(builtinID);
Intrinsic::ID intrinsicID = Intrinsic::not_intrinsic;
StringRef prefix =
llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
if (!prefix.empty()) {
intrinsicID = Intrinsic::getIntrinsicForClangBuiltin(prefix, name);
// NOTE we don't need to perform a compatibility flag check here since the
// intrinsics are declared in Builtins*.def via LANGBUILTIN which filter the
// MS builtins via ALL_MS_LANGUAGES and are filtered earlier.
if (intrinsicID == Intrinsic::not_intrinsic)
intrinsicID = Intrinsic::getIntrinsicForMSBuiltin(prefix, name);
}
if (intrinsicID != Intrinsic::not_intrinsic) {
unsigned iceArguments = 0;
ASTContext::GetBuiltinTypeError error;
getContext().GetBuiltinType(builtinID, error, &iceArguments);
assert(error == ASTContext::GE_None && "Should not codegen an error");
StringRef name = Intrinsic::getName(intrinsicID);
// cir::LLVMIntrinsicCallOp expects intrinsic name to not have prefix
// "llvm." For example, `llvm.nvvm.barrier0` should be passed as
// `nvvm.barrier0`.
assert(name.starts_with("llvm.") && "expected llvm. prefix");
name = name.drop_front(/*strlen("llvm.")=*/5);
cir::FuncType intrinsicType =
getIntrinsicType(*this, &getMLIRContext(), intrinsicID);
SmallVector<mlir::Value> args;
const FunctionDecl *fd = e->getDirectCallee();
for (unsigned i = 0; i < e->getNumArgs(); i++) {
mlir::Value argValue =
emitScalarOrConstFoldImmArg(iceArguments, i, e->getArg(i));
// If the intrinsic arg type is different from the builtin arg type
// we need to do a bit cast.
mlir::Type argType = argValue.getType();
mlir::Type expectedTy = intrinsicType.getInput(i);
// Correct integer signedness based on AST parameter type
mlir::Type correctedExpectedTy = expectedTy;
if (fd && i < fd->getNumParams()) {
correctedExpectedTy = correctIntegerSignedness(
expectedTy, fd->getParamDecl(i)->getType(), &getMLIRContext());
}
if (mlir::isa<cir::PointerType>(expectedTy)) {
bool argIsPointer = mlir::isa<cir::PointerType>(argType);
bool argIsVectorOfPointer = false;
if (auto vecTy = dyn_cast<mlir::VectorType>(argType))
argIsVectorOfPointer =
mlir::isa<cir::PointerType>(vecTy.getElementType());
if (!argIsPointer && !argIsVectorOfPointer) {
cgm.errorNYI(
e->getSourceRange(),
"intrinsic expects a pointer type (NYI for non-pointer)");
return getUndefRValue(e->getType());
}
// Pointer handling (address-space cast / bitcast fallback).
if (argType != expectedTy)
argValue = getCorrectedPtr(argValue, expectedTy, builder);
} else {
// Non-pointer expected type: if needed, bitcast to the corrected
// expected type to match signedness/representation.
if (argType != correctedExpectedTy)
argValue = builder.createBitcast(argValue, correctedExpectedTy);
}
args.push_back(argValue);
}
// Correct return type signedness based on AST return type before creating
// the call, avoiding unnecessary casts in the IR.
mlir::Type correctedReturnType = intrinsicType.getReturnType();
if (fd) {
correctedReturnType =
correctIntegerSignedness(intrinsicType.getReturnType(),
fd->getReturnType(), &getMLIRContext());
}
cir::LLVMIntrinsicCallOp intrinsicCall = cir::LLVMIntrinsicCallOp::create(
builder, getLoc(e->getExprLoc()), builder.getStringAttr(name),
correctedReturnType, args);
mlir::Value intrinsicRes = intrinsicCall.getResult();
if (isa<cir::VoidType>(correctedReturnType))
return RValue::get(nullptr);
return RValue::get(intrinsicRes);
}
// Some target-specific builtins can have aggregate return values, e.g.
// __builtin_arm_mve_vld2q_u32. So if the result is an aggregate, force
// returnValue to be non-null, so that the target-specific emission code can
// always just emit into it.
cir::TypeEvaluationKind evalKind = getEvaluationKind(e->getType());
if (evalKind == cir::TEK_Aggregate && returnValue.isNull()) {
cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
return getUndefRValue(e->getType());
}
// Now see if we can emit a target-specific builtin.
// FIXME: This is a temporary mechanism (double-optional semantics) that will
// go away once everything is implemented:
// 1. return `mlir::Value{}` for cases where we have issued the diagnostic.
// 2. return `std::nullopt` in cases where we didn't issue a diagnostic
// but also didn't handle the builtin.
if (std::optional<mlir::Value> rst =
emitTargetBuiltinExpr(builtinID, e, returnValue)) {
mlir::Value v = rst.value();
// CIR dialect operations may have no results, no values will be returned
// even if it executes successfully.
if (!v)
return RValue::get(nullptr);
switch (evalKind) {
case cir::TEK_Scalar:
if (mlir::isa<cir::VoidType>(v.getType()))
return RValue::get(nullptr);
return RValue::get(v);
case cir::TEK_Aggregate:
cgm.errorNYI(e->getSourceRange(), "aggregate return value from builtin");
return getUndefRValue(e->getType());
case cir::TEK_Complex:
llvm_unreachable("No current target builtin returns complex");
}
llvm_unreachable("Bad evaluation kind in EmitBuiltinExpr");
}
cgm.errorNYI(e->getSourceRange(),
std::string("unimplemented builtin call: ") +
getContext().BuiltinInfo.getName(builtinID));
return getUndefRValue(e->getType());
}
static std::optional<mlir::Value>
emitTargetArchBuiltinExpr(CIRGenFunction *cgf, unsigned builtinID,
const CallExpr *e, ReturnValueSlot &returnValue,
llvm::Triple::ArchType arch) {
// When compiling in HipStdPar mode we have to be conservative in rejecting
// target specific features in the FE, and defer the possible error to the
// AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is
// referenced by an accelerator executable function, we emit an error.
// Returning nullptr here leads to the builtin being handled in
// EmitStdParUnsupportedBuiltin.
if (cgf->getLangOpts().HIPStdPar && cgf->getLangOpts().CUDAIsDevice &&
arch != cgf->getTarget().getTriple().getArch())
return std::nullopt;
switch (arch) {
case llvm::Triple::arm:
case llvm::Triple::armeb:
case llvm::Triple::thumb:
case llvm::Triple::thumbeb:
// These are actually NYI, but that will be reported by emitBuiltinExpr.
// At this point, we don't even know that the builtin is target-specific.
return std::nullopt;
case llvm::Triple::aarch64:
case llvm::Triple::aarch64_32:
case llvm::Triple::aarch64_be:
return cgf->emitAArch64BuiltinExpr(builtinID, e, returnValue, arch);
case llvm::Triple::bpfeb:
case llvm::Triple::bpfel:
// These are actually NYI, but that will be reported by emitBuiltinExpr.
// At this point, we don't even know that the builtin is target-specific.
return std::nullopt;
case llvm::Triple::x86:
case llvm::Triple::x86_64:
return cgf->emitX86BuiltinExpr(builtinID, e);
case llvm::Triple::ppc:
case llvm::Triple::ppcle:
case llvm::Triple::ppc64:
case llvm::Triple::ppc64le:
case llvm::Triple::r600:
// These are actually NYI, but that will be reported by emitBuiltinExpr.
// At this point, we don't even know that the builtin is target-specific.
return std::nullopt;
case llvm::Triple::amdgpu:
return cgf->emitAMDGPUBuiltinExpr(builtinID, e);
case llvm::Triple::systemz:
return std::nullopt;
case llvm::Triple::nvptx:
case llvm::Triple::nvptx64:
return cgf->emitNVPTXBuiltinExpr(builtinID, e);
case llvm::Triple::wasm32:
case llvm::Triple::wasm64:
case llvm::Triple::hexagon:
// These are actually NYI, but that will be reported by emitBuiltinExpr.
// At this point, we don't even know that the builtin is target-specific.
return std::nullopt;
case llvm::Triple::riscv32:
case llvm::Triple::riscv64:
return cgf->emitRISCVBuiltinExpr(builtinID, e);
default:
return std::nullopt;
}
}
std::optional<mlir::Value>
CIRGenFunction::emitTargetBuiltinExpr(unsigned builtinID, const CallExpr *e,
ReturnValueSlot &returnValue) {
if (getContext().BuiltinInfo.isAuxBuiltinID(builtinID)) {
assert(getContext().getAuxTargetInfo() && "Missing aux target info");
return emitTargetArchBuiltinExpr(
this, getContext().BuiltinInfo.getAuxBuiltinID(builtinID), e,
returnValue, getContext().getAuxTargetInfo()->getTriple().getArch());
}
return emitTargetArchBuiltinExpr(this, builtinID, e, returnValue,
getTarget().getTriple().getArch());
}
mlir::Value CIRGenFunction::emitScalarOrConstFoldImmArg(
const unsigned iceArguments, const unsigned idx, const Expr *argExpr) {
mlir::Value arg = {};
if ((iceArguments & (1 << idx)) == 0) {
arg = emitScalarExpr(argExpr);
} else {
// If this is required to be a constant, constant fold it so that we
// know that the generated intrinsic gets a ConstantInt.
const std::optional<llvm::APSInt> result =
argExpr->getIntegerConstantExpr(getContext());
assert(result && "Expected argument to be a constant");
arg = builder.getConstInt(getLoc(argExpr->getSourceRange()), *result);
}
return arg;
}
/// Given a builtin id for a function like "__builtin_fabsf", return a Function*
/// for "fabsf".
cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
unsigned builtinID) {
assert(astContext.BuiltinInfo.isLibFunction(builtinID));
// Get the name, skip over the __builtin_ prefix (if necessary). We may have
// to build this up so provide a small stack buffer to handle the vast
// majority of names.
llvm::SmallString<64> name;
assert(!cir::MissingFeatures::asmLabelAttr());
name = astContext.BuiltinInfo.getName(builtinID).substr(10);
GlobalDecl d(fd);
mlir::Type type = convertType(fd->getType());
return getOrCreateCIRFunction(name, type, d, /*forVTable=*/false);
}
mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {
mlir::Value argValue = evaluateExprAsBool(e);
if (!sanOpts.has(SanitizerKind::Builtin))
return argValue;
assert(!cir::MissingFeatures::sanitizers());
cgm.errorNYI(e->getSourceRange(),
"emitCheckedArgForAssume: sanitizers are NYI");
return {};
}
void CIRGenFunction::emitVAStart(mlir::Value vaList) {
// LLVM codegen casts to *i8, no real gain on doing this for CIRGen this
// early, defer to LLVM lowering.
cir::VAStartOp::create(builder, vaList.getLoc(), vaList);
}
void CIRGenFunction::emitVAEnd(mlir::Value vaList) {
cir::VAEndOp::create(builder, vaList.getLoc(), vaList);
}
// FIXME(cir): This completely abstracts away the ABI with a generic CIR Op. By
// default this lowers to llvm.va_arg which is incomplete and not ABI-compliant
// on most targets so cir.va_arg will need some ABI handling in LoweringPrepare
mlir::Value CIRGenFunction::emitVAArg(VAArgExpr *ve) {
assert(!cir::MissingFeatures::msabi());
assert(!cir::MissingFeatures::vlas());
mlir::Location loc = cgm.getLoc(ve->getExprLoc());
mlir::Type type = convertType(ve->getType());
mlir::Value vaList = emitVAListRef(ve->getSubExpr()).getPointer();
return cir::VAArgOp::create(builder, loc, type, vaList);
}
mlir::Value CIRGenFunction::emitBuiltinObjectSize(const Expr *e, unsigned type,
cir::IntType resType,
mlir::Value emittedE,
bool isDynamic) {
// If this is a pass_object_size parameter, load the implicit size arg.
//
// BOS type compatibility: a pass_object_size annotation with one type can
// satisfy a __builtin_object_size query with a different type when the
// annotated type is a safe approximation. Type 0 (max, whole object) is
// an overestimate for type 1 (max, closest surrounding subobject), and
// type 3 (min, closest surrounding subobject) is an underestimate for
// type 2 (min, whole object).
enum BOSType {
MaxWholeObject = 0,
MaxSubobject = 1,
MinWholeObject = 2,
MinSubobject = 3,
};
if (auto *dre = dyn_cast<DeclRefExpr>(e->IgnoreParenImpCasts())) {
auto *param = dyn_cast<ParmVarDecl>(dre->getDecl());
auto *objSizeAttr = dre->getDecl()->getAttr<PassObjectSizeAttr>();
if (param && objSizeAttr) {
auto from = objSizeAttr->getType();
bool compatible = from == static_cast<int>(type) ||
(from == MaxWholeObject && type == MaxSubobject) ||
(from == MinSubobject && type == MinWholeObject);
if (compatible) {
const ImplicitParamDecl *sizeDecl = sizeArguments.lookup(param);
assert(sizeDecl && "expected pass_object_size implicit param");
DeclMapTy::iterator declIter = localDeclMap.find(sizeDecl);
assert(declIter != localDeclMap.end());
Address addr = declIter->second;
return emitLoadOfScalar(addr, /*volatile=*/false,
getContext().getSizeType(), e->getBeginLoc(),
LValueBaseInfo(AlignmentSource::Decl));
}
}
}
// LLVM can't handle type=3 appropriately, and __builtin_object_size shouldn't
// evaluate e for side-effects. In either case, just like original LLVM
// lowering, we shouldn't lower to `cir.objsize` but to a constant instead.
if (type == 3 || (!emittedE && e->HasSideEffects(getContext())))
return builder.getConstInt(getLoc(e->getSourceRange()), resType,
(type & 2) ? 0 : -1);
mlir::Value ptr = emittedE ? emittedE : emitScalarExpr(e);
assert(mlir::isa<cir::PointerType>(ptr.getType()) &&
"Non-pointer passed to __builtin_object_size?");
assert(!cir::MissingFeatures::countedBySize());
// Extract the min/max mode from type. CIR only supports type 0
// (max, whole object) and type 2 (min, whole object), not type 1 or 3
// (closest subobject variants).
const bool min = ((type & 2) != 0);
// For GCC compatibility, __builtin_object_size treats NULL as unknown size.
auto op =
cir::ObjSizeOp::create(builder, getLoc(e->getSourceRange()), resType, ptr,
min, /*nullUnknown=*/true, isDynamic);
return op.getResult();
}
mlir::Value CIRGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *e, unsigned type, cir::IntType resType, mlir::Value emittedE,
bool isDynamic) {
if (std::optional<uint64_t> objectSize =
e->tryEvaluateObjectSize(getContext(), type))
return builder.getConstInt(getLoc(e->getSourceRange()), resType,
*objectSize);
return emitBuiltinObjectSize(e, type, resType, emittedE, isDynamic);
}