| //====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file implements lowering of CIR operations to LLVMIR. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "LowerToLLVM.h" |
| |
| #include <array> |
| #include <optional> |
| |
| #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| #include "mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h" |
| #include "mlir/Dialect/DLTI/DLTI.h" |
| #include "mlir/Dialect/Func/IR/FuncOps.h" |
| #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
| #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
| #include "mlir/Dialect/OpenMP/Transforms/Passes.h" |
| #include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h" |
| #include "mlir/IR/BuiltinAttributes.h" |
| #include "mlir/IR/BuiltinDialect.h" |
| #include "mlir/IR/BuiltinOps.h" |
| #include "mlir/IR/Location.h" |
| #include "mlir/IR/Types.h" |
| #include "mlir/Pass/Pass.h" |
| #include "mlir/Pass/PassManager.h" |
| #include "mlir/Support/LLVM.h" |
| #include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Export.h" |
| #include "mlir/Transforms/DialectConversion.h" |
| #include "clang/CIR/Dialect/IR/CIRAttrs.h" |
| #include "clang/CIR/Dialect/IR/CIRDialect.h" |
| #include "clang/CIR/Dialect/IR/CIRTypes.h" |
| #include "clang/CIR/Dialect/Passes.h" |
| #include "clang/CIR/Dialect/Transforms/CIRTransformUtils.h" |
| #include "clang/CIR/LoweringHelpers.h" |
| #include "clang/CIR/MissingFeatures.h" |
| #include "clang/CIR/Passes.h" |
| #include "llvm/ADT/MapVector.h" |
| #include "llvm/ADT/StringMap.h" |
| #include "llvm/ADT/TypeSwitch.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/Support/Casting.h" |
| #include "llvm/Support/ErrorHandling.h" |
| #include "llvm/Support/TimeProfiler.h" |
| #include "llvm/Support/VirtualFileSystem.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| using namespace cir; |
| using namespace llvm; |
| |
| namespace cir { |
| namespace direct { |
| |
| //===----------------------------------------------------------------------===// |
| // Helper Methods |
| //===----------------------------------------------------------------------===// |
| |
| namespace { |
| /// If the given type is a vector type, return the vector's element type. |
| /// Otherwise return the given type unchanged. |
| mlir::Type elementTypeIfVector(mlir::Type type) { |
| return llvm::TypeSwitch<mlir::Type, mlir::Type>(type) |
| .Case<cir::VectorType, mlir::VectorType>( |
| [](auto p) { return p.getElementType(); }) |
| .Default([](mlir::Type p) { return p; }); |
| } |
| } // namespace |
| |
| /// In-memory storage width in bits for a _BitInt(N): N rounded up to the type's |
| /// ABI alignment. This equals sizeof(_BitInt(N)) * 8 on the default target |
| /// (e.g. _BitInt(6) -> 8, _BitInt(17) -> 32, _BitInt(128) -> 128). |
| static unsigned getBitIntMemoryStorageBits(cir::IntType ty, |
| const mlir::DataLayout &dataLayout) { |
| uint64_t alignBits = ty.getABIAlignment(dataLayout, {}) * 8; |
| return llvm::alignTo(ty.getWidth(), alignBits); |
| } |
| |
| /// A _BitInt(N) whose padded storage integer iM has a larger alloc size than |
| /// its M/8 store size is laid out by clang as a byte array, not a plain integer |
| /// (e.g. _BitInt(129) -> i192 with alloc size 32 != store size 24). That |
| /// "split" storage form is not yet implemented; lowerings must detect it and |
| /// report errorNYI rather than emit the wrong-sized integer. |
| static bool isSplitStorageBitInt(cir::IntType ty, |
| const mlir::DataLayout &dataLayout) { |
| if (!ty.isBitInt()) |
| return false; |
| unsigned storageBits = getBitIntMemoryStorageBits(ty, dataLayout); |
| auto storageTy = mlir::IntegerType::get(ty.getContext(), storageBits); |
| uint64_t storeSize = storageBits / 8; |
| uint64_t allocSize = |
| llvm::alignTo(storeSize, dataLayout.getTypeABIAlignment(storageTy)); |
| return allocSize != storeSize; |
| } |
| |
| /// Given a type convertor and a data layout, convert the given type to a type |
| /// that is suitable for memory operations. For example, this can be used to |
| /// lower cir.bool accesses to i8. |
| static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter, |
| mlir::DataLayout const &dataLayout, |
| mlir::Type type) { |
| // TODO(cir): Handle other types similarly to clang's codegen |
| // convertTypeForMemory |
| if (isa<cir::BoolType>(type)) { |
| return mlir::IntegerType::get(type.getContext(), |
| dataLayout.getTypeSizeInBits(type)); |
| } |
| |
| if (auto vecTy = mlir::dyn_cast<cir::VectorType>(type)) { |
| if (mlir::isa<cir::BoolType>(vecTy.getElementType())) { |
| assert(!cir::MissingFeatures::hlsl()); |
| // Pad to at least one byte. |
| uint64_t bytePadded = std::max<uint64_t>(vecTy.getSize(), 8); |
| return mlir::IntegerType::get(type.getContext(), bytePadded); |
| } |
| } |
| |
| // _BitInt(N) keeps its literal width as a value but is stored in a padded |
| // integer iM in memory, the same way bool is i1 as a value and i8 in memory. |
| // The byte-array storage form for wide split widths is not implemented; a |
| // null return signals that, and op lowerings turn it into errorNYI. |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(type); |
| intTy && intTy.isBitInt()) { |
| if (isSplitStorageBitInt(intTy, dataLayout)) |
| return {}; |
| return mlir::IntegerType::get( |
| type.getContext(), getBitIntMemoryStorageBits(intTy, dataLayout)); |
| } |
| |
| return converter.convertType(type); |
| } |
| |
| /// Alignment to use for a memory access whose op carries no explicit alignment. |
| /// For _BitInt the storage integer iM's ABI alignment (e.g. i128's 16) |
| /// over-aligns the value, so use the CIR _BitInt ABI alignment (e.g. 8). |
| static uint64_t getMemoryFallbackAlignment(mlir::Type cirType, |
| mlir::Type llvmMemType, |
| const mlir::DataLayout &dataLayout) { |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(cirType); |
| intTy && intTy.isBitInt()) |
| return intTy.getABIAlignment(dataLayout, {}); |
| return dataLayout.getTypeABIAlignment(llvmMemType); |
| } |
| |
| static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src, |
| mlir::IntegerType dstTy, |
| bool isSigned = false) { |
| mlir::Type srcTy = src.getType(); |
| assert(mlir::isa<mlir::IntegerType>(srcTy)); |
| |
| unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth(); |
| unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth(); |
| mlir::Location loc = src.getLoc(); |
| |
| if (dstWidth > srcWidth && isSigned) |
| return mlir::LLVM::SExtOp::create(bld, loc, dstTy, src); |
| if (dstWidth > srcWidth) |
| return mlir::LLVM::ZExtOp::create(bld, loc, dstTy, src); |
| if (dstWidth < srcWidth) |
| return mlir::LLVM::TruncOp::create(bld, loc, dstTy, src); |
| return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src); |
| } |
| |
| /// Cast a _BitInt(N) value between its literal width iN and its padded |
| /// in-memory storage iM (sign/zero-extend to memory per signedness so the |
| /// padding bits are well-defined, truncate back on load; matches classic |
| /// CodeGen). Callers must first rule out the unsupported byte-array storage |
| /// form (isSplitStorageBitInt / a null convertTypeForMemory result). |
| static mlir::Value |
| castBitIntMemoryStorage(mlir::ConversionPatternRewriter &rewriter, |
| const mlir::DataLayout &dataLayout, cir::IntType intTy, |
| mlir::Value value, bool toMemory) { |
| unsigned storageBits = getBitIntMemoryStorageBits(intTy, dataLayout); |
| if (storageBits == intTy.getWidth()) |
| return value; |
| unsigned dstBits = toMemory ? storageBits : intTy.getWidth(); |
| return createIntCast(rewriter, value, rewriter.getIntegerType(dstBits), |
| /*isSigned=*/toMemory && intTy.isSigned()); |
| } |
| |
| static mlir::LLVM::Visibility |
| lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) { |
| switch (visibilityKind) { |
| case cir::VisibilityKind::Default: |
| return ::mlir::LLVM::Visibility::Default; |
| case cir::VisibilityKind::Hidden: |
| return ::mlir::LLVM::Visibility::Hidden; |
| case cir::VisibilityKind::Protected: |
| return ::mlir::LLVM::Visibility::Protected; |
| } |
| } |
| |
| static mlir::Value |
| emitBoolVecConversion(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Value srcVec, unsigned numElementsDst) { |
| auto srcTy = mlir::cast<mlir::VectorType>(srcVec.getType()); |
| unsigned numElementsSrc = srcTy.getNumElements(); |
| if (numElementsSrc == numElementsDst) |
| return srcVec; |
| |
| SmallVector<int32_t, 8> mask(numElementsDst, -1); |
| for (unsigned i : llvm::seq(std::min(numElementsDst, numElementsSrc))) |
| mask[i] = i; |
| |
| mlir::Location loc = srcVec.getLoc(); |
| auto poison = mlir::LLVM::PoisonOp::create(rewriter, loc, srcVec.getType()); |
| return mlir::LLVM::ShuffleVectorOp::create(rewriter, loc, srcVec, poison, |
| mask); |
| } |
| |
| /// Emits the value from memory as expected by its users. Should be called when |
| /// the memory represetnation of a CIR type is not equal to its scalar |
| /// representation. |
| static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter, |
| const mlir::TypeConverter &converter, |
| mlir::DataLayout const &dataLayout, |
| cir::LoadOp op, mlir::Value value) { |
| // TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory |
| if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) { |
| // Create a cast value from specified size in datalayout to i1 |
| assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy))); |
| return createIntCast(rewriter, value, rewriter.getI1Type()); |
| } |
| |
| // Convert the `iN` back to boolean vectors |
| if (auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) { |
| if (mlir::isa<cir::BoolType>(vecTy.getElementType())) { |
| auto rawIntTy = mlir::cast<mlir::IntegerType>(value.getType()); |
| auto paddedVecTy = |
| cir::VectorType::get(vecTy.getElementType(), rawIntTy.getWidth()); |
| mlir::Type mlirVecTy = converter.convertType(paddedVecTy); |
| // Bitcast iP --> <P x i1>. |
| auto v = mlir::LLVM::BitcastOp::create(rewriter, value.getLoc(), |
| mlirVecTy, value); |
| // Shuffle <P x i1> --> <N x i1> (N is the actual bit size). |
| return emitBoolVecConversion(rewriter, v, vecTy.getSize()); |
| } |
| } |
| |
| // Truncate the padded storage integer back to the _BitInt's literal width. |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(op.getType()); |
| intTy && intTy.isBitInt()) |
| return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value, |
| /*toMemory=*/false); |
| |
| return value; |
| } |
| |
| /// Emits a value to memory with the expected scalar type. Should be called when |
| /// the memory represetnation of a CIR type is not equal to its scalar |
| /// representation. |
| static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter, |
| mlir::DataLayout const &dataLayout, |
| mlir::Type origType, mlir::Value value) { |
| |
| // TODO(cir): Handle other types similarly to clang's codegen EmitToMemory |
| if (auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) { |
| // Create zext of value from i1 to i8 |
| mlir::IntegerType memType = |
| rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy)); |
| return createIntCast(rewriter, value, memType); |
| } |
| |
| // Boolean vectors use `iN` as storage type |
| if (auto vecTy = mlir::dyn_cast<cir::VectorType>(origType)) { |
| if (mlir::isa<cir::BoolType>(vecTy.getElementType())) { |
| uint64_t bytePadded = std::max<uint64_t>(vecTy.getSize(), 8); |
| auto resultTy = mlir::IntegerType::get(origType.getContext(), bytePadded); |
| value = emitBoolVecConversion(rewriter, value, resultTy.getWidth()); |
| return mlir::LLVM::BitcastOp::create(rewriter, value.getLoc(), resultTy, |
| value); |
| } |
| } |
| |
| // Sign/zero-extend the _BitInt value to its padded storage integer. |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(origType); |
| intTy && intTy.isBitInt()) |
| return castBitIntMemoryStorage(rewriter, dataLayout, intTy, value, |
| /*toMemory=*/true); |
| |
| return value; |
| } |
| |
| mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) { |
| using CIR = cir::GlobalLinkageKind; |
| using LLVM = mlir::LLVM::Linkage; |
| |
| switch (linkage) { |
| case CIR::AppendingLinkage: |
| return LLVM::Appending; |
| case CIR::AvailableExternallyLinkage: |
| return LLVM::AvailableExternally; |
| case CIR::CommonLinkage: |
| return LLVM::Common; |
| case CIR::ExternalLinkage: |
| return LLVM::External; |
| case CIR::ExternalWeakLinkage: |
| return LLVM::ExternWeak; |
| case CIR::InternalLinkage: |
| return LLVM::Internal; |
| case CIR::LinkOnceAnyLinkage: |
| return LLVM::Linkonce; |
| case CIR::LinkOnceODRLinkage: |
| return LLVM::LinkonceODR; |
| case CIR::PrivateLinkage: |
| return LLVM::Private; |
| case CIR::WeakAnyLinkage: |
| return LLVM::Weak; |
| case CIR::WeakODRLinkage: |
| return LLVM::WeakODR; |
| }; |
| llvm_unreachable("Unknown CIR linkage type"); |
| } |
| |
| static mlir::LLVM::CConv convertCallingConv(cir::CallingConv callingConv) { |
| using CIR = cir::CallingConv; |
| using LLVM = mlir::LLVM::CConv; |
| |
| switch (callingConv) { |
| case CIR::C: |
| return LLVM::C; |
| case CIR::SpirKernel: |
| return LLVM::SPIR_KERNEL; |
| case CIR::SpirFunction: |
| return LLVM::SPIR_FUNC; |
| case CIR::PTXKernel: |
| return LLVM::PTX_Kernel; |
| case CIR::AMDGPUKernel: |
| return LLVM::AMDGPU_KERNEL; |
| } |
| llvm_unreachable("Unknown calling convention"); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite( |
| cir::CopyOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>()); |
| const mlir::Value length = mlir::LLVM::ConstantOp::create( |
| rewriter, op.getLoc(), rewriter.getI64Type(), |
| op.getCopySizeInBytes(layout)); |
| assert(!cir::MissingFeatures::aggValueSlotVolatile()); |
| |
| uint64_t dstTypeAlign = dataLayout.getTypeABIAlignment(convertTypeForMemory( |
| *getTypeConverter(), dataLayout, op.getDst().getType().getPointee())); |
| uint64_t srcTypeAlign = dataLayout.getTypeABIAlignment(convertTypeForMemory( |
| *getTypeConverter(), dataLayout, op.getSrc().getType().getPointee())); |
| |
| mlir::NamedAttribute dstAlignAttr = rewriter.getNamedAttr( |
| mlir::LLVM::LLVMDialect::getAlignAttrName(), |
| rewriter.getI64IntegerAttr(op.getDstAlignment().value_or(dstTypeAlign))); |
| mlir::NamedAttribute srcAlignAttr = rewriter.getNamedAttr( |
| mlir::LLVM::LLVMDialect::getAlignAttrName(), |
| rewriter.getI64IntegerAttr(op.getSrcAlignment().value_or(srcTypeAlign))); |
| mlir::ArrayAttr argAttrs = rewriter.getArrayAttr({ |
| /*dst_attrs=*/rewriter.getDictionaryAttr({dstAlignAttr}), |
| /*src_attrs=*/rewriter.getDictionaryAttr({srcAlignAttr}), |
| }); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>( |
| op, adaptor.getDst(), adaptor.getSrc(), length, op.getIsVolatile(), |
| /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, |
| /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr, /*arg_attrs=*/argAttrs, |
| /*res_attrs=*/nullptr); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMemCpyOpLowering::matchAndRewrite( |
| cir::MemCpyOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>( |
| op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(), |
| /*isVolatile=*/false); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMemMoveOpLowering::matchAndRewrite( |
| cir::MemMoveOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MemmoveOp>( |
| op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(), |
| /*isVolatile=*/false); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMemSetOpLowering::matchAndRewrite( |
| cir::MemSetOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| auto memset = rewriter.replaceOpWithNewOp<mlir::LLVM::MemsetOp>( |
| op, adaptor.getDst(), adaptor.getVal(), adaptor.getLen(), |
| /*isVolatile=*/false); |
| |
| if (op.getAlignmentAttr()) { |
| // Construct a list full of empty attributes. |
| llvm::SmallVector<mlir::Attribute> attrs{memset.getNumOperands(), |
| rewriter.getDictionaryAttr({})}; |
| llvm::SmallVector<mlir::NamedAttribute> destAttrs; |
| destAttrs.push_back( |
| {mlir::LLVM::LLVMDialect::getAlignAttrName(), op.getAlignmentAttr()}); |
| attrs[memset.odsIndex_dst] = rewriter.getDictionaryAttr(destAttrs); |
| |
| auto arrayAttr = rewriter.getArrayAttr(attrs); |
| memset.setArgAttrsAttr(arrayAttr); |
| } |
| |
| return mlir::success(); |
| } |
| |
| static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Value llvmSrc, mlir::Type llvmDstIntTy, |
| bool isUnsigned, uint64_t cirSrcWidth, |
| uint64_t cirDstIntWidth) { |
| if (cirSrcWidth == cirDstIntWidth) |
| return llvmSrc; |
| |
| auto loc = llvmSrc.getLoc(); |
| if (cirSrcWidth < cirDstIntWidth) { |
| if (isUnsigned) |
| return mlir::LLVM::ZExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); |
| return mlir::LLVM::SExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); |
| } |
| |
| // Otherwise truncate |
| return mlir::LLVM::TruncOp::create(rewriter, loc, llvmDstIntTy, llvmSrc); |
| } |
| |
| class CIRAttrToValue { |
| public: |
| CIRAttrToValue(mlir::Operation *parentOp, |
| mlir::ConversionPatternRewriter &rewriter, |
| mlir::SymbolTableCollection &symbolTables, |
| const mlir::TypeConverter *converter, |
| LLVMBlockAddressInfo *blockInfoAddr = nullptr) |
| : parentOp(parentOp), rewriter(rewriter), symbolTables(symbolTables), |
| converter(converter), blockInfoAddr(blockInfoAddr) {} |
| |
| #define GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS |
| #include "clang/CIR/Dialect/IR/CIRLowering.inc" |
| #undef GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS |
| |
| private: |
| mlir::Operation *parentOp; |
| mlir::ConversionPatternRewriter &rewriter; |
| mlir::SymbolTableCollection &symbolTables; |
| const mlir::TypeConverter *converter; |
| // Only available when lowering global initializers that may contain block |
| // address attributes. Used to resolve a BlockAddrInfoAttr to its block tag. |
| LLVMBlockAddressInfo *blockInfoAddr; |
| }; |
| |
| /// Switches on the type of attribute and calls the appropriate conversion. |
| mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, |
| const mlir::Attribute attr, |
| mlir::ConversionPatternRewriter &rewriter, |
| mlir::SymbolTableCollection &symbolTables, |
| const mlir::TypeConverter *converter, |
| LLVMBlockAddressInfo *blockInfoAddr) { |
| CIRAttrToValue valueConverter(parentOp, rewriter, symbolTables, converter, |
| blockInfoAddr); |
| mlir::Value value = valueConverter.visit(attr); |
| if (!value) |
| llvm_unreachable("unhandled attribute type"); |
| return value; |
| } |
| |
| void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow, |
| cir::SideEffect sideEffect, |
| mlir::LLVM::MemoryEffectsAttr &memoryEffect, |
| bool &noUnwind, bool &willReturn, |
| bool &noReturn) { |
| using mlir::LLVM::ModRefInfo; |
| |
| switch (sideEffect) { |
| case cir::SideEffect::All: |
| memoryEffect = {}; |
| noUnwind = isNothrow; |
| willReturn = false; |
| break; |
| |
| case cir::SideEffect::Pure: |
| memoryEffect = mlir::LLVM::MemoryEffectsAttr::get( |
| callOp->getContext(), /*other=*/ModRefInfo::Ref, |
| /*argMem=*/ModRefInfo::Ref, |
| /*inaccessibleMem=*/ModRefInfo::Ref, |
| /*errnoMem=*/ModRefInfo::Ref, |
| /*targetMem0=*/ModRefInfo::Ref, |
| /*targetMem1=*/ModRefInfo::Ref); |
| noUnwind = true; |
| willReturn = true; |
| break; |
| |
| case cir::SideEffect::Const: |
| memoryEffect = mlir::LLVM::MemoryEffectsAttr::get( |
| callOp->getContext(), /*other=*/ModRefInfo::NoModRef, |
| /*argMem=*/ModRefInfo::NoModRef, |
| /*inaccessibleMem=*/ModRefInfo::NoModRef, |
| /*errnoMem=*/ModRefInfo::NoModRef, |
| /*targetMem0=*/ModRefInfo::NoModRef, |
| /*targetMem1=*/ModRefInfo::NoModRef); |
| noUnwind = true; |
| willReturn = true; |
| break; |
| } |
| |
| noReturn = callOp->hasAttr(CIRDialect::getNoReturnAttrName()); |
| } |
| |
| static mlir::LLVM::CallIntrinsicOp |
| createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Location loc, const llvm::Twine &intrinsicName, |
| mlir::Type resultTy, mlir::ValueRange operands, |
| mlir::LLVM::FastmathFlags fastmathFlags = {}) { |
| auto intrinsicNameAttr = |
| mlir::StringAttr::get(rewriter.getContext(), intrinsicName); |
| auto fmfAttr = |
| mlir::LLVM::FastmathFlagsAttr::get(rewriter.getContext(), fastmathFlags); |
| // CallIntrinsicOp has distinct void / result create overloads. The FMF |
| // builders take a TypeRange for results. |
| if (resultTy) |
| return mlir::LLVM::CallIntrinsicOp::create( |
| rewriter, loc, mlir::TypeRange{resultTy}, intrinsicNameAttr, operands, |
| fmfAttr); |
| return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, intrinsicNameAttr, |
| operands, fmfAttr); |
| } |
| |
| static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp( |
| mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op, |
| const llvm::Twine &intrinsicName, mlir::Type resultTy, |
| mlir::ValueRange operands, mlir::LLVM::FastmathFlags fastmathFlags = {}) { |
| mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp( |
| rewriter, op->getLoc(), intrinsicName, resultTy, operands, fastmathFlags); |
| rewriter.replaceOp(op, callIntrinOp.getOperation()); |
| return callIntrinOp; |
| } |
| |
| static llvm::StringRef getConstrainedRoundingMetadata(cir::FenvAttr fenv) { |
| std::optional<cir::FPDynamicRoundingMode> rounding = |
| fenv.getDynamicRoundingMode(); |
| if (!rounding) |
| return "round.tonearest"; |
| switch (*rounding) { |
| case cir::FPDynamicRoundingMode::ToNearest: |
| return "round.tonearest"; |
| case cir::FPDynamicRoundingMode::Downward: |
| return "round.downward"; |
| case cir::FPDynamicRoundingMode::Upward: |
| return "round.upward"; |
| case cir::FPDynamicRoundingMode::UpwardZero: |
| return "round.towardzero"; |
| case cir::FPDynamicRoundingMode::ToNearestAway: |
| return "round.tonearestaway"; |
| case cir::FPDynamicRoundingMode::Unknown: |
| return "round.dynamic"; |
| } |
| llvm_unreachable("unknown FP dynamic rounding mode"); |
| } |
| |
| static llvm::StringRef getConstrainedExceptMetadata(cir::FenvAttr fenv) { |
| std::optional<cir::FPExceptionMode> exceptMode = fenv.getExceptMode(); |
| if (exceptMode == cir::FPExceptionMode::Masked) |
| return "fpexcept.ignore"; |
| mlir::BoolAttr strictExcept = fenv.getStrictExcept(); |
| if (!strictExcept) |
| return "fpexcept.ignore"; |
| return strictExcept.getValue() ? "fpexcept.strict" : "fpexcept.maytrap"; |
| } |
| |
| static mlir::Value |
| createFenvMetadataValue(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Location loc, llvm::StringRef str) { |
| auto mdString = mlir::LLVM::MDStringAttr::get( |
| rewriter.getContext(), mlir::StringAttr::get(rewriter.getContext(), str)); |
| return mlir::LLVM::MetadataAsValueOp::create(rewriter, loc, mdString); |
| } |
| |
| mlir::LogicalResult lowerToConstrainedFPIntrinsic( |
| mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv, |
| mlir::Type llvmResTy, mlir::ConversionPatternRewriter &rewriter, |
| llvm::StringRef constrainedMnemonic, bool hasRoundingMode, |
| mlir::LLVM::FastmathFlags fastmathFlags) { |
| mlir::Location loc = op->getLoc(); |
| llvm::SmallVector<mlir::Value> callOperands(operands.begin(), operands.end()); |
| if (hasRoundingMode) |
| callOperands.push_back(createFenvMetadataValue( |
| rewriter, loc, getConstrainedRoundingMetadata(fenv))); |
| callOperands.push_back(createFenvMetadataValue( |
| rewriter, loc, getConstrainedExceptMetadata(fenv))); |
| |
| replaceOpWithCallLLVMIntrinsicOp( |
| rewriter, op, "llvm.experimental.constrained." + constrainedMnemonic, |
| llvmResTy, callOperands, fastmathFlags); |
| return mlir::success(); |
| } |
| |
| template <typename LLVMOp> |
| mlir::LogicalResult lowerConstrainableFPOp( |
| mlir::Operation *op, mlir::ValueRange operands, cir::FenvAttr fenv, |
| const mlir::TypeConverter &typeConverter, |
| mlir::ConversionPatternRewriter &rewriter, |
| llvm::StringRef constrainedMnemonic, bool hasRoundingMode) { |
| mlir::Type llvmResTy = typeConverter.convertType(op->getResultTypes()[0]); |
| if (!llvmResTy) |
| return op->emitError("expected LLVM result type for floating-point op"); |
| |
| if (!fenv) { |
| rewriter.replaceOpWithNewOp<LLVMOp>(op, llvmResTy, operands); |
| return mlir::success(); |
| } |
| |
| return lowerToConstrainedFPIntrinsic(op, operands, fenv, llvmResTy, rewriter, |
| constrainedMnemonic, hasRoundingMode); |
| } |
| |
| mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite( |
| cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Result is Optional on the op, so void intrinsics have zero |
| // results; leave llvmResTy null in that case. |
| mlir::Type llvmResTy; |
| if (op->getNumResults() != 0) { |
| llvmResTy = getTypeConverter()->convertType(op->getResultTypes()[0]); |
| if (!llvmResTy) |
| return op.emitError("expected LLVM result type"); |
| } |
| StringRef name = op.getIntrinsicName(); |
| |
| // Some LLVM intrinsics require ElementType attribute to be attached to |
| // the argument of pointer type. That prevents us from generating LLVM IR |
| // because from LLVM dialect, we have LLVM IR like the below which fails |
| // LLVM IR verification. |
| // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2) |
| // The expected LLVM IR should be like |
| // %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2) |
| // TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way |
| // to set LLVM IR attribute. |
| assert(!cir::MissingFeatures::intrinsicElementTypeSupport()); |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy, |
| adaptor.getOperands()); |
| return mlir::success(); |
| } |
| |
| /// BoolAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::BoolAttr boolAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| mlir::Value boolVal = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, converter->convertType(boolAttr.getType()), |
| boolAttr.getValue()); |
| return emitToMemory(rewriter, layout, boolAttr.getType(), boolVal); |
| } |
| |
| /// IntAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| // Materialize the value at its literal width, then widen to the in-memory |
| // storage type (a no-op except for _BitInt) so aggregate members built here |
| // match the iM struct/array fields produced by convertTypeForMemory. |
| mlir::Value val = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, converter->convertType(intAttr.getType()), |
| intAttr.getValue()); |
| return emitToMemory(rewriter, layout, intAttr.getType(), val); |
| } |
| |
| /// FPAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| return mlir::LLVM::ConstantOp::create( |
| rewriter, loc, converter->convertType(fltAttr.getType()), |
| fltAttr.getValue()); |
| } |
| |
| /// ConstComplexAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) { |
| auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType()); |
| mlir::Type complexElemTy = complexType.getElementType(); |
| mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy); |
| |
| mlir::Attribute components[2]; |
| if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) { |
| components[0] = rewriter.getIntegerAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue()); |
| components[1] = rewriter.getIntegerAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue()); |
| } else { |
| components[0] = rewriter.getFloatAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue()); |
| components[1] = rewriter.getFloatAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue()); |
| } |
| |
| mlir::Location loc = parentOp->getLoc(); |
| return mlir::LLVM::ConstantOp::create( |
| rewriter, loc, converter->convertType(complexAttr.getType()), |
| rewriter.getArrayAttr(components)); |
| } |
| |
| /// ConstPtrAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| if (ptrAttr.isNullValue()) { |
| return mlir::LLVM::ZeroOp::create( |
| rewriter, loc, converter->convertType(ptrAttr.getType())); |
| } |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| mlir::Value ptrVal = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, |
| rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())), |
| ptrAttr.getValue().getInt()); |
| return mlir::LLVM::IntToPtrOp::create( |
| rewriter, loc, converter->convertType(ptrAttr.getType()), ptrVal); |
| } |
| |
| /// BlockAddrInfoAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::BlockAddrInfoAttr blockAddrInfo) { |
| assert(blockInfoAddr && |
| "block address lowering requires LLVMBlockAddressInfo"); |
| // A block address is lowered to an llvm.blockaddress op that references a |
| // block tag inside the target function. The matching block tag may not have |
| // been emitted yet, in which case the address is recorded as unresolved and |
| // patched up later in resolveBlockAddressOp. |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::LLVM::BlockTagOp matchLabel = |
| blockInfoAddr->lookupBlockTag(blockAddrInfo); |
| mlir::LLVM::BlockTagAttr tagAttr = |
| matchLabel ? matchLabel.getTag() : mlir::LLVM::BlockTagAttr{}; |
| auto blkAddr = mlir::LLVM::BlockAddressAttr::get( |
| rewriter.getContext(), blockAddrInfo.getFunc(), tagAttr); |
| auto blockAddressOp = mlir::LLVM::BlockAddressOp::create( |
| rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), |
| blkAddr); |
| if (!matchLabel) |
| blockInfoAddr->addUnresolvedBlockAddress(blockAddressOp, blockAddrInfo); |
| return blockAddressOp; |
| } |
| |
| /// BlockAddrDiffAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::BlockAddrDiffAttr blockAddrDiff) { |
| assert(blockInfoAddr && |
| "block address lowering requires LLVMBlockAddressInfo"); |
| // A block-address difference initializer is lowered to the difference of the |
| // two block addresses: trunc(ptrtoint(lhs) - ptrtoint(rhs)). Just like a |
| // single block address, each referenced block tag may not have been emitted |
| // yet, in which case it is recorded as unresolved and patched up later in |
| // resolveBlockAddressOp. |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| mlir::MLIRContext *ctx = rewriter.getContext(); |
| auto ptrTy = mlir::LLVM::LLVMPointerType::get(ctx); |
| |
| auto emitBlockAddr = [&](mlir::StringAttr label) -> mlir::Value { |
| auto info = cir::BlockAddrInfoAttr::get( |
| ctx, blockAddrDiff.getFunc().getValue(), label.getValue()); |
| mlir::LLVM::BlockTagOp matchLabel = blockInfoAddr->lookupBlockTag(info); |
| mlir::LLVM::BlockTagAttr tagAttr = |
| matchLabel ? matchLabel.getTag() : mlir::LLVM::BlockTagAttr{}; |
| auto blkAddr = mlir::LLVM::BlockAddressAttr::get( |
| ctx, blockAddrDiff.getFunc(), tagAttr); |
| auto addrOp = |
| mlir::LLVM::BlockAddressOp::create(rewriter, loc, ptrTy, blkAddr); |
| if (!matchLabel) |
| blockInfoAddr->addUnresolvedBlockAddress(addrOp, info); |
| return addrOp; |
| }; |
| |
| mlir::Value lhsAddr = emitBlockAddr(blockAddrDiff.getLhsLabel()); |
| mlir::Value rhsAddr = emitBlockAddr(blockAddrDiff.getRhsLabel()); |
| |
| // Compute the difference in a pointer-sized integer, then truncate to the |
| // initializer's type. LLVM is sensitive about the exact format of the |
| // address-of-label difference, so the truncation must happen after the |
| // subtraction. |
| mlir::Type intptrTy = |
| rewriter.getIntegerType(layout.getTypeSizeInBits(ptrTy)); |
| mlir::Value lhsInt = |
| mlir::LLVM::PtrToIntOp::create(rewriter, loc, intptrTy, lhsAddr); |
| mlir::Value rhsInt = |
| mlir::LLVM::PtrToIntOp::create(rewriter, loc, intptrTy, rhsAddr); |
| mlir::Value diffVal = |
| mlir::LLVM::SubOp::create(rewriter, loc, lhsInt, rhsInt); |
| |
| mlir::Type resultTy = converter->convertType(blockAddrDiff.getType()); |
| mlir::Value result = diffVal; |
| if (resultTy != intptrTy) |
| result = mlir::LLVM::TruncOp::create(rewriter, loc, resultTy, diffVal); |
| return result; |
| } |
| |
| // ConstArrayAttr visitor |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) { |
| mlir::Type llvmTy = converter->convertType(attr.getType()); |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::Value result; |
| |
| // When the array can be represented as a single dense constant, emit one |
| // llvm.mlir.constant instead of a chain of llvm.insertvalue ops. |
| if (std::optional<mlir::Attribute> denseAttr = |
| lowerConstArrayAttr(attr, symbolTables, converter)) |
| return mlir::LLVM::ConstantOp::create(rewriter, loc, llvmTy, *denseAttr); |
| |
| if (attr.hasTrailingZeros()) { |
| mlir::Type arrayTy = attr.getType(); |
| result = mlir::LLVM::ZeroOp::create(rewriter, loc, |
| converter->convertType(arrayTy)); |
| } else { |
| result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); |
| } |
| |
| // Iteratively lower each constant element of the array. |
| if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) { |
| for (auto [idx, elt] : llvm::enumerate(arrayAttr)) { |
| mlir::Value init = visit(elt); |
| result = |
| mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); |
| } |
| } else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) { |
| // TODO(cir): this diverges from traditional lowering. Normally the string |
| // would be a global constant that is memcopied. |
| auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType()); |
| assert(arrayTy && "String attribute must have an array type"); |
| mlir::Type eltTy = arrayTy.getElementType(); |
| for (auto [idx, elt] : llvm::enumerate(strAttr)) { |
| auto init = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, converter->convertType(eltTy), elt); |
| result = |
| mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); |
| } |
| } else { |
| llvm_unreachable("unexpected ConstArrayAttr elements"); |
| } |
| |
| return result; |
| } |
| |
| /// ConstRecord visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) { |
| mlir::Type llvmTy = converter->convertType(constRecord.getType()); |
| mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| llvmTy = adjustGlobalTypeForInit(llvmTy, constRecord, *converter, dataLayout); |
| const mlir::Location loc = parentOp->getLoc(); |
| mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); |
| |
| // Iteratively lower each constant element of the record. |
| for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) { |
| mlir::Value init = visit(elt); |
| result = |
| mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); |
| } |
| |
| return result; |
| } |
| |
| /// ConstVectorAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) { |
| const mlir::Type llvmTy = converter->convertType(attr.getType()); |
| const mlir::Location loc = parentOp->getLoc(); |
| |
| SmallVector<mlir::Attribute> mlirValues; |
| for (const mlir::Attribute elementAttr : attr.getElts()) { |
| mlir::Attribute mlirAttr; |
| if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) { |
| mlirAttr = rewriter.getIntegerAttr( |
| converter->convertType(intAttr.getType()), intAttr.getValue()); |
| } else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) { |
| mlirAttr = rewriter.getFloatAttr( |
| converter->convertType(floatAttr.getType()), floatAttr.getValue()); |
| } else { |
| llvm_unreachable( |
| "vector constant with an element that is neither an int nor a float"); |
| } |
| mlirValues.push_back(mlirAttr); |
| } |
| |
| return mlir::LLVM::ConstantOp::create( |
| rewriter, loc, llvmTy, |
| mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy), |
| mlirValues)); |
| } |
| |
| // GlobalViewAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) { |
| auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>(); |
| mlir::DataLayout dataLayout(moduleOp); |
| mlir::Type sourceType; |
| unsigned sourceAddrSpace = 0; |
| llvm::StringRef symName; |
| mlir::Operation *sourceSymbol = |
| symbolTables.lookupSymbolIn(moduleOp, globalAttr.getSymbol()); |
| if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) { |
| sourceType = llvmSymbol.getType(); |
| symName = llvmSymbol.getSymName(); |
| sourceAddrSpace = llvmSymbol.getAddrSpace(); |
| } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) { |
| sourceType = |
| convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType()); |
| symName = cirSymbol.getSymName(); |
| if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| cirSymbol.getAddrSpaceAttr())) |
| sourceAddrSpace = targetAS.getValue(); |
| } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) { |
| sourceType = llvmFun.getFunctionType(); |
| symName = llvmFun.getSymName(); |
| } else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) { |
| sourceType = converter->convertType(fun.getFunctionType()); |
| symName = fun.getSymName(); |
| } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) { |
| sourceType = alias.getType(); |
| symName = alias.getSymName(); |
| } else { |
| llvm_unreachable("Unexpected GlobalOp type"); |
| } |
| |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::Value addrOp = mlir::LLVM::AddressOfOp::create( |
| rewriter, loc, |
| mlir::LLVM::LLVMPointerType::get(rewriter.getContext(), sourceAddrSpace), |
| symName); |
| |
| if (globalAttr.getIndices()) { |
| llvm::SmallVector<mlir::LLVM::GEPArg> indices; |
| |
| if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>( |
| sourceType)) |
| indices.push_back(0); |
| |
| for (mlir::Attribute idx : globalAttr.getIndices()) { |
| auto intAttr = mlir::cast<mlir::IntegerAttr>(idx); |
| indices.push_back(intAttr.getValue().getSExtValue()); |
| } |
| mlir::Type resTy = addrOp.getType(); |
| mlir::Type eltTy = converter->convertType(sourceType); |
| addrOp = |
| mlir::LLVM::GEPOp::create(rewriter, loc, resTy, eltTy, addrOp, indices, |
| mlir::LLVM::GEPNoWrapFlags::none); |
| } |
| |
| // We can have a global view with an integer type in the case of method |
| // pointers. With the Itanium ABI, the #cir.method attribute is lowered to a |
| // #cir.global_view with a pointer-sized integer representing the address of |
| // the method. |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(globalAttr.getType())) { |
| mlir::Type llvmDstTy = converter->convertType(globalAttr.getType()); |
| return mlir::LLVM::PtrToIntOp::create(rewriter, parentOp->getLoc(), |
| llvmDstTy, addrOp); |
| } |
| |
| if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) { |
| auto llvmDstTy = converter->convertType<mlir::LLVM::LLVMPointerType>(ptrTy); |
| unsigned dstAddrSpace = llvmDstTy.getAddressSpace(); |
| |
| if (sourceAddrSpace != dstAddrSpace) |
| addrOp = mlir::LLVM::AddrSpaceCastOp::create(rewriter, parentOp->getLoc(), |
| llvmDstTy, addrOp); |
| |
| mlir::Type llvmEltTy = |
| convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee()); |
| |
| // No further cast needed if the pointee type already matches. |
| if (llvmEltTy == sourceType) |
| return addrOp; |
| |
| // With opaque pointers, the pointer type is already correct (either from |
| // the original AddressOfOp or after an addrspacecast) — skip the |
| // redundant bitcast. |
| if (addrOp.getType() == llvmDstTy) |
| return addrOp; |
| |
| return mlir::LLVM::BitcastOp::create(rewriter, parentOp->getLoc(), |
| llvmDstTy, addrOp); |
| } |
| |
| if (mlir::isa<cir::VPtrType>(globalAttr.getType())) |
| return addrOp; |
| |
| llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr"); |
| } |
| |
| // TypeInfoAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) { |
| mlir::Type llvmTy = converter->convertType(typeInfoAttr.getType()); |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); |
| |
| for (auto [idx, elt] : llvm::enumerate(typeInfoAttr.getData())) { |
| mlir::Value init = visit(elt); |
| result = |
| mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); |
| } |
| |
| return result; |
| } |
| |
| /// UndefAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| return mlir::LLVM::UndefOp::create( |
| rewriter, loc, |
| convertTypeForMemory(*converter, layout, undefAttr.getType())); |
| } |
| |
| /// PoisonAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::PoisonAttr poisonAttr) { |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| return mlir::LLVM::PoisonOp::create( |
| rewriter, loc, |
| convertTypeForMemory(*converter, layout, poisonAttr.getType())); |
| } |
| |
| // VTableAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) { |
| mlir::Type llvmTy = converter->convertType(vtableArr.getType()); |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy); |
| |
| for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) { |
| mlir::Value init = visit(elt); |
| result = |
| mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx); |
| } |
| |
| return result; |
| } |
| |
| /// ZeroAttr visitor. |
| mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) { |
| mlir::Location loc = parentOp->getLoc(); |
| mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>()); |
| return mlir::LLVM::ZeroOp::create( |
| rewriter, loc, convertTypeForMemory(*converter, layout, attr.getType())); |
| } |
| |
| // This class handles rewriting initializer attributes for types that do not |
| // require region initialization. |
| class GlobalInitAttrRewriter { |
| public: |
| GlobalInitAttrRewriter(mlir::Type type, |
| mlir::ConversionPatternRewriter &rewriter) |
| : llvmType(type), rewriter(rewriter) {} |
| |
| mlir::Attribute visit(mlir::Attribute attr) { |
| return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr) |
| .Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>( |
| [&](auto attrT) { return visitCirAttr(attrT); }) |
| .Default([&](auto attrT) { return mlir::Attribute(); }); |
| } |
| |
| mlir::Attribute visitCirAttr(cir::IntAttr attr) { |
| // A _BitInt(N) global stores its value in a padded integer iM; sign/zero- |
| // extend the APInt to that width (a no-op for plain integers, whose value |
| // width already matches llvmType) so the IntegerAttr is well-typed. |
| llvm::APInt val = attr.getValue(); |
| auto destTy = mlir::cast<mlir::IntegerType>(llvmType); |
| if (val.getBitWidth() != destTy.getWidth()) { |
| auto cirIntTy = mlir::cast<cir::IntType>(attr.getType()); |
| val = cirIntTy.isSigned() ? val.sext(destTy.getWidth()) |
| : val.zext(destTy.getWidth()); |
| } |
| return rewriter.getIntegerAttr(llvmType, val); |
| } |
| |
| mlir::Attribute visitCirAttr(cir::FPAttr attr) { |
| return rewriter.getFloatAttr(llvmType, attr.getValue()); |
| } |
| |
| mlir::Attribute visitCirAttr(cir::BoolAttr attr) { |
| return rewriter.getBoolAttr(attr.getValue()); |
| } |
| |
| private: |
| mlir::Type llvmType; |
| mlir::ConversionPatternRewriter &rewriter; |
| }; |
| |
| // This pass requires the CIR to be in a "flat" state. All blocks in each |
| // function must belong to the parent region. Once scopes and control flow |
| // are implemented in CIR, a pass will be run before this one to flatten |
| // the CIR and get it into the state that this pass requires. |
| struct ConvertCIRToLLVMPass |
| : public mlir::PassWrapper<ConvertCIRToLLVMPass, |
| mlir::OperationPass<mlir::ModuleOp>> { |
| void getDependentDialects(mlir::DialectRegistry ®istry) const override { |
| registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect, |
| mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>(); |
| } |
| void runOnOperation() final; |
| |
| void processCIRAttrs(mlir::ModuleOp module); |
| |
| void resolveBlockAddressOp(LLVMBlockAddressInfo &blockInfoAddr); |
| |
| /// Collect (symbol_name, annotations, loc) from cir.func and cir.global ops |
| /// before the conversion runs (the annotations attribute is dropped during |
| /// FuncOp/GlobalOp lowering). |
| void collectGlobalAnnotations(mlir::ModuleOp module); |
| |
| /// Emit @llvm.global.annotations and supporting string/args constants from |
| /// the previously-collected annotations. Mirrors what OGCG produces. |
| void buildGlobalAnnotationsVar(mlir::ModuleOp module); |
| |
| StringRef getDescription() const override { |
| return "Convert the prepared CIR dialect module to LLVM dialect"; |
| } |
| |
| StringRef getArgument() const override { return "cir-flat-to-llvm"; } |
| |
| private: |
| /// One annotation entry collected pre-conversion. |
| struct CollectedAnnotation { |
| mlir::StringAttr symName; |
| cir::AnnotationAttr annotation; |
| mlir::Location loc; |
| CollectedAnnotation(mlir::StringAttr symName, |
| cir::AnnotationAttr annotation, mlir::Location loc) |
| : symName(symName), annotation(annotation), loc(loc) {} |
| }; |
| llvm::SmallVector<CollectedAnnotation> collectedAnnotations; |
| }; |
| |
| mlir::LogicalResult CIRToLLVMIsFPClassOpLowering::matchAndRewrite( |
| cir::IsFPClassOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value src = adaptor.getSrc(); |
| cir::FPClassTest flags = adaptor.getFlags(); |
| mlir::IntegerType retTy = rewriter.getI1Type(); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::IsFPClass>( |
| op, retTy, src, static_cast<uint32_t>(flags)); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSignBitOpLowering::matchAndRewrite( |
| cir::SignBitOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::isPPC_FP128Ty()); |
| |
| mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>()); |
| int width = layout.getTypeSizeInBits(op.getInput().getType()); |
| if (auto longDoubleType = |
| mlir::dyn_cast<cir::LongDoubleType>(op.getInput().getType())) { |
| if (mlir::isa<cir::FP80Type>(longDoubleType.getUnderlying())) { |
| // If the underlying type of LongDouble is FP80Type, |
| // DataLayout::getTypeSizeInBits returns 128. |
| // See https://github.com/llvm/clangir/issues/1057. |
| // Set the width to 80 manually. |
| width = 80; |
| } |
| } |
| mlir::Type intTy = mlir::IntegerType::get(rewriter.getContext(), width); |
| auto bitcast = mlir::LLVM::BitcastOp::create(rewriter, op->getLoc(), intTy, |
| adaptor.getInput()); |
| |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, op->getLoc(), intTy, 0); |
| auto cmpResult = mlir::LLVM::ICmpOp::create(rewriter, op.getLoc(), |
| mlir::LLVM::ICmpPredicate::slt, |
| bitcast.getResult(), zero); |
| rewriter.replaceOp(op, cmpResult); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite( |
| cir::AssumeOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value cond = adaptor.getPredicate(); |
| if (op.getBundleKind() == cir::AssumeBundleKind::None) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond); |
| return mlir::success(); |
| } |
| |
| llvm::StringRef tag = cir::stringifyAssumeBundleKind(op.getBundleKind()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond, tag, |
| adaptor.getBundleArgs()); |
| return mlir::success(); |
| } |
| |
| static mlir::LLVM::AtomicOrdering |
| getLLVMMemOrder(std::optional<cir::MemOrder> memorder) { |
| if (!memorder) |
| return mlir::LLVM::AtomicOrdering::not_atomic; |
| switch (*memorder) { |
| case cir::MemOrder::Relaxed: |
| return mlir::LLVM::AtomicOrdering::monotonic; |
| case cir::MemOrder::Consume: |
| case cir::MemOrder::Acquire: |
| return mlir::LLVM::AtomicOrdering::acquire; |
| case cir::MemOrder::Release: |
| return mlir::LLVM::AtomicOrdering::release; |
| case cir::MemOrder::AcquireRelease: |
| return mlir::LLVM::AtomicOrdering::acq_rel; |
| case cir::MemOrder::SequentiallyConsistent: |
| return mlir::LLVM::AtomicOrdering::seq_cst; |
| } |
| llvm_unreachable("unknown memory order"); |
| } |
| |
| static llvm::StringRef getLLVMSyncScope(cir::SyncScopeKind syncScope) { |
| switch (syncScope) { |
| case cir::SyncScopeKind::SingleThread: |
| return "singlethread"; |
| case cir::SyncScopeKind::Workgroup: |
| return "block"; |
| default: |
| return ""; |
| } |
| } |
| |
| static std::optional<llvm::StringRef> |
| getLLVMSyncScope(std::optional<cir::SyncScopeKind> syncScope) { |
| if (syncScope.has_value()) |
| return getLLVMSyncScope(*syncScope); |
| return std::nullopt; |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicCmpXchgOpLowering::matchAndRewrite( |
| cir::AtomicCmpXchgOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value expected = adaptor.getExpected(); |
| mlir::Value desired = adaptor.getDesired(); |
| |
| auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create( |
| rewriter, op.getLoc(), adaptor.getPtr(), expected, desired, |
| getLLVMMemOrder(adaptor.getSuccOrder()), |
| getLLVMMemOrder(adaptor.getFailOrder()), |
| getLLVMSyncScope(op.getSyncScope())); |
| |
| cmpxchg.setAlignment(adaptor.getAlignment()); |
| cmpxchg.setWeak(adaptor.getWeak()); |
| cmpxchg.setVolatile_(adaptor.getIsVolatile()); |
| |
| // Check result and apply stores accordingly. |
| auto old = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(), |
| cmpxchg.getResult(), 0); |
| auto cmp = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(), |
| cmpxchg.getResult(), 1); |
| |
| rewriter.replaceOp(op, {old, cmp}); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite( |
| cir::AtomicXchgOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::atomicSyncScopeID()); |
| mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder()); |
| llvm::StringRef llvmSyncScope = getLLVMSyncScope(adaptor.getSyncScope()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>( |
| op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(), |
| llvmOrder, llvmSyncScope); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite( |
| cir::AtomicTestAndSetOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::atomicSyncScopeID()); |
| |
| mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder()); |
| |
| auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| rewriter.getI8Type(), 1); |
| auto rmw = mlir::LLVM::AtomicRMWOp::create( |
| rewriter, op.getLoc(), mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), |
| one, llvmOrder, /*syncscope=*/llvm::StringRef(), |
| adaptor.getAlignment().value_or(0), op.getIsVolatile()); |
| |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| rewriter.getI8Type(), 0); |
| auto cmp = mlir::LLVM::ICmpOp::create( |
| rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, rmw, zero); |
| |
| rewriter.replaceOp(op, cmp); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite( |
| cir::AtomicClearOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::atomicSyncScopeID()); |
| |
| mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder()); |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| rewriter.getI8Type(), 0); |
| auto store = mlir::LLVM::StoreOp::create( |
| rewriter, op.getLoc(), zero, adaptor.getPtr(), |
| adaptor.getAlignment().value_or(0), op.getIsVolatile(), |
| /*isNonTemporal=*/false, /*isInvariantGroup=*/false, llvmOrder); |
| |
| rewriter.replaceOp(op, store); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicFenceOpLowering::matchAndRewrite( |
| cir::AtomicFenceOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getOrdering()); |
| |
| auto fence = mlir::LLVM::FenceOp::create(rewriter, op.getLoc(), llvmOrder); |
| fence.setSyncscope(getLLVMSyncScope(adaptor.getSyncscope())); |
| |
| rewriter.replaceOp(op, fence); |
| |
| return mlir::success(); |
| } |
| |
| static mlir::LLVM::AtomicBinOp |
| getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt) { |
| switch (k) { |
| case cir::AtomicFetchKind::Add: |
| return isInt ? mlir::LLVM::AtomicBinOp::add : mlir::LLVM::AtomicBinOp::fadd; |
| case cir::AtomicFetchKind::Sub: |
| return isInt ? mlir::LLVM::AtomicBinOp::sub : mlir::LLVM::AtomicBinOp::fsub; |
| case cir::AtomicFetchKind::And: |
| return mlir::LLVM::AtomicBinOp::_and; |
| case cir::AtomicFetchKind::Xor: |
| return mlir::LLVM::AtomicBinOp::_xor; |
| case cir::AtomicFetchKind::Or: |
| return mlir::LLVM::AtomicBinOp::_or; |
| case cir::AtomicFetchKind::Nand: |
| return mlir::LLVM::AtomicBinOp::nand; |
| case cir::AtomicFetchKind::Max: { |
| if (!isInt) |
| return mlir::LLVM::AtomicBinOp::fmax; |
| return isSignedInt ? mlir::LLVM::AtomicBinOp::max |
| : mlir::LLVM::AtomicBinOp::umax; |
| } |
| case cir::AtomicFetchKind::Min: { |
| if (!isInt) |
| return mlir::LLVM::AtomicBinOp::fmin; |
| return isSignedInt ? mlir::LLVM::AtomicBinOp::min |
| : mlir::LLVM::AtomicBinOp::umin; |
| } |
| case cir::AtomicFetchKind::UIncWrap: |
| return mlir::LLVM::AtomicBinOp::uinc_wrap; |
| case cir::AtomicFetchKind::UDecWrap: |
| return mlir::LLVM::AtomicBinOp::udec_wrap; |
| case cir::AtomicFetchKind::Maximum: |
| return mlir::LLVM::AtomicBinOp::fmaximum; |
| case cir::AtomicFetchKind::Minimum: |
| return mlir::LLVM::AtomicBinOp::fminimum; |
| case cir::AtomicFetchKind::MaximumNum: |
| return mlir::LLVM::AtomicBinOp::fmaximumnum; |
| case cir::AtomicFetchKind::MinimumNum: |
| return mlir::LLVM::AtomicBinOp::fminimumnum; |
| } |
| llvm_unreachable("Unknown atomic fetch opcode"); |
| } |
| |
| static llvm::StringLiteral getLLVMBinopForPostAtomic(cir::AtomicFetchKind k, |
| bool isInt) { |
| switch (k) { |
| case cir::AtomicFetchKind::Add: |
| return isInt ? mlir::LLVM::AddOp::getOperationName() |
| : mlir::LLVM::FAddOp::getOperationName(); |
| case cir::AtomicFetchKind::Sub: |
| return isInt ? mlir::LLVM::SubOp::getOperationName() |
| : mlir::LLVM::FSubOp::getOperationName(); |
| case cir::AtomicFetchKind::And: |
| return mlir::LLVM::AndOp::getOperationName(); |
| case cir::AtomicFetchKind::Xor: |
| return mlir::LLVM::XOrOp::getOperationName(); |
| case cir::AtomicFetchKind::Or: |
| return mlir::LLVM::OrOp::getOperationName(); |
| case cir::AtomicFetchKind::Nand: |
| // There's no nand binop in LLVM, this is later fixed with a not. |
| return mlir::LLVM::AndOp::getOperationName(); |
| case cir::AtomicFetchKind::Max: |
| case cir::AtomicFetchKind::Min: |
| llvm_unreachable("handled in buildMinMaxPostOp"); |
| case cir::AtomicFetchKind::UIncWrap: |
| case cir::AtomicFetchKind::UDecWrap: |
| case cir::AtomicFetchKind::Maximum: |
| case cir::AtomicFetchKind::Minimum: |
| case cir::AtomicFetchKind::MaximumNum: |
| case cir::AtomicFetchKind::MinimumNum: |
| llvm_unreachable("uinc_wrap, udec_wrap, maximum, minimum, maximum_num, and " |
| "minimum_num are always fetch_first"); |
| } |
| llvm_unreachable("Unknown atomic fetch opcode"); |
| } |
| |
| mlir::Value CIRToLLVMAtomicFetchOpLowering::buildPostOp( |
| cir::AtomicFetchOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, |
| bool isInt) const { |
| SmallVector<mlir::Value> atomicOperands = {rmwVal, adaptor.getVal()}; |
| SmallVector<mlir::Type> atomicResTys = {rmwVal.getType()}; |
| return rewriter |
| .create(op.getLoc(), |
| rewriter.getStringAttr( |
| getLLVMBinopForPostAtomic(op.getBinop(), isInt)), |
| atomicOperands, atomicResTys, {}) |
| ->getResult(0); |
| } |
| |
| mlir::Value CIRToLLVMAtomicFetchOpLowering::buildMinMaxPostOp( |
| cir::AtomicFetchOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, bool isInt, |
| bool isSigned) const { |
| mlir::Location loc = op.getLoc(); |
| |
| if (!isInt) { |
| if (op.getBinop() == cir::AtomicFetchKind::Max) |
| return mlir::LLVM::MaxNumOp::create(rewriter, loc, rmwVal, |
| adaptor.getVal()); |
| return mlir::LLVM::MinNumOp::create(rewriter, loc, rmwVal, |
| adaptor.getVal()); |
| } |
| |
| mlir::LLVM::ICmpPredicate pred; |
| if (op.getBinop() == cir::AtomicFetchKind::Max) { |
| pred = isSigned ? mlir::LLVM::ICmpPredicate::sgt |
| : mlir::LLVM::ICmpPredicate::ugt; |
| } else { // Min |
| pred = isSigned ? mlir::LLVM::ICmpPredicate::slt |
| : mlir::LLVM::ICmpPredicate::ult; |
| } |
| mlir::Value cmp = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, |
| mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), pred), rmwVal, |
| adaptor.getVal()); |
| return mlir::LLVM::SelectOp::create(rewriter, loc, cmp, rmwVal, |
| adaptor.getVal()); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite( |
| cir::AtomicFetchOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| bool isInt = false; |
| bool isSignedInt = false; |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(op.getVal().getType())) { |
| isInt = true; |
| isSignedInt = intTy.isSigned(); |
| } else if (mlir::isa<cir::SingleType, cir::DoubleType>( |
| op.getVal().getType())) { |
| isInt = false; |
| } else { |
| return op.emitError() << "Unsupported type: " << op.getVal().getType(); |
| } |
| |
| mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder()); |
| llvm::StringRef llvmSyncScope = getLLVMSyncScope(op.getSyncScope()); |
| mlir::LLVM::AtomicBinOp llvmBinOp = |
| getLLVMAtomicBinOp(op.getBinop(), isInt, isSignedInt); |
| auto rmwVal = mlir::LLVM::AtomicRMWOp::create( |
| rewriter, op.getLoc(), llvmBinOp, adaptor.getPtr(), adaptor.getVal(), |
| llvmOrder, llvmSyncScope); |
| |
| mlir::Value result = rmwVal.getResult(); |
| if (!op.getFetchFirst()) { |
| if (op.getBinop() == cir::AtomicFetchKind::Max || |
| op.getBinop() == cir::AtomicFetchKind::Min) |
| result = buildMinMaxPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt, |
| isSignedInt); |
| else |
| result = buildPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt); |
| |
| // Compensate lack of nand binop in LLVM IR. |
| if (op.getBinop() == cir::AtomicFetchKind::Nand) { |
| auto negOne = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| result.getType(), -1); |
| result = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), result, negOne); |
| } |
| } |
| |
| rewriter.replaceOp(op, result); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite( |
| cir::BitClrsbOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| adaptor.getInput().getType(), 0); |
| auto isNeg = mlir::LLVM::ICmpOp::create( |
| rewriter, op.getLoc(), |
| mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), |
| mlir::LLVM::ICmpPredicate::slt), |
| adaptor.getInput(), zero); |
| |
| auto negOne = mlir::LLVM::ConstantOp::create( |
| rewriter, op.getLoc(), adaptor.getInput().getType(), -1); |
| auto flipped = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), |
| adaptor.getInput(), negOne); |
| |
| auto select = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isNeg, |
| flipped, adaptor.getInput()); |
| |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto clz = mlir::LLVM::CountLeadingZerosOp::create( |
| rewriter, op.getLoc(), resTy, select, /*is_zero_poison=*/false); |
| |
| auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); |
| auto res = mlir::LLVM::SubOp::create(rewriter, op.getLoc(), clz, one, |
| mlir::LLVM::IntegerOverflowFlags::nuw); |
| rewriter.replaceOp(op, res); |
| |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite( |
| cir::BitClzOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto llvmOp = mlir::LLVM::CountLeadingZerosOp::create( |
| rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); |
| rewriter.replaceOp(op, llvmOp); |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite( |
| cir::BitCtzOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto llvmOp = mlir::LLVM::CountTrailingZerosOp::create( |
| rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero()); |
| rewriter.replaceOp(op, llvmOp); |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite( |
| cir::BitFfsOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto ctz = mlir::LLVM::CountTrailingZerosOp::create(rewriter, op.getLoc(), |
| resTy, adaptor.getInput(), |
| /*is_zero_poison=*/true); |
| |
| auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); |
| auto ctzAddOne = mlir::LLVM::AddOp::create(rewriter, op.getLoc(), ctz, one); |
| |
| auto zeroInputTy = mlir::LLVM::ConstantOp::create( |
| rewriter, op.getLoc(), adaptor.getInput().getType(), 0); |
| auto isZero = mlir::LLVM::ICmpOp::create( |
| rewriter, op.getLoc(), |
| mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), |
| mlir::LLVM::ICmpPredicate::eq), |
| adaptor.getInput(), zeroInputTy); |
| |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 0); |
| auto res = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isZero, zero, |
| ctzAddOne); |
| rewriter.replaceOp(op, res); |
| |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite( |
| cir::BitParityOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto popcnt = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy, |
| adaptor.getInput()); |
| |
| auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1); |
| auto popcntMod2 = |
| mlir::LLVM::AndOp::create(rewriter, op.getLoc(), popcnt, one); |
| rewriter.replaceOp(op, popcntMod2); |
| |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite( |
| cir::BitPopcountOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto resTy = getTypeConverter()->convertType(op.getType()); |
| auto llvmOp = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy, |
| adaptor.getInput()); |
| rewriter.replaceOp(op, llvmOp); |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite( |
| cir::BrCondOp brOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // When ZExtOp is implemented, we'll need to check if the condition is a |
| // ZExtOp and if so, delete it if it has a single use. |
| assert(!cir::MissingFeatures::zextOp()); |
| |
| mlir::Value i1Condition = adaptor.getCond(); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>( |
| brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(), |
| brOp.getDestFalse(), adaptor.getDestOperandsFalse()); |
| |
| return mlir::success(); |
| } |
| |
| mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const { |
| return getTypeConverter()->convertType(ty); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite( |
| cir::CastOp castOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // For arithmetic conversions, LLVM IR uses the same instruction to convert |
| // both individual scalars and entire vectors. This lowering pass handles |
| // both situations. |
| |
| switch (castOp.getKind()) { |
| case cir::CastKind::array_to_ptrdecay: { |
| const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType()); |
| mlir::Value sourceValue = adaptor.getSrc(); |
| mlir::Type targetType = convertTy(ptrTy); |
| mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout, |
| ptrTy.getPointee()); |
| llvm::SmallVector<mlir::LLVM::GEPArg> offset{0}; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| castOp, targetType, elementTy, sourceValue, offset); |
| break; |
| } |
| case cir::CastKind::int_to_bool: { |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| // getZeroAttr yields a splat for vector source types so this also |
| // handles element-wise int-to-bool conversions (e.g. an ext_vector |
| // __builtin_convertvector to bool). |
| mlir::Value zeroInt = mlir::LLVM::ConstantOp::create( |
| rewriter, castOp.getLoc(), llvmSrcVal.getType(), |
| rewriter.getZeroAttr(llvmSrcVal.getType())); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( |
| castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt); |
| break; |
| } |
| case cir::CastKind::integral: { |
| mlir::Type srcType = castOp.getSrc().getType(); |
| mlir::Type dstType = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstType = getTypeConverter()->convertType(dstType); |
| cir::IntType srcIntType = |
| mlir::cast<cir::IntType>(elementTypeIfVector(srcType)); |
| cir::IntType dstIntType = |
| mlir::cast<cir::IntType>(elementTypeIfVector(dstType)); |
| rewriter.replaceOp(castOp, getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType, |
| srcIntType.isUnsigned(), |
| srcIntType.getWidth(), |
| dstIntType.getWidth())); |
| break; |
| } |
| case cir::CastKind::floating: { |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType()); |
| |
| mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType()); |
| mlir::Type dstTy = elementTypeIfVector(castOp.getType()); |
| |
| if (!mlir::isa<cir::FPTypeInterface>(dstTy) || |
| !mlir::isa<cir::FPTypeInterface>(srcTy)) |
| return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy; |
| |
| auto getFloatWidth = [](mlir::Type ty) -> unsigned { |
| return mlir::cast<cir::FPTypeInterface>(ty).getWidth(); |
| }; |
| |
| bool isTrunc = getFloatWidth(srcTy) > getFloatWidth(dstTy); |
| if (cir::FenvAttr fenv = castOp.getFenvAttr()) { |
| // fptrunc takes rounding mode + exception behavior; fpext takes only |
| // exception behavior. |
| return lowerToConstrainedFPIntrinsic( |
| castOp, llvmSrcVal, fenv, llvmDstTy, rewriter, |
| isTrunc ? "fptrunc" : "fpext", /*hasRoundingMode=*/isTrunc); |
| } |
| if (isTrunc) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::int_to_ptr: { |
| auto dstTy = mlir::cast<cir::PointerType>(castOp.getType()); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::ptr_to_int: { |
| auto dstTy = mlir::cast<cir::IntType>(castOp.getType()); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::float_to_bool: { |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| auto kind = mlir::LLVM::FCmpPredicate::une; |
| |
| // Check if float is not equal to zero. getZeroAttr yields a splat |
| // for vector source types so this also handles element-wise |
| // float-to-bool conversions. |
| auto zeroFloat = mlir::LLVM::ConstantOp::create( |
| rewriter, castOp.getLoc(), llvmSrcVal.getType(), |
| rewriter.getZeroAttr(llvmSrcVal.getType())); |
| |
| // Extend comparison result to either bool (C++) or int (C). |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp, kind, llvmSrcVal, |
| zeroFloat); |
| |
| return mlir::success(); |
| } |
| case cir::CastKind::bool_to_int: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| // Compare element widths so this also handles vector bool -> int casts. |
| auto srcElemTy = mlir::cast<mlir::IntegerType>( |
| elementTypeIfVector(llvmSrcVal.getType())); |
| auto dstElemTy = mlir::cast<cir::IntType>(elementTypeIfVector(dstTy)); |
| |
| if (srcElemTy.getWidth() == dstElemTy.getWidth()) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::bool_to_float: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::int_to_float: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| bool isSigned = |
| mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType())) |
| .isSigned(); |
| if (cir::FenvAttr fenv = castOp.getFenvAttr()) { |
| return lowerToConstrainedFPIntrinsic( |
| castOp, llvmSrcVal, fenv, llvmDstTy, rewriter, |
| isSigned ? "sitofp" : "uitofp", /*hasRoundingMode=*/true); |
| } |
| if (isSigned) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::float_to_int: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| bool isSigned = |
| mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType())) |
| .isSigned(); |
| if (cir::FenvAttr fenv = castOp.getFenvAttr()) { |
| return lowerToConstrainedFPIntrinsic( |
| castOp, llvmSrcVal, fenv, llvmDstTy, rewriter, |
| isSigned ? "fptosi" : "fptoui", /*hasRoundingMode=*/false); |
| } |
| if (isSigned) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::bitcast: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| |
| assert(!MissingFeatures::cxxABI()); |
| assert(!MissingFeatures::dataMemberType()); |
| |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| return mlir::success(); |
| } |
| case cir::CastKind::ptr_to_bool: { |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(rewriter, castOp.getLoc(), |
| llvmSrcVal.getType()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( |
| castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr); |
| break; |
| } |
| case cir::CastKind::address_space: { |
| mlir::Type dstTy = castOp.getType(); |
| mlir::Value llvmSrcVal = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy, |
| llvmSrcVal); |
| break; |
| } |
| case cir::CastKind::member_ptr_to_bool: |
| assert(!MissingFeatures::cxxABI()); |
| assert(!MissingFeatures::methodType()); |
| break; |
| default: { |
| return castOp.emitError("Unhandled cast kind: ") |
| << castOp.getKindAttrName(); |
| } |
| } |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBuiltinIntCastOpLowering::matchAndRewrite( |
| cir::BuiltinIntCastOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Both the CIR integer and the builtin integer/index lower to LLVM integer |
| // types, so this cast becomes an integer resize. Signedness is taken from |
| // the CIR integer side (the builtin/index side is treated as signless). |
| bool isUnsigned = true; |
| if (auto cirSrc = mlir::dyn_cast<cir::IntType>(op.getSrc().getType())) |
| isUnsigned = cirSrc.isUnsigned(); |
| else if (auto cirDst = mlir::dyn_cast<cir::IntType>(op.getType())) |
| isUnsigned = cirDst.isUnsigned(); |
| |
| mlir::Value llvmSrc = adaptor.getSrc(); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(op.getType()); |
| auto srcIntTy = mlir::cast<mlir::IntegerType>(llvmSrc.getType()); |
| auto dstIntTy = mlir::cast<mlir::IntegerType>(llvmDstTy); |
| unsigned srcWidth = srcIntTy.getWidth(); |
| unsigned dstWidth = dstIntTy.getWidth(); |
| |
| // Fixed-width builtin integers must match the CIR integer width. |
| // If the converted LLVM widths differ, the non-CIR side must have been |
| // 'index' type (target dependent width). |
| assert((srcWidth == dstWidth || |
| mlir::isa<mlir::IndexType>(op.getSrc().getType()) || |
| mlir::isa<mlir::IndexType>(op.getType())) && |
| "only index casts may change width during lowering"); |
| |
| // For equal widths getLLVMIntCast returns the source unchanged, so casts |
| // between CIR integers and fixed-width builtin integers lower to a no-op. |
| rewriter.replaceOp(op, getLLVMIntCast(rewriter, llvmSrc, dstIntTy, isUnsigned, |
| srcWidth, dstWidth)); |
| return mlir::success(); |
| } |
| |
| static mlir::Value convertToIndexTy(mlir::ConversionPatternRewriter &rewriter, |
| mlir::ModuleOp mod, mlir::Value index, |
| mlir::Type baseTy, cir::IntType strideTy) { |
| mlir::Operation *indexOp = index.getDefiningOp(); |
| if (!indexOp) |
| return index; |
| |
| auto indexType = mlir::cast<mlir::IntegerType>(index.getType()); |
| mlir::DataLayout llvmLayout(mod); |
| std::optional<uint64_t> layoutWidth = llvmLayout.getTypeIndexBitwidth(baseTy); |
| |
| // If there is no change in width, don't do anything. |
| if (!layoutWidth || *layoutWidth == indexType.getWidth()) |
| return index; |
| |
| // If the index comes from a subtraction, make sure the extension happens |
| // before it. To achieve that, look at unary minus, which already got |
| // lowered to "sub 0, x". |
| auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp); |
| bool rewriteSub = false; |
| if (sub) { |
| if (auto lhsConst = |
| dyn_cast<mlir::LLVM::ConstantOp>(sub.getLhs().getDefiningOp())) { |
| auto lhsConstInt = mlir::dyn_cast<mlir::IntegerAttr>(lhsConst.getValue()); |
| if (lhsConstInt && lhsConstInt.getValue() == 0) { |
| index = sub.getRhs(); |
| rewriteSub = true; |
| } |
| } |
| } |
| |
| auto llvmDstType = rewriter.getIntegerType(*layoutWidth); |
| bool isUnsigned = strideTy && strideTy.isUnsigned(); |
| index = getLLVMIntCast(rewriter, index, llvmDstType, isUnsigned, |
| indexType.getWidth(), *layoutWidth); |
| |
| if (rewriteSub) { |
| index = mlir::LLVM::SubOp::create( |
| rewriter, index.getLoc(), |
| mlir::LLVM::ConstantOp::create(rewriter, index.getLoc(), |
| index.getType(), 0), |
| index); |
| // TODO: ensure sub is trivially dead now. |
| rewriter.eraseOp(sub); |
| } |
| |
| return index; |
| } |
| |
| mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite( |
| cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| const mlir::TypeConverter *tc = getTypeConverter(); |
| const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType()); |
| |
| mlir::Type elementTy = |
| convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementType()); |
| |
| // void and function types doesn't really have a layout to use in GEPs, |
| // make it i8 instead. |
| if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) || |
| mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy)) |
| elementTy = mlir::IntegerType::get(elementTy.getContext(), 8, |
| mlir::IntegerType::Signless); |
| // Zero-extend, sign-extend or trunc the pointer value. |
| mlir::Value index = adaptor.getStride(); |
| index = convertToIndexTy( |
| rewriter, ptrStrideOp->getParentOfType<mlir::ModuleOp>(), index, |
| adaptor.getBase().getType(), |
| dyn_cast<cir::IntType>(ptrStrideOp.getOperand(1).getType())); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMGetElementOpLowering::matchAndRewrite( |
| cir::GetElementOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| if (auto arrayTy = |
| mlir::dyn_cast<cir::ArrayType>(op.getBaseType().getPointee())) { |
| const mlir::TypeConverter *converter = getTypeConverter(); |
| const mlir::Type llArrayTy = converter->convertType(arrayTy); |
| const mlir::Type llResultTy = converter->convertType(op.getType()); |
| mlir::Type elementTy = |
| convertTypeForMemory(*converter, dataLayout, op.getElementType()); |
| |
| // void and function types don't really have a layout to use in GEPs, |
| // make it i8 instead. |
| if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) || |
| mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy)) |
| elementTy = rewriter.getIntegerType(8); |
| |
| mlir::Value index = adaptor.getIndex(); |
| index = |
| convertToIndexTy(rewriter, op->getParentOfType<mlir::ModuleOp>(), index, |
| adaptor.getBase().getType(), |
| dyn_cast<cir::IntType>(op.getOperand(1).getType())); |
| |
| // Since the base address is a pointer to an aggregate, the first |
| // offset is always zero. The second offset tell us which member it |
| // will access. |
| std::array<mlir::LLVM::GEPArg, 2> offset{0, index}; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResultTy, llArrayTy, |
| adaptor.getBase(), offset); |
| return mlir::success(); |
| } |
| |
| op.emitError() << "NYI: GetElementOp lowering to LLVM for non-array"; |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite( |
| cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type resultType = |
| getTypeConverter()->convertType(baseClassOp.getType()); |
| mlir::Value derivedAddr = adaptor.getDerivedAddr(); |
| llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = { |
| adaptor.getOffset().getZExtValue()}; |
| mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8, |
| mlir::IntegerType::Signless); |
| if (adaptor.getOffset().getZExtValue() == 0) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>( |
| baseClassOp, resultType, adaptor.getDerivedAddr()); |
| return mlir::success(); |
| } |
| |
| if (baseClassOp.getAssumeNotNull()) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| baseClassOp, resultType, byteType, derivedAddr, offset); |
| } else { |
| auto loc = baseClassOp.getLoc(); |
| mlir::Value isNull = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr, |
| mlir::LLVM::ZeroOp::create(rewriter, loc, derivedAddr.getType())); |
| mlir::Value adjusted = mlir::LLVM::GEPOp::create( |
| rewriter, loc, resultType, byteType, derivedAddr, offset); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull, |
| derivedAddr, adjusted); |
| } |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMDerivedClassAddrOpLowering::matchAndRewrite( |
| cir::DerivedClassAddrOp derivedClassOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type resultType = |
| getTypeConverter()->convertType(derivedClassOp.getType()); |
| mlir::Value baseAddr = adaptor.getBaseAddr(); |
| // The offset is set in the operation as an unsigned value, but it must be |
| // applied as a negative offset. |
| int64_t offsetVal = -(adaptor.getOffset().getZExtValue()); |
| if (offsetVal == 0) { |
| // If the offset is zero, we can just return the base address, |
| rewriter.replaceOp(derivedClassOp, baseAddr); |
| return mlir::success(); |
| } |
| llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {offsetVal}; |
| mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8, |
| mlir::IntegerType::Signless); |
| if (derivedClassOp.getAssumeNotNull()) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| derivedClassOp, resultType, byteType, baseAddr, offset, |
| mlir::LLVM::GEPNoWrapFlags::inbounds); |
| } else { |
| mlir::Location loc = derivedClassOp.getLoc(); |
| mlir::Value isNull = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::eq, baseAddr, |
| mlir::LLVM::ZeroOp::create(rewriter, loc, baseAddr.getType())); |
| mlir::Value adjusted = |
| mlir::LLVM::GEPOp::create(rewriter, loc, resultType, byteType, baseAddr, |
| offset, mlir::LLVM::GEPNoWrapFlags::inbounds); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(derivedClassOp, isNull, |
| baseAddr, adjusted); |
| } |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMFMaxNumOpLowering::matchAndRewrite( |
| cir::FMaxNumOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type resTy = typeConverter->convertType(op.getType()); |
| if (cir::FenvAttr fenv = op.getFenvAttr()) |
| return lowerToConstrainedFPIntrinsic( |
| op, adaptor.getOperands(), fenv, resTy, rewriter, "maxnum", |
| /*hasRoundingMode=*/false, mlir::LLVM::FastmathFlags::nsz); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MaxNumOp>( |
| op, resTy, adaptor.getLhs(), adaptor.getRhs(), |
| mlir::LLVM::FastmathFlags::nsz); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMFMinNumOpLowering::matchAndRewrite( |
| cir::FMinNumOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type resTy = typeConverter->convertType(op.getType()); |
| if (cir::FenvAttr fenv = op.getFenvAttr()) |
| return lowerToConstrainedFPIntrinsic( |
| op, adaptor.getOperands(), fenv, resTy, rewriter, "minnum", |
| /*hasRoundingMode=*/false, mlir::LLVM::FastmathFlags::nsz); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MinNumOp>( |
| op, resTy, adaptor.getLhs(), adaptor.getRhs(), |
| mlir::LLVM::FastmathFlags::nsz); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite( |
| cir::AllocaOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value size = |
| op.isDynamic() |
| ? adaptor.getDynAllocSize() |
| : mlir::LLVM::ConstantOp::create( |
| rewriter, op.getLoc(), |
| typeConverter->convertType(rewriter.getI32Type()), 1); |
| mlir::Type elementTy = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType()); |
| if (!elementTy) |
| return op.emitError() |
| << "NYI: lowering alloca of a type with no memory representation"; |
| mlir::Type resultTy = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); |
| |
| assert(!cir::MissingFeatures::addressSpace()); |
| assert(!cir::MissingFeatures::opAllocaAnnotations()); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(op, resultTy, elementTy, |
| size, op.getAlignment()); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite( |
| cir::RotateOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as |
| // the operand. |
| mlir::Value input = adaptor.getInput(); |
| if (op.isRotateLeft()) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input, |
| adaptor.getAmount()); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input, |
| adaptor.getAmount()); |
| return mlir::LogicalResult::success(); |
| } |
| |
| /// The `llvm.byval`, `llvm.sret`, and `llvm.byref` argument attributes carry |
| /// the pointee type as a TypeAttr. After the CallConvLowering pass that type |
| /// is still a CIR record; remap it to the lowered LLVM type so translation to |
| /// LLVM IR does not encounter a CIR type in an attribute. Returns the input |
| /// unchanged when there is nothing to convert. |
| static mlir::ArrayAttr |
| convertTypedArgAttrs(mlir::ArrayAttr argAttrs, |
| const mlir::TypeConverter &converter, |
| mlir::MLIRContext *ctx) { |
| if (!argAttrs) |
| return argAttrs; |
| bool changed = false; |
| SmallVector<mlir::Attribute> loweredArgAttrs; |
| loweredArgAttrs.reserve(argAttrs.size()); |
| for (mlir::Attribute a : argAttrs) { |
| auto dict = cast<mlir::DictionaryAttr>(a); |
| SmallVector<mlir::NamedAttribute> entries(dict.begin(), dict.end()); |
| for (mlir::NamedAttribute &entry : entries) { |
| StringRef name = entry.getName().strref(); |
| if (name != mlir::LLVM::LLVMDialect::getByValAttrName() && |
| name != mlir::LLVM::LLVMDialect::getStructRetAttrName() && |
| name != mlir::LLVM::LLVMDialect::getByRefAttrName()) |
| continue; |
| auto typeAttr = dyn_cast<mlir::TypeAttr>(entry.getValue()); |
| if (!typeAttr) |
| continue; |
| mlir::Type lowered = converter.convertType(typeAttr.getValue()); |
| if (lowered && lowered != typeAttr.getValue()) { |
| entry.setValue(mlir::TypeAttr::get(lowered)); |
| changed = true; |
| } |
| } |
| loweredArgAttrs.push_back(mlir::DictionaryAttr::get(ctx, entries)); |
| } |
| return changed ? mlir::ArrayAttr::get(ctx, loweredArgAttrs) : argAttrs; |
| } |
| |
| static void lowerCallAttributes(cir::CIRCallOpInterface op, |
| const mlir::TypeConverter &converter, |
| SmallVectorImpl<mlir::NamedAttribute> &result) { |
| for (mlir::NamedAttribute attr : op->getAttrs()) { |
| if (attr.getName() == CIRDialect::getCalleeAttrName() || |
| attr.getName() == CIRDialect::getSideEffectAttrName() || |
| attr.getName() == CIRDialect::getNoThrowAttrName() || |
| attr.getName() == CIRDialect::getNoUnwindAttrName() || |
| attr.getName() == CIRDialect::getNoReturnAttrName() || |
| attr.getName() == op.getInlineKindAttrName() || |
| attr.getName() == CIRDialect::getMustTailAttrName()) |
| continue; |
| |
| assert(!cir::MissingFeatures::opFuncExtraAttrs()); |
| if (attr.getName() == CIRDialect::getArgAttrsAttrName()) { |
| auto argAttrs = cast<mlir::ArrayAttr>(attr.getValue()); |
| result.emplace_back( |
| attr.getName(), |
| convertTypedArgAttrs(argAttrs, converter, op->getContext())); |
| continue; |
| } |
| result.push_back(attr); |
| } |
| } |
| |
| static mlir::LogicalResult |
| rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands, |
| mlir::ConversionPatternRewriter &rewriter, |
| const mlir::TypeConverter *converter, |
| mlir::SymbolTableCollection &symbolTables, |
| mlir::FlatSymbolRefAttr calleeAttr, |
| mlir::Block *continueBlock = nullptr, |
| mlir::Block *landingPadBlock = nullptr) { |
| llvm::SmallVector<mlir::Type, 8> llvmResults; |
| mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes(); |
| auto call = cast<cir::CIRCallOpInterface>(op); |
| |
| if (converter->convertTypes(cirResults, llvmResults).failed()) |
| return mlir::failure(); |
| |
| assert(!cir::MissingFeatures::opCallCallConv()); |
| |
| mlir::LLVM::MemoryEffectsAttr memoryEffects; |
| bool noUnwind = false; |
| bool willReturn = false; |
| bool noReturn = false; |
| convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(), |
| memoryEffects, noUnwind, willReturn, noReturn); |
| |
| SmallVector<mlir::NamedAttribute, 4> attributes; |
| lowerCallAttributes(call, *converter, attributes); |
| |
| mlir::LLVM::LLVMFunctionType llvmFnTy; |
| |
| // Temporary to handle the case where we need to prepend an operand if the |
| // callee is an alias. |
| SmallVector<mlir::Value> adjustedCallOperands; |
| |
| if (calleeAttr) { // direct call |
| mlir::Operation *callee = |
| symbolTables.lookupNearestSymbolFrom(op, calleeAttr); |
| if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) { |
| llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>( |
| fn.getFunctionType()); |
| assert(llvmFnTy && "Failed to convert function type"); |
| } else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) { |
| // If the callee was an alias. In that case, |
| // we need to prepend the address of the alias to the operands. The |
| // way aliases work in the LLVM dialect is a little counter-intuitive. |
| // The AliasOp itself is a pseudo-function that returns the address of |
| // the global value being aliased, but when we generate the call we |
| // need to insert an operation that gets the address of the AliasOp. |
| // This all gets sorted out when the LLVM dialect is lowered to LLVM IR. |
| auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr); |
| auto addrOfAlias = |
| mlir::LLVM::AddressOfOp::create( |
| rewriter, op->getLoc(), |
| mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr) |
| .getResult(); |
| adjustedCallOperands.push_back(addrOfAlias); |
| |
| // Now add the regular operands and assign this to the range value. |
| llvm::append_range(adjustedCallOperands, callOperands); |
| callOperands = adjustedCallOperands; |
| |
| // Clear the callee attribute because we're calling an alias. |
| calleeAttr = {}; |
| llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType()); |
| } else { |
| // Was this an ifunc? |
| return op->emitError("Unexpected callee type!"); |
| } |
| } else { // indirect call |
| assert(!op->getOperands().empty() && |
| "operands list must no be empty for the indirect call"); |
| auto calleeTy = op->getOperands().front().getType(); |
| auto calleePtrTy = cast<cir::PointerType>(calleeTy); |
| auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee()); |
| llvm::append_range(adjustedCallOperands, callOperands); |
| llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>( |
| converter->convertType(calleeFuncTy)); |
| } |
| |
| assert(!cir::MissingFeatures::opCallCallConv()); |
| |
| if (landingPadBlock) { |
| auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::InvokeOp>( |
| op, llvmFnTy, calleeAttr, callOperands, continueBlock, |
| mlir::ValueRange{}, landingPadBlock, mlir::ValueRange{}); |
| newOp->setAttrs(attributes); |
| } else { |
| auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>( |
| op, llvmFnTy, calleeAttr, callOperands); |
| newOp->setAttrs(attributes); |
| if (memoryEffects) |
| newOp.setMemoryEffectsAttr(memoryEffects); |
| newOp.setNoUnwind(noUnwind); |
| newOp.setWillReturn(willReturn); |
| newOp.setNoreturn(noReturn); |
| if (op->hasAttr(CIRDialect::getMustTailAttrName())) |
| newOp.setTailCallKind(mlir::LLVM::TailCallKind::MustTail); |
| |
| if (std::optional<cir::InlineKind> inlineKind = call.getInlineKind()) { |
| newOp.setNoInline(*inlineKind == cir::InlineKind::NoInline); |
| newOp.setInlineHint(*inlineKind == cir::InlineKind::InlineHint); |
| newOp.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline); |
| } |
| } |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite( |
| cir::CallOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter, |
| getTypeConverter(), symbolTables, |
| op.getCalleeAttr()); |
| } |
| |
| mlir::LogicalResult CIRToLLVMTryCallOpLowering::matchAndRewrite( |
| cir::TryCallOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::opCallCallConv()); |
| return rewriteCallOrInvoke( |
| op.getOperation(), adaptor.getOperands(), rewriter, getTypeConverter(), |
| symbolTables, op.getCalleeAttr(), op.getNormalDest(), op.getUnwindDest()); |
| } |
| |
| mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite( |
| cir::ReturnAddrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType()); |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress", |
| llvmPtrTy, adaptor.getOperands()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite( |
| cir::FrameAddrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType()); |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy, |
| adaptor.getOperands()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMClearCacheOpLowering::matchAndRewrite( |
| cir::ClearCacheOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value begin = adaptor.getBegin(); |
| mlir::Value end = adaptor.getEnd(); |
| auto intrinNameAttr = |
| mlir::StringAttr::get(op.getContext(), "llvm.clear_cache"); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::CallIntrinsicOp>( |
| op, mlir::Type{}, intrinNameAttr, mlir::ValueRange{begin, end}); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAddrOfReturnAddrOpLowering::matchAndRewrite( |
| cir::AddrOfReturnAddrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType()); |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.addressofreturnaddress", |
| llvmPtrTy, adaptor.getOperands()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite( |
| cir::LoadOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type llvmTy = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); |
| if (!llvmTy) |
| return op.emitError() |
| << "NYI: lowering load of a type with no memory representation"; |
| mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder()); |
| std::optional<size_t> opAlign = op.getAlignment(); |
| unsigned alignment = (unsigned)opAlign.value_or( |
| getMemoryFallbackAlignment(op.getType(), llvmTy, dataLayout)); |
| |
| assert(!cir::MissingFeatures::lowerModeOptLevel()); |
| |
| std::optional<llvm::StringRef> llvmSyncScope = |
| getLLVMSyncScope(op.getSyncScope()); |
| |
| mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create( |
| rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment, |
| op.getIsVolatile(), /*isNonTemporal=*/op.getIsNontemporal(), |
| /*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false, ordering, |
| llvmSyncScope.value_or(std::string())); |
| if (mlir::Attribute domain = op->getAttr("cir.riscv_nontemporal_domain")) |
| newLoad->setAttr("cir.riscv_nontemporal_domain", domain); |
| |
| // Convert adapted result to its original type if needed. |
| mlir::Value result = emitFromMemory(rewriter, *getTypeConverter(), dataLayout, |
| op, newLoad.getResult()); |
| rewriter.replaceOp(op, result); |
| assert(!cir::MissingFeatures::opLoadStoreTbaa()); |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult |
| cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite( |
| cir::VecMaskedLoadOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type llvmResTy = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType()); |
| if (!llvmResTy) |
| return op.emitError() |
| << "NYI: lowering masked load of a type with no memory " |
| "representation"; |
| |
| std::optional<size_t> opAlign = op.getAlignment(); |
| unsigned alignment = |
| (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmResTy)); |
| |
| mlir::IntegerAttr alignAttr = rewriter.getI32IntegerAttr(alignment); |
| |
| auto newLoad = mlir::LLVM::MaskedLoadOp::create( |
| rewriter, op.getLoc(), llvmResTy, adaptor.getAddr(), adaptor.getMask(), |
| adaptor.getPassThru(), alignAttr); |
| |
| rewriter.replaceOp(op, newLoad.getResult()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite( |
| cir::StoreOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder()); |
| mlir::Type valueType = op.getValue().getType(); |
| const mlir::Type llvmTy = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, valueType); |
| if (!llvmTy) |
| return op.emitError() |
| << "NYI: lowering store of a type with no memory representation"; |
| std::optional<size_t> opAlign = op.getAlignment(); |
| unsigned alignment = (unsigned)opAlign.value_or( |
| getMemoryFallbackAlignment(valueType, llvmTy, dataLayout)); |
| |
| assert(!cir::MissingFeatures::lowerModeOptLevel()); |
| |
| // Convert adapted value to its memory type if needed. |
| mlir::Value value = emitToMemory(rewriter, dataLayout, |
| op.getValue().getType(), adaptor.getValue()); |
| assert(!cir::MissingFeatures::opLoadStoreTbaa()); |
| |
| std::optional<llvm::StringRef> llvmSyncScope = |
| getLLVMSyncScope(op.getSyncScope()); |
| |
| mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create( |
| rewriter, op->getLoc(), value, adaptor.getAddr(), alignment, |
| op.getIsVolatile(), |
| /*isNonTemporal=*/op.getIsNontemporal(), /*isInvariantGroup=*/false, |
| memorder, llvmSyncScope.value_or(std::string())); |
| if (mlir::Attribute domain = op->getAttr("cir.riscv_nontemporal_domain")) |
| storeOp->setAttr("cir.riscv_nontemporal_domain", domain); |
| rewriter.replaceOp(op, storeOp); |
| assert(!cir::MissingFeatures::opLoadStoreTbaa()); |
| return mlir::LogicalResult::success(); |
| } |
| |
| static mlir::Type getConstArrayBaseElementType(mlir::Type ty) { |
| while (auto arrTy = mlir::dyn_cast<cir::ArrayType>(ty)) |
| ty = arrTy.getElementType(); |
| return ty; |
| } |
| |
| static bool isBulkLowerableConstArrayBaseElement(mlir::Type baseElemTy) { |
| return mlir::isa<cir::PointerType, cir::IntType, cir::BoolType, |
| cir::FPTypeInterface, cir::RecordType>(baseElemTy); |
| } |
| |
| mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite( |
| cir::ConstantOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Attribute attr = op.getValue(); |
| |
| if (mlir::isa<cir::PoisonAttr>(attr)) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>( |
| op, getTypeConverter()->convertType(op.getType())); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<cir::UndefAttr>(attr)) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::UndefOp>( |
| op, getTypeConverter()->convertType(op.getType())); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<mlir::IntegerType>(op.getType())) { |
| // Verified cir.const operations cannot actually be of these types, but the |
| // lowering pass may generate temporary cir.const operations with these |
| // types. This is OK since MLIR allows unverified operations to be alive |
| // during a pass as long as they don't live past the end of the pass. |
| attr = op.getValue(); |
| } else if (mlir::isa<cir::BoolType>(op.getType())) { |
| int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue(); |
| attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()), |
| value); |
| } else if (mlir::isa<cir::IntType>(op.getType())) { |
| // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint |
| if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) { |
| // We can have a global view with an integer type in the case of method |
| // pointers, but the lowering of those doesn't go through this path. |
| // They are handled in the visitCirAttr. This is left as an error until |
| // we have a test case that reaches it. |
| assert(!cir::MissingFeatures::globalViewIntLowering()); |
| op.emitError() << "global view with integer type"; |
| return mlir::failure(); |
| } |
| |
| attr = rewriter.getIntegerAttr( |
| typeConverter->convertType(op.getType()), |
| mlir::cast<cir::IntAttr>(op.getValue()).getValue()); |
| } else if (mlir::isa<cir::FPTypeInterface>(op.getType())) { |
| attr = rewriter.getFloatAttr( |
| typeConverter->convertType(op.getType()), |
| mlir::cast<cir::FPAttr>(op.getValue()).getValue()); |
| } else if (mlir::isa<cir::PointerType>(op.getType())) { |
| // Optimize with dedicated LLVM op for null pointers. |
| if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) { |
| if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>( |
| op, typeConverter->convertType(op.getType())); |
| return mlir::success(); |
| } |
| } |
| // Lower GlobalViewAttr to llvm.mlir.addressof |
| if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) { |
| auto newOp = lowerCirAttrAsValue(op, gv, rewriter, symbolTables, |
| getTypeConverter()); |
| rewriter.replaceOp(op, newOp); |
| return mlir::success(); |
| } |
| attr = op.getValue(); |
| } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) { |
| const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue()); |
| if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue())) |
| return op.emitError() << "array does not have a constant initializer"; |
| |
| std::optional<mlir::Attribute> denseAttr; |
| if (constArr && (denseAttr = lowerConstArrayAttr(constArr, symbolTables, |
| typeConverter))) { |
| attr = denseAttr.value(); |
| } else { |
| const mlir::Value initVal = lowerCirAttrAsValue( |
| op, op.getValue(), rewriter, symbolTables, typeConverter); |
| rewriter.replaceOp(op, initVal); |
| return mlir::success(); |
| } |
| } else if (const auto recordAttr = |
| mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) { |
| auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, symbolTables, |
| typeConverter); |
| rewriter.replaceOp(op, initVal); |
| return mlir::success(); |
| } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) { |
| rewriter.replaceOp(op, |
| lowerCirAttrAsValue(op, op.getValue(), rewriter, |
| symbolTables, getTypeConverter())); |
| return mlir::success(); |
| } else if (mlir::isa<cir::RecordType>(op.getType())) { |
| if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(attr)) { |
| mlir::Value initVal = |
| lowerCirAttrAsValue(op, attr, rewriter, symbolTables, typeConverter); |
| rewriter.replaceOp(op, initVal); |
| return mlir::success(); |
| } |
| return op.emitError() << "unsupported lowering for record constant type " |
| << op.getType(); |
| } else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) { |
| mlir::Type complexElemTy = complexTy.getElementType(); |
| mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy); |
| |
| if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) { |
| mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy); |
| mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr}); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>( |
| op, getTypeConverter()->convertType(op.getType()), array); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<cir::UndefAttr>(op.getValue())) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::UndefOp>( |
| op, getTypeConverter()->convertType(op.getType())); |
| return mlir::success(); |
| } |
| |
| auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue()); |
| |
| mlir::Attribute components[2]; |
| if (mlir::isa<cir::IntType>(complexElemTy)) { |
| components[0] = rewriter.getIntegerAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue()); |
| components[1] = rewriter.getIntegerAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue()); |
| } else { |
| components[0] = rewriter.getFloatAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue()); |
| components[1] = rewriter.getFloatAttr( |
| complexElemLLVMTy, |
| mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue()); |
| } |
| |
| attr = rewriter.getArrayAttr(components); |
| } else { |
| return op.emitError() << "unsupported constant type " << op.getType(); |
| } |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>( |
| op, getTypeConverter()->convertType(op.getType()), attr); |
| |
| return mlir::success(); |
| } |
| |
| static uint64_t getTypeSize(mlir::Type type, mlir::Operation &op) { |
| mlir::DataLayout layout(op.getParentOfType<mlir::ModuleOp>()); |
| // For LLVM purposes we treat void as u8. |
| if (isa<cir::VoidType>(type)) |
| type = cir::IntType::get(type.getContext(), 8, /*isSigned=*/false); |
| return llvm::divideCeil(layout.getTypeSizeInBits(type), 8); |
| } |
| |
| mlir::LogicalResult CIRToLLVMPrefetchOpLowering::matchAndRewrite( |
| cir::PrefetchOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::Prefetch>( |
| op, adaptor.getAddr(), adaptor.getIsWrite(), adaptor.getLocality(), |
| /*DataCache=*/1); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite( |
| cir::PtrDiffOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto dstTy = mlir::cast<cir::IntType>(op.getType()); |
| mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy); |
| |
| auto lhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy, |
| adaptor.getLhs()); |
| auto rhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy, |
| adaptor.getRhs()); |
| |
| auto diff = |
| mlir::LLVM::SubOp::create(rewriter, op.getLoc(), llvmDstTy, lhs, rhs); |
| |
| cir::PointerType ptrTy = op.getLhs().getType(); |
| assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee()); |
| uint64_t typeSize = getTypeSize(ptrTy.getPointee(), *op); |
| |
| // Avoid silly division by 1. |
| mlir::Value resultVal = diff.getResult(); |
| if (typeSize != 1) { |
| auto typeSizeVal = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| llvmDstTy, typeSize); |
| |
| if (dstTy.isUnsigned()) { |
| auto uDiv = |
| mlir::LLVM::UDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal); |
| uDiv.setIsExact(true); |
| resultVal = uDiv.getResult(); |
| } else { |
| auto sDiv = |
| mlir::LLVM::SDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal); |
| sDiv.setIsExact(true); |
| resultVal = sDiv.getResult(); |
| } |
| } |
| rewriter.replaceOp(op, resultVal); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite( |
| cir::ExpectOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // TODO(cir): do not generate LLVM intrinsics under -O0 |
| assert(!cir::MissingFeatures::optInfoAttr()); |
| |
| std::optional<llvm::APFloat> prob = op.getProb(); |
| if (prob) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>( |
| op, adaptor.getVal(), adaptor.getExpected(), prob.value()); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(), |
| adaptor.getExpected()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAbsOpLowering::matchAndRewrite( |
| cir::AbsOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type resTy = typeConverter->convertType(op.getType()); |
| auto absOp = mlir::LLVM::AbsOp::create(rewriter, op.getLoc(), resTy, |
| adaptor.getOperands()[0], |
| adaptor.getMinIsPoison()); |
| rewriter.replaceOp(op, absOp); |
| return mlir::success(); |
| } |
| |
| /// Convert the `cir.func` attributes to `llvm.func` attributes. |
| /// Only retain those attributes that are not constructed by |
| /// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out |
| /// argument attributes. |
| void CIRToLLVMFuncOpLowering::lowerFuncAttributes( |
| cir::FuncOp func, bool filterArgAndResAttrs, |
| SmallVectorImpl<mlir::NamedAttribute> &result) const { |
| for (mlir::NamedAttribute attr : func->getAttrs()) { |
| if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() || |
| attr.getName() == func.getFunctionTypeAttrName() || |
| attr.getName() == getLinkageAttrNameString() || |
| attr.getName() == func.getCallingConvAttrName() || |
| attr.getName() == func.getDsoLocalAttrName() || |
| attr.getName() == func.getInlineKindAttrName() || |
| attr.getName() == func.getSideEffectAttrName() || |
| attr.getName() == CIRDialect::getNoReturnAttrName() || |
| attr.getName() == CIRDialect::getStrictFPAttrName() || |
| attr.getName() == func.getAnnotationsAttrName() || |
| (filterArgAndResAttrs && |
| (attr.getName() == func.getArgAttrsAttrName() || |
| attr.getName() == func.getResAttrsAttrName()))) |
| continue; |
| |
| assert(!cir::MissingFeatures::opFuncExtraAttrs()); |
| if (attr.getName() == func.getArgAttrsAttrName()) { |
| auto argAttrs = cast<mlir::ArrayAttr>(attr.getValue()); |
| result.emplace_back( |
| attr.getName(), |
| convertTypedArgAttrs(argAttrs, *getTypeConverter(), getContext())); |
| continue; |
| } |
| result.push_back(attr); |
| } |
| } |
| |
| mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias( |
| cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| SmallVector<mlir::NamedAttribute, 4> attributes; |
| lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes); |
| |
| mlir::Location loc = op.getLoc(); |
| auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>( |
| op, ty, convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(), |
| mlir::LLVM::ThreadLocalMode::NotThreadLocal, attributes); |
| |
| // Create the alias body |
| mlir::OpBuilder builder(op.getContext()); |
| mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion()); |
| builder.setInsertionPointToStart(block); |
| // The type of AddressOfOp is always a pointer. |
| assert(!cir::MissingFeatures::addressSpace()); |
| mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext()); |
| auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee); |
| mlir::LLVM::ReturnOp::create(builder, loc, addrOp); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite( |
| cir::FuncOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| cir::FuncType fnType = op.getFunctionType(); |
| bool isDsoLocal = op.getDsoLocal(); |
| mlir::TypeConverter::SignatureConversion signatureConversion( |
| fnType.getNumInputs()); |
| |
| for (const auto &argType : llvm::enumerate(fnType.getInputs())) { |
| mlir::Type convertedType = typeConverter->convertType(argType.value()); |
| if (!convertedType) |
| return mlir::failure(); |
| signatureConversion.addInputs(argType.index(), convertedType); |
| } |
| |
| mlir::Type resultType = |
| getTypeConverter()->convertType(fnType.getReturnType()); |
| |
| // Create the LLVM function operation. |
| mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get( |
| resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()), |
| signatureConversion.getConvertedTypes(), |
| /*isVarArg=*/fnType.isVarArg()); |
| |
| // If this is an alias, it needs to be lowered to llvm::AliasOp. |
| if (std::optional<llvm::StringRef> aliasee = op.getAliasee()) |
| return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter); |
| |
| // LLVMFuncOp expects a single FileLine Location instead of a fused |
| // location. |
| mlir::Location loc = op.getLoc(); |
| if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc)) |
| loc = fusedLoc.getLocations()[0]; |
| assert((mlir::isa<mlir::FileLineColLoc>(loc) || |
| mlir::isa<mlir::UnknownLoc>(loc)) && |
| "expected single location or unknown location here"); |
| |
| mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage()); |
| mlir::LLVM::CConv cconv = convertCallingConv(op.getCallingConv()); |
| SmallVector<mlir::NamedAttribute, 4> attributes; |
| lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes); |
| |
| mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create( |
| rewriter, loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv, |
| mlir::SymbolRefAttr(), attributes); |
| |
| assert(!cir::MissingFeatures::opFuncMultipleReturnVals()); |
| |
| if (std::optional<cir::SideEffect> sideEffectKind = op.getSideEffect()) { |
| switch (*sideEffectKind) { |
| case cir::SideEffect::All: |
| break; |
| case cir::SideEffect::Pure: |
| fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get( |
| fn.getContext(), |
| /*other=*/mlir::LLVM::ModRefInfo::Ref, |
| /*argMem=*/mlir::LLVM::ModRefInfo::Ref, |
| /*inaccessibleMem=*/mlir::LLVM::ModRefInfo::Ref, |
| /*errnoMem=*/mlir::LLVM::ModRefInfo::Ref, |
| /*targetMem0=*/mlir::LLVM::ModRefInfo::Ref, |
| /*targetMem1=*/mlir::LLVM::ModRefInfo::Ref)); |
| fn.setNoUnwind(true); |
| fn.setWillReturn(true); |
| break; |
| case cir::SideEffect::Const: |
| fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get( |
| fn.getContext(), |
| /*other=*/mlir::LLVM::ModRefInfo::NoModRef, |
| /*argMem=*/mlir::LLVM::ModRefInfo::NoModRef, |
| /*inaccessibleMem=*/mlir::LLVM::ModRefInfo::NoModRef, |
| /*errnoMem=*/mlir::LLVM::ModRefInfo::NoModRef, |
| /*targetMem0=*/mlir::LLVM::ModRefInfo::NoModRef, |
| /*targetMem1=*/mlir::LLVM::ModRefInfo::NoModRef)); |
| fn.setNoUnwind(true); |
| fn.setWillReturn(true); |
| break; |
| } |
| } |
| |
| if (op->hasAttr(CIRDialect::getNoReturnAttrName())) |
| fn.setNoreturn(true); |
| |
| // The LLVM dialect's LLVMFuncOp has no dedicated field for the `strictfp` |
| // function attribute, so route it through the `passthrough` array. The MLIR |
| // LLVM IR translator forwards `passthrough` entries to LLVM IR as function |
| // attributes. |
| if (op->hasAttr(CIRDialect::getStrictFPAttrName())) |
| fn.setPassthroughAttr(rewriter.getArrayAttr( |
| {rewriter.getStringAttr(CIRDialect::getStrictFPAttrName())})); |
| |
| if (std::optional<cir::InlineKind> inlineKind = op.getInlineKind()) { |
| fn.setNoInline(*inlineKind == cir::InlineKind::NoInline); |
| fn.setInlineHint(*inlineKind == cir::InlineKind::InlineHint); |
| fn.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline); |
| } |
| |
| if (std::optional<llvm::StringRef> personality = op.getPersonality()) |
| fn.setPersonality(*personality); |
| |
| fn.setVisibility_( |
| lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility())); |
| |
| rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end()); |
| if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter, |
| &signatureConversion))) |
| return mlir::failure(); |
| |
| rewriter.eraseOp(op); |
| |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite( |
| cir::GetGlobalOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // FIXME(cir): Premature DCE to avoid lowering stuff we're not using. |
| // CIRGen should mitigate this and not emit the get_global. |
| if (op->getUses().empty()) { |
| rewriter.eraseOp(op); |
| return mlir::success(); |
| } |
| |
| mlir::Type type = getTypeConverter()->convertType(op.getType()); |
| mlir::Operation *newop = mlir::LLVM::AddressOfOp::create( |
| rewriter, op.getLoc(), type, op.getName()); |
| |
| if (op.getTls()) { |
| // Handle access to TLS via intrinsic. |
| newop = mlir::LLVM::ThreadlocalAddressOp::create(rewriter, op.getLoc(), |
| type, newop->getResult(0)); |
| } |
| |
| rewriter.replaceOp(op, newop); |
| return mlir::success(); |
| } |
| |
| llvm::SmallVector<mlir::NamedAttribute> |
| CIRToLLVMGlobalOpLowering::lowerGlobalAttributes( |
| cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const { |
| SmallVector<mlir::NamedAttribute> attributes; |
| |
| if (mlir::StringAttr sectionAttr = op.getSectionAttr()) |
| attributes.push_back(rewriter.getNamedAttr("section", sectionAttr)); |
| |
| mlir::LLVM::VisibilityAttr visibility = mlir::LLVM::VisibilityAttr::get( |
| getContext(), |
| lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility())); |
| attributes.push_back(rewriter.getNamedAttr("visibility_", visibility)); |
| |
| if (op->getAttr(CUDAExternallyInitializedAttr::getMnemonic())) |
| attributes.push_back(rewriter.getNamedAttr("externally_initialized", |
| rewriter.getUnitAttr())); |
| |
| return attributes; |
| } |
| |
| static mlir::LLVM::ThreadLocalMode |
| convertTlsModelAttrToLLVM(TLSModelAttr attr) { |
| // assert that we can just static-cast these. |
| #define CHECK_ENUM(CIR, LLVM_VAL) \ |
| static_assert(static_cast<unsigned>(TLSModel::CIR) == \ |
| static_cast<unsigned>(mlir::LLVM::ThreadLocalMode::LLVM_VAL)) |
| CHECK_ENUM(GeneralDynamic, GeneralDynamic); |
| CHECK_ENUM(LocalDynamic, LocalDynamic); |
| CHECK_ENUM(InitialExec, InitialExec); |
| CHECK_ENUM(LocalExec, LocalExec); |
| #undef CHECK_ENUM |
| |
| if (!attr) |
| return mlir::LLVM::ThreadLocalMode::NotThreadLocal; |
| |
| return static_cast<mlir::LLVM::ThreadLocalMode>(attr.getValue()); |
| } |
| |
| /// Replace CIR global with a region initialized LLVM global and update |
| /// insertion point to the end of the initializer block. |
| void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp( |
| cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type llvmType = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, op.getSymType()); |
| |
| // Keep the global's type in sync with the value built by CIRAttrToValue: a |
| // flexible array member initializer requires an oversized anonymous struct. |
| if (std::optional<mlir::Attribute> init = op.getInitialValue()) |
| llvmType = adjustGlobalTypeForInit(llvmType, *init, *getTypeConverter(), |
| dataLayout); |
| |
| // FIXME: These default values are placeholders until the the equivalent |
| // attributes are available on cir.global ops. This duplicates code |
| // in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go |
| // away when the placeholders are no longer needed. |
| const bool isConst = op.getConstant(); |
| unsigned addrSpace = 0; |
| if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| op.getAddrSpaceAttr())) |
| addrSpace = targetAS.getValue(); |
| const bool isDsoLocal = op.getDsoLocal(); |
| mlir::LLVM::ThreadLocalMode threadLocalMode = |
| convertTlsModelAttrToLLVM(op.getTlsModelAttr()); |
| const uint64_t alignment = op.getAlignment().value_or(0); |
| const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage()); |
| const StringRef symbol = op.getSymName(); |
| mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter); |
| |
| SmallVector<mlir::NamedAttribute> attributes = |
| lowerGlobalAttributes(op, rewriter); |
| |
| mlir::LLVM::GlobalOp newGlobalOp = |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>( |
| op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace, |
| isDsoLocal, threadLocalMode, comdatAttr, attributes); |
| newGlobalOp.getRegion().emplaceBlock(); |
| rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock()); |
| } |
| |
| mlir::LogicalResult |
| CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal( |
| cir::GlobalOp op, mlir::Attribute init, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // TODO: Generalize this handling when more types are needed here. |
| assert( |
| (isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr, cir::ConstArrayAttr, |
| cir::ConstRecordAttr, cir::ConstVectorAttr, cir::ConstPtrAttr, |
| cir::ConstComplexAttr, cir::GlobalViewAttr, cir::TypeInfoAttr, |
| cir::UndefAttr, cir::PoisonAttr, cir::VTableAttr, cir::ZeroAttr>( |
| init))); |
| |
| // TODO(cir): once LLVM's dialect has proper equivalent attributes this |
| // should be updated. For now, we use a custom op to initialize globals |
| // to the appropriate value. |
| const mlir::Location loc = op.getLoc(); |
| setupRegionInitializedLLVMGlobalOp(op, rewriter); |
| |
| // Pass blockInfoAddr so that block address initializers (either as the whole |
| // initializer or nested inside an aggregate) can be resolved by the |
| // BlockAddrInfoAttr visitor. |
| CIRAttrToValue valueConverter(op, rewriter, symbolTables, typeConverter, |
| &blockInfoAddr); |
| mlir::Value value = valueConverter.visit(init); |
| mlir::LLVM::ReturnOp::create(rewriter, loc, value); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite( |
| cir::GlobalOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // If this global requires non-trivial initialization or destruction, |
| // that needs to be moved to runtime handlers during LoweringPrepare. |
| if (!op.getCtorRegion().empty() || !op.getDtorRegion().empty()) |
| return op.emitError() << "GlobalOp ctor and dtor regions should be removed " |
| "in LoweringPrepare"; |
| |
| std::optional<mlir::Attribute> init = op.getInitialValue(); |
| |
| // Fetch required values to create LLVM op. |
| const mlir::Type cirSymType = op.getSymType(); |
| |
| // This is the LLVM dialect type. |
| mlir::Type llvmType = |
| convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType); |
| if (!llvmType) |
| return op.emitError() |
| << "NYI: lowering global of a type with no memory representation"; |
| |
| // A flexible array member initializer makes the constant larger than the |
| // record's declared type, so the global must use an oversized anonymous |
| // struct instead. |
| if (init.has_value()) |
| llvmType = adjustGlobalTypeForInit(llvmType, *init, *getTypeConverter(), |
| dataLayout); |
| |
| // FIXME: These default values are placeholders until the the equivalent |
| // attributes are available on cir.global ops. |
| const bool isConst = op.getConstant(); |
| unsigned addrSpace = 0; |
| if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| op.getAddrSpaceAttr())) |
| addrSpace = targetAS.getValue(); |
| const bool isDsoLocal = op.getDsoLocal(); |
| mlir::LLVM::ThreadLocalMode threadLocalMode = |
| convertTlsModelAttrToLLVM(op.getTlsModelAttr()); |
| const uint64_t alignment = op.getAlignment().value_or(0); |
| const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage()); |
| const StringRef symbol = op.getSymName(); |
| SmallVector<mlir::NamedAttribute> attributes = |
| lowerGlobalAttributes(op, rewriter); |
| |
| // If this is a variable alias, lower it to llvm.mlir.alias. |
| if (std::optional<llvm::StringRef> aliasee = op.getAliasee()) { |
| mlir::Location loc = op.getLoc(); |
| auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>( |
| op, llvmType, linkage, symbol, isDsoLocal, threadLocalMode, attributes); |
| |
| mlir::OpBuilder builder(op.getContext()); |
| mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion()); |
| builder.setInsertionPointToStart(block); |
| mlir::Type ptrTy = |
| mlir::LLVM::LLVMPointerType::get(getContext(), addrSpace); |
| auto addrOp = |
| mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, *aliasee); |
| mlir::LLVM::ReturnOp::create(builder, loc, addrOp); |
| return mlir::success(); |
| } |
| |
| if (init.has_value()) { |
| if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) { |
| GlobalInitAttrRewriter initRewriter(llvmType, rewriter); |
| init = initRewriter.visit(init.value()); |
| // If initRewriter returned a null attribute, init will have a value but |
| // the value will be null. If that happens, initRewriter didn't handle the |
| // attribute type. It probably needs to be added to |
| // GlobalInitAttrRewriter. |
| if (!init.value()) { |
| op.emitError() << "unsupported initializer '" << init.value() << "'"; |
| return mlir::failure(); |
| } |
| } else if (auto constArr = |
| mlir::dyn_cast<cir::ConstArrayAttr>(init.value())) { |
| // Bulk-emit llvm.mlir.global when lowerConstArrayAttr can build the |
| // whole initializer as one aggregate attribute (no insertvalue |
| // region). Leaf type must match what lowerConstArrayAttr handles |
| // (pointers, integers, bools, floats, and string literals with |
| // trailing_zeros). |
| if (isBulkLowerableConstArrayBaseElement( |
| getConstArrayBaseElementType(constArr.getType()))) { |
| mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>(); |
| if (std::optional<mlir::Attribute> bulkInit = lowerConstArrayAttr( |
| constArr, symbolTables, typeConverter, modOp)) { |
| mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>( |
| op, llvmType, isConst, linkage, symbol, bulkInit.value(), |
| alignment, addrSpace, isDsoLocal, threadLocalMode, comdatAttr, |
| attributes); |
| return mlir::success(); |
| } |
| } |
| return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter); |
| } else if (auto constRecord = |
| mlir::dyn_cast<cir::ConstRecordAttr>(init.value())) { |
| // Bulk-emit llvm.mlir.global when every member of the record can be |
| // lowered to a constant attribute. The LLVM dialect global translation |
| // turns an ArrayAttr (one element per struct field) into an |
| // llvm::ConstantStruct, so the whole initializer becomes a single |
| // attribute on the global instead of an insertvalue region. |
| mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>(); |
| if (std::optional<mlir::Attribute> bulkInit = lowerConstRecordAttr( |
| constRecord, symbolTables, typeConverter, modOp)) { |
| mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>( |
| op, llvmType, isConst, linkage, symbol, bulkInit.value(), alignment, |
| addrSpace, isDsoLocal, threadLocalMode, comdatAttr, attributes); |
| return mlir::success(); |
| } |
| return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter); |
| } else if (mlir::isa<cir::BlockAddrDiffAttr, cir::BlockAddrInfoAttr, |
| cir::ConstVectorAttr, cir::ConstRecordAttr, |
| cir::ConstPtrAttr, cir::ConstComplexAttr, |
| cir::GlobalViewAttr, cir::TypeInfoAttr, cir::UndefAttr, |
| cir::PoisonAttr, cir::VTableAttr, cir::ZeroAttr>( |
| init.value())) { |
| // TODO(cir): once LLVM's dialect has proper equivalent attributes this |
| // should be updated. For now, we use a custom op to initialize globals |
| // to the appropriate value. |
| return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter); |
| } else { |
| // We will only get here if new initializer types are added and this |
| // code is not updated to handle them. |
| op.emitError() << "unsupported initializer '" << init.value() << "'"; |
| return mlir::failure(); |
| } |
| } |
| |
| mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>( |
| op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()), |
| alignment, addrSpace, isDsoLocal, threadLocalMode, comdatAttr, |
| attributes); |
| |
| return mlir::success(); |
| } |
| |
| mlir::SymbolRefAttr |
| CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op, |
| mlir::OpBuilder &builder) const { |
| if (!op.getComdat()) |
| return mlir::SymbolRefAttr{}; |
| |
| mlir::ModuleOp modOp = op->getParentOfType<mlir::ModuleOp>(); |
| mlir::OpBuilder::InsertionGuard guard(builder); |
| StringRef comdatName("__llvm_comdat_globals"); |
| if (!comdatOp) { |
| builder.setInsertionPointToStart(modOp.getBody()); |
| comdatOp = |
| mlir::LLVM::ComdatOp::create(builder, modOp.getLoc(), comdatName); |
| } |
| |
| if (auto comdatSelector = comdatOp.lookupSymbol<mlir::LLVM::ComdatSelectorOp>( |
| op.getSymName())) { |
| return mlir::SymbolRefAttr::get( |
| builder.getContext(), comdatName, |
| mlir::FlatSymbolRefAttr::get(comdatSelector.getSymNameAttr())); |
| } |
| |
| builder.setInsertionPointToStart(&comdatOp.getBody().back()); |
| auto selectorOp = mlir::LLVM::ComdatSelectorOp::create( |
| builder, comdatOp.getLoc(), op.getSymName(), |
| mlir::LLVM::comdat::Comdat::Any); |
| return mlir::SymbolRefAttr::get( |
| builder.getContext(), comdatName, |
| mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr())); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite( |
| cir::SwitchFlatOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| llvm::SmallVector<mlir::APInt, 8> caseValues; |
| for (mlir::Attribute val : op.getCaseValues()) { |
| auto intAttr = cast<cir::IntAttr>(val); |
| caseValues.push_back(intAttr.getValue()); |
| } |
| |
| llvm::SmallVector<mlir::Block *, 8> caseDestinations; |
| llvm::SmallVector<mlir::ValueRange, 8> caseOperands; |
| |
| for (mlir::Block *x : op.getCaseDestinations()) |
| caseDestinations.push_back(x); |
| |
| for (mlir::OperandRange x : op.getCaseOperands()) |
| caseOperands.push_back(x); |
| |
| // Set switch op to branch to the newly created blocks. |
| rewriter.setInsertionPoint(op); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>( |
| op, adaptor.getCondition(), op.getDefaultDestination(), |
| op.getDefaultOperands(), caseValues, caseDestinations, caseOperands); |
| return mlir::success(); |
| } |
| |
| static mlir::LLVM::IntegerOverflowFlags nswFlag(bool nsw) { |
| return nsw ? mlir::LLVM::IntegerOverflowFlags::nsw |
| : mlir::LLVM::IntegerOverflowFlags::none; |
| } |
| |
| template <typename CIROp, typename LLVMIntOp> |
| static mlir::LogicalResult |
| lowerIncDecOp(CIROp op, typename CIROp::Adaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) { |
| mlir::Type llvmType = adaptor.getInput().getType(); |
| mlir::Location loc = op.getLoc(); |
| |
| auto maybeNSW = nswFlag(op.getNoSignedWrap()); |
| mlir::LLVM::ConstantOp one; |
| if (mlir::isa<cir::VectorType>(op.getType())) { |
| mlir::DenseIntElementsAttr oneVec = mlir::DenseIntElementsAttr::get( |
| mlir::cast<mlir::ShapedType>(llvmType), 1); |
| one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, oneVec); |
| } else { |
| one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1); |
| } |
| rewriter.replaceOpWithNewOp<LLVMIntOp>(op, adaptor.getInput(), one, maybeNSW); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMIncOpLowering::matchAndRewrite( |
| cir::IncOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerIncDecOp<cir::IncOp, mlir::LLVM::AddOp>(op, adaptor, rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMDecOpLowering::matchAndRewrite( |
| cir::DecOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerIncDecOp<cir::DecOp, mlir::LLVM::SubOp>(op, adaptor, rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMinusOpLowering::matchAndRewrite( |
| cir::MinusOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| bool isVector = mlir::isa<cir::VectorType>(op.getType()); |
| mlir::Type llvmType = adaptor.getInput().getType(); |
| mlir::Location loc = op.getLoc(); |
| |
| auto maybeNSW = nswFlag(op.getNoSignedWrap()); |
| mlir::Value zero; |
| if (isVector) |
| zero = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmType); |
| else |
| zero = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 0); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, zero, adaptor.getInput(), |
| maybeNSW); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMNotOpLowering::matchAndRewrite( |
| cir::NotOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type elementType = elementTypeIfVector(op.getType()); |
| bool isVector = mlir::isa<cir::VectorType>(op.getType()); |
| mlir::Type llvmType = adaptor.getInput().getType(); |
| mlir::Location loc = op.getLoc(); |
| |
| if (mlir::isa<cir::IntType>(elementType)) { |
| mlir::Value minusOne; |
| if (isVector) { |
| const uint64_t numElements = |
| mlir::dyn_cast<cir::VectorType>(op.getType()).getSize(); |
| SmallVector<int32_t> values(numElements, -1); |
| mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values); |
| minusOne = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, denseVec); |
| } else { |
| minusOne = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, -1); |
| } |
| rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), |
| minusOne); |
| return mlir::success(); |
| } |
| if (mlir::isa<cir::BoolType>(elementType)) { |
| auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), one); |
| return mlir::success(); |
| } |
| return op.emitError() << "Unsupported type for bitwise NOT"; |
| } |
| |
| static bool isIntTypeUnsigned(mlir::Type type) { |
| // TODO: Ideally, we should only need to check cir::IntType here. |
| return mlir::isa<cir::IntType>(type) |
| ? mlir::cast<cir::IntType>(type).isUnsigned() |
| : mlir::cast<mlir::IntegerType>(type).isUnsigned(); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Binary Op Lowering |
| //===----------------------------------------------------------------------===// |
| |
| template <typename BinOp> |
| static mlir::LLVM::IntegerOverflowFlags intOverflowFlag(BinOp op) { |
| if (op.getNoUnsignedWrap()) |
| return mlir::LLVM::IntegerOverflowFlags::nuw; |
| if (op.getNoSignedWrap()) |
| return mlir::LLVM::IntegerOverflowFlags::nsw; |
| return mlir::LLVM::IntegerOverflowFlags::none; |
| } |
| |
| /// Lower an arithmetic op that supports saturation, overflow flags, and an FP |
| /// Lower an integer Add/Sub op that may use saturating-arithmetic semantics. |
| template <typename UIntSatOp, typename SIntSatOp, typename IntOp, |
| typename CIROp> |
| static mlir::LogicalResult |
| lowerSaturatableArithOp(CIROp op, mlir::Value lhs, mlir::Value rhs, |
| mlir::ConversionPatternRewriter &rewriter) { |
| const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType()); |
| assert(cir::isIntOrBoolType(eltType) && |
| "saturatable arith op expects integer operand types"); |
| if (op.getSaturated()) { |
| if (isIntTypeUnsigned(eltType)) |
| rewriter.replaceOpWithNewOp<UIntSatOp>(op, lhs, rhs); |
| else |
| rewriter.replaceOpWithNewOp<SIntSatOp>(op, lhs, rhs); |
| return mlir::success(); |
| } |
| rewriter.replaceOpWithNewOp<IntOp>(op, lhs, rhs, intOverflowFlag(op)); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAddOpLowering::matchAndRewrite( |
| cir::AddOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerSaturatableArithOp<mlir::LLVM::UAddSat, mlir::LLVM::SAddSat, |
| mlir::LLVM::AddOp>(op, adaptor.getLhs(), |
| adaptor.getRhs(), rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSubOpLowering::matchAndRewrite( |
| cir::SubOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerSaturatableArithOp<mlir::LLVM::USubSat, mlir::LLVM::SSubSat, |
| mlir::LLVM::SubOp>(op, adaptor.getLhs(), |
| adaptor.getRhs(), rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMulOpLowering::matchAndRewrite( |
| cir::MulOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(cir::isIntOrBoolType(elementTypeIfVector(op.getRhs().getType())) && |
| "cir.mul expects integer operand types"); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>( |
| op, adaptor.getLhs(), adaptor.getRhs(), intOverflowFlag(op)); |
| return mlir::success(); |
| } |
| |
| /// Lower an integer Div/Rem op to its signed or unsigned LLVM counterpart. |
| template <typename UIntOp, typename SIntOp, typename CIROp> |
| static mlir::LogicalResult |
| lowerIntBinaryOp(CIROp op, mlir::Value lhs, mlir::Value rhs, |
| mlir::ConversionPatternRewriter &rewriter) { |
| const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType()); |
| assert(cir::isIntOrBoolType(eltType) && |
| "integer binary op expects integer operand types"); |
| if (isIntTypeUnsigned(eltType)) |
| rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs); |
| else |
| rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMDivOpLowering::matchAndRewrite( |
| cir::DivOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerIntBinaryOp<mlir::LLVM::UDivOp, mlir::LLVM::SDivOp>( |
| op, adaptor.getLhs(), adaptor.getRhs(), rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMRemOpLowering::matchAndRewrite( |
| cir::RemOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerIntBinaryOp<mlir::LLVM::URemOp, mlir::LLVM::SRemOp>( |
| op, adaptor.getLhs(), adaptor.getRhs(), rewriter); |
| } |
| |
| template <typename CIROp, typename UIntOp, typename SIntOp> |
| static mlir::LogicalResult |
| lowerMinMaxOp(CIROp op, typename CIROp::Adaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) { |
| const mlir::Value lhs = adaptor.getLhs(); |
| const mlir::Value rhs = adaptor.getRhs(); |
| if (isIntTypeUnsigned(elementTypeIfVector(op.getRhs().getType()))) |
| rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs); |
| else |
| rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMaxOpLowering::matchAndRewrite( |
| cir::MaxOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerMinMaxOp<cir::MaxOp, mlir::LLVM::UMaxOp, mlir::LLVM::SMaxOp>( |
| op, adaptor, rewriter); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMinOpLowering::matchAndRewrite( |
| cir::MinOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerMinMaxOp<cir::MinOp, mlir::LLVM::UMinOp, mlir::LLVM::SMinOp>( |
| op, adaptor, rewriter); |
| } |
| |
| /// Convert from a CIR comparison kind to an LLVM IR integral comparison kind. |
| static mlir::LLVM::ICmpPredicate |
| convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) { |
| using CIR = cir::CmpOpKind; |
| using LLVMICmp = mlir::LLVM::ICmpPredicate; |
| switch (kind) { |
| case CIR::eq: |
| return LLVMICmp::eq; |
| case CIR::ne: |
| return LLVMICmp::ne; |
| case CIR::lt: |
| return (isSigned ? LLVMICmp::slt : LLVMICmp::ult); |
| case CIR::le: |
| return (isSigned ? LLVMICmp::sle : LLVMICmp::ule); |
| case CIR::gt: |
| return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt); |
| case CIR::ge: |
| return (isSigned ? LLVMICmp::sge : LLVMICmp::uge); |
| case CIR::one: |
| case CIR::uno: |
| llvm_unreachable("FP-only comparison used with integer type"); |
| } |
| llvm_unreachable("Unknown CmpOpKind"); |
| } |
| |
| /// Convert from a CIR comparison kind to an LLVM IR floating-point comparison |
| /// kind. |
| static mlir::LLVM::FCmpPredicate |
| convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) { |
| using CIR = cir::CmpOpKind; |
| using LLVMFCmp = mlir::LLVM::FCmpPredicate; |
| switch (kind) { |
| case CIR::eq: |
| return LLVMFCmp::oeq; |
| case CIR::ne: |
| return LLVMFCmp::une; |
| case CIR::lt: |
| return LLVMFCmp::olt; |
| case CIR::le: |
| return LLVMFCmp::ole; |
| case CIR::gt: |
| return LLVMFCmp::ogt; |
| case CIR::ge: |
| return LLVMFCmp::oge; |
| case CIR::one: |
| return LLVMFCmp::one; |
| case CIR::uno: |
| return LLVMFCmp::uno; |
| } |
| llvm_unreachable("Unknown CmpOpKind"); |
| } |
| |
| static llvm::StringRef |
| convertCmpKindToConstrainedFCmpPredicate(cir::CmpOpKind kind) { |
| using CIR = cir::CmpOpKind; |
| switch (kind) { |
| case CIR::eq: |
| return "oeq"; |
| case CIR::ne: |
| return "une"; |
| case CIR::lt: |
| return "olt"; |
| case CIR::le: |
| return "ole"; |
| case CIR::gt: |
| return "ogt"; |
| case CIR::ge: |
| return "oge"; |
| case CIR::one: |
| return "one"; |
| case CIR::uno: |
| return "uno"; |
| } |
| llvm_unreachable("Unknown CmpOpKind"); |
| } |
| |
| static bool isSignalingConstrainedFCmp(cir::CmpOpKind kind) { |
| using CIR = cir::CmpOpKind; |
| switch (kind) { |
| case CIR::lt: |
| case CIR::le: |
| case CIR::gt: |
| case CIR::ge: |
| return true; |
| case CIR::eq: |
| case CIR::ne: |
| case CIR::one: |
| case CIR::uno: |
| return false; |
| } |
| llvm_unreachable("Unknown CmpOpKind"); |
| } |
| |
| static mlir::LLVM::CallIntrinsicOp |
| createConstrainedFCmpCall(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Location loc, mlir::Value lhs, mlir::Value rhs, |
| cir::CmpOpKind kind, cir::FenvAttr fenv, |
| mlir::Type llvmResTy) { |
| llvm::SmallVector<mlir::Value, 4> callOperands = { |
| lhs, rhs, |
| createFenvMetadataValue(rewriter, loc, |
| convertCmpKindToConstrainedFCmpPredicate(kind)), |
| createFenvMetadataValue(rewriter, loc, |
| getConstrainedExceptMetadata(fenv))}; |
| llvm::StringRef intrinsicName = isSignalingConstrainedFCmp(kind) |
| ? "llvm.experimental.constrained.fcmps" |
| : "llvm.experimental.constrained.fcmp"; |
| return createCallLLVMIntrinsicOp(rewriter, loc, intrinsicName, llvmResTy, |
| callOperands); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite( |
| cir::CmpOp cmpOp, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type type = cmpOp.getLhs().getType(); |
| |
| if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) { |
| bool isSigned = mlir::isa<cir::IntType>(type) |
| ? mlir::cast<cir::IntType>(type).isSigned() |
| : mlir::cast<mlir::IntegerType>(type).isSigned(); |
| mlir::LLVM::ICmpPredicate kind = |
| convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( |
| cmpOp, kind, adaptor.getLhs(), adaptor.getRhs()); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<cir::BoolType, cir::PointerType, cir::VPtrType>(type)) { |
| // Booleans, including enums with a boolean underlying type, compare as |
| // unsigned integers, as do pointers and !cir.vptr, which lowers to a |
| // pointer. |
| mlir::LLVM::ICmpPredicate kind = |
| convertCmpKindToICmpPredicate(cmpOp.getKind(), /*isSigned=*/false); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>( |
| cmpOp, kind, adaptor.getLhs(), adaptor.getRhs()); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<cir::FPTypeInterface>(type)) { |
| mlir::Type llvmResTy = getTypeConverter()->convertType(cmpOp.getType()); |
| if (cir::FenvAttr fenv = cmpOp.getFenvAttr()) { |
| mlir::LLVM::CallIntrinsicOp call = createConstrainedFCmpCall( |
| rewriter, cmpOp.getLoc(), adaptor.getLhs(), adaptor.getRhs(), |
| cmpOp.getKind(), fenv, llvmResTy); |
| rewriter.replaceOp(cmpOp, call.getResult(0)); |
| return mlir::success(); |
| } |
| mlir::LLVM::FCmpPredicate kind = |
| convertCmpKindToFCmpPredicate(cmpOp.getKind()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>( |
| cmpOp, kind, adaptor.getLhs(), adaptor.getRhs()); |
| return mlir::success(); |
| } |
| |
| if (mlir::isa<cir::ComplexType>(type)) { |
| mlir::Value lhs = adaptor.getLhs(); |
| mlir::Value rhs = adaptor.getRhs(); |
| mlir::Location loc = cmpOp.getLoc(); |
| |
| auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType()); |
| mlir::Type complexElemTy = |
| getTypeConverter()->convertType(complexType.getElementType()); |
| |
| auto lhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); |
| auto lhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); |
| auto rhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); |
| auto rhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); |
| |
| if (cmpOp.getKind() == cir::CmpOpKind::eq) { |
| if (complexElemTy.isInteger()) { |
| auto realCmp = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal); |
| auto imagCmp = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp); |
| return mlir::success(); |
| } |
| |
| auto realCmp = mlir::LLVM::FCmpOp::create( |
| rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal); |
| auto imagCmp = mlir::LLVM::FCmpOp::create( |
| rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp); |
| return mlir::success(); |
| } |
| |
| if (cmpOp.getKind() == cir::CmpOpKind::ne) { |
| if (complexElemTy.isInteger()) { |
| auto realCmp = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal); |
| auto imagCmp = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp); |
| return mlir::success(); |
| } |
| |
| auto realCmp = mlir::LLVM::FCmpOp::create( |
| rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal); |
| auto imagCmp = mlir::LLVM::FCmpOp::create( |
| rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp); |
| return mlir::success(); |
| } |
| } |
| |
| return cmpOp.emitError() << "unsupported type for CmpOp: " << type; |
| } |
| |
| /// Shared lowering logic for checked binary arithmetic overflow operations. |
| /// The \p opStr parameter specifies the arithmetic operation name used in the |
| /// LLVM intrinsic (e.g., "add", "sub", "mul"). |
| template <typename OpTy> |
| static mlir::LogicalResult |
| lowerBinOpOverflow(OpTy op, typename OpTy::Adaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter, |
| const mlir::TypeConverter *typeConverter, |
| llvm::StringRef opStr) { |
| mlir::Location loc = op.getLoc(); |
| cir::IntType operandTy = op.getLhs().getType(); |
| // The result type may be a `cir.bool`, which behaves as a 1-bit unsigned |
| // integer for the purposes of the checked arithmetic. |
| mlir::Type resultTy = op.getResult().getType(); |
| auto resultIntTy = mlir::dyn_cast<cir::IntType>(resultTy); |
| unsigned resultWidth = resultIntTy ? resultIntTy.getWidth() : 1; |
| bool resultSigned = resultIntTy && resultIntTy.getIsSigned(); |
| |
| bool sign = operandTy.getIsSigned() || resultSigned; |
| unsigned width = |
| std::max(operandTy.getWidth() + (sign && operandTy.isUnsigned()), |
| resultWidth + (sign && !resultSigned)); |
| |
| mlir::IntegerType encompassedLLVMTy = rewriter.getIntegerType(width); |
| |
| mlir::Value lhs = adaptor.getLhs(); |
| mlir::Value rhs = adaptor.getRhs(); |
| if (operandTy.getWidth() < width) { |
| if (operandTy.isSigned()) { |
| lhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, lhs); |
| rhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, rhs); |
| } else { |
| lhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, lhs); |
| rhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, rhs); |
| } |
| } |
| |
| // The intrinsic name is `@llvm.{s|u}{op}.with.overflow.i{width}` |
| std::string intrinName = ("llvm." + llvm::Twine(sign ? 's' : 'u') + opStr + |
| ".with.overflow.i" + llvm::Twine(width)) |
| .str(); |
| auto intrinNameAttr = mlir::StringAttr::get(op.getContext(), intrinName); |
| |
| mlir::IntegerType overflowLLVMTy = rewriter.getI1Type(); |
| auto intrinRetTy = mlir::LLVM::LLVMStructType::getLiteral( |
| rewriter.getContext(), {encompassedLLVMTy, overflowLLVMTy}); |
| |
| auto callLLVMIntrinOp = mlir::LLVM::CallIntrinsicOp::create( |
| rewriter, loc, intrinRetTy, intrinNameAttr, mlir::ValueRange{lhs, rhs}); |
| mlir::Value intrinRet = callLLVMIntrinOp.getResult(0); |
| |
| mlir::Value result = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, intrinRet, ArrayRef<int64_t>{0}) |
| .getResult(); |
| mlir::Value overflow = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, intrinRet, ArrayRef<int64_t>{1}) |
| .getResult(); |
| |
| if (resultWidth < width) { |
| mlir::Type resultLLVMTy = typeConverter->convertType(resultTy); |
| auto truncResult = |
| mlir::LLVM::TruncOp::create(rewriter, loc, resultLLVMTy, result); |
| |
| // Extend the truncated result back to the encompassing type to check for |
| // any overflows during the truncation. |
| mlir::Value truncResultExt; |
| if (resultSigned) |
| truncResultExt = mlir::LLVM::SExtOp::create( |
| rewriter, loc, encompassedLLVMTy, truncResult); |
| else |
| truncResultExt = mlir::LLVM::ZExtOp::create( |
| rewriter, loc, encompassedLLVMTy, truncResult); |
| auto truncOverflow = mlir::LLVM::ICmpOp::create( |
| rewriter, loc, mlir::LLVM::ICmpPredicate::ne, truncResultExt, result); |
| |
| result = truncResult; |
| overflow = mlir::LLVM::OrOp::create(rewriter, loc, overflow, truncOverflow); |
| } |
| |
| mlir::Type boolLLVMTy = |
| typeConverter->convertType(op.getOverflow().getType()); |
| if (boolLLVMTy != rewriter.getI1Type()) |
| overflow = mlir::LLVM::ZExtOp::create(rewriter, loc, boolLLVMTy, overflow); |
| |
| rewriter.replaceOp(op, mlir::ValueRange{result, overflow}); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAddOverflowOpLowering::matchAndRewrite( |
| cir::AddOverflowOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "add"); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSubOverflowOpLowering::matchAndRewrite( |
| cir::SubOverflowOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "sub"); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMulOverflowOpLowering::matchAndRewrite( |
| cir::MulOverflowOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "mul"); |
| } |
| |
| mlir::LogicalResult CIRToLLVMFrexpOpLowering::matchAndRewrite( |
| cir::FrexpOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Location loc = op.getLoc(); |
| mlir::Type fpLLVMTy = |
| getTypeConverter()->convertType(op.getResult().getType()); |
| mlir::Type intLLVMTy = getTypeConverter()->convertType(op.getExp().getType()); |
| |
| auto structTy = mlir::LLVM::LLVMStructType::getLiteral(rewriter.getContext(), |
| {fpLLVMTy, intLLVMTy}); |
| |
| auto callOp = createCallLLVMIntrinsicOp(rewriter, loc, "llvm.frexp", structTy, |
| adaptor.getSrc()); |
| mlir::Value result = callOp.getResult(0); |
| |
| mlir::Value mantissa = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, result, 0); |
| mlir::Value exponent = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, result, 1); |
| rewriter.replaceOp(op, mlir::ValueRange{mantissa, exponent}); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMModfOpLowering::matchAndRewrite( |
| cir::ModfOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Location loc = op.getLoc(); |
| mlir::Type fpLLVMTy = |
| getTypeConverter()->convertType(op.getFractional().getType()); |
| |
| auto structTy = mlir::LLVM::LLVMStructType::getLiteral(rewriter.getContext(), |
| {fpLLVMTy, fpLLVMTy}); |
| |
| auto callOp = createCallLLVMIntrinsicOp(rewriter, loc, "llvm.modf", structTy, |
| adaptor.getSrc()); |
| mlir::Value result = callOp.getResult(0); |
| |
| mlir::Value fractional = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, result, 0); |
| mlir::Value integral = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, result, 1); |
| rewriter.replaceOp(op, mlir::ValueRange{fractional, integral}); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite( |
| cir::ShiftOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert((op.getValue().getType() == op.getType()) && |
| "inconsistent operands' types NYI"); |
| |
| const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Value amt = adaptor.getAmount(); |
| mlir::Value val = adaptor.getValue(); |
| |
| auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType()); |
| bool isUnsigned; |
| if (cirAmtTy) { |
| auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType()); |
| isUnsigned = cirValTy.isUnsigned(); |
| |
| // Ensure shift amount is the same type as the value. Some undefined |
| // behavior might occur in the casts below as per [C99 6.5.7.3]. |
| // Vector type shift amount needs no cast as type consistency is expected to |
| // be already be enforced at CIRGen. |
| if (cirAmtTy) |
| amt = getLLVMIntCast(rewriter, amt, llvmTy, true, cirAmtTy.getWidth(), |
| cirValTy.getWidth()); |
| } else { |
| auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType()); |
| isUnsigned = |
| mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned(); |
| } |
| |
| // Lower to the proper LLVM shift operation. |
| if (op.getIsShiftleft()) { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt); |
| return mlir::success(); |
| } |
| |
| if (isUnsigned) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt); |
| else |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite( |
| cir::SelectOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr { |
| auto definingOp = value.getDefiningOp<cir::ConstantOp>(); |
| if (!definingOp) |
| return {}; |
| |
| auto constValue = definingOp.getValueAttr<cir::BoolAttr>(); |
| if (!constValue) |
| return {}; |
| |
| return constValue; |
| }; |
| |
| // Two special cases in the LLVMIR codegen of select op: |
| // - select %0, %1, false => and %0, %1 |
| // - select %0, true, %1 => or %0, %1 |
| if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) { |
| cir::BoolAttr trueValue = getConstantBool(op.getTrueValue()); |
| cir::BoolAttr falseValue = getConstantBool(op.getFalseValue()); |
| if (falseValue && !falseValue.getValue()) { |
| // select %0, %1, false => and %0, %1 |
| rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(), |
| adaptor.getTrueValue()); |
| return mlir::success(); |
| } |
| if (trueValue && trueValue.getValue()) { |
| // select %0, true, %1 => or %0, %1 |
| rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(), |
| adaptor.getFalseValue()); |
| return mlir::success(); |
| } |
| } |
| |
| mlir::Value llvmCondition = adaptor.getCondition(); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>( |
| op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue()); |
| |
| return mlir::success(); |
| } |
| |
| static void prepareTypeConverter(mlir::LLVMTypeConverter &converter, |
| mlir::DataLayout &dataLayout) { |
| converter.addConversion([&](cir::PointerType type) -> mlir::Type { |
| mlir::ptr::MemorySpaceAttrInterface addrSpaceAttr = type.getAddrSpace(); |
| unsigned numericAS = 0; |
| |
| if (auto targetAsAttr = |
| mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>( |
| addrSpaceAttr)) |
| numericAS = targetAsAttr.getValue(); |
| return mlir::LLVM::LLVMPointerType::get(type.getContext(), numericAS); |
| }); |
| converter.addConversion([&](cir::VPtrType type) -> mlir::Type { |
| assert(!cir::MissingFeatures::addressSpace()); |
| return mlir::LLVM::LLVMPointerType::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::ArrayType type) -> mlir::Type { |
| mlir::Type ty = |
| convertTypeForMemory(converter, dataLayout, type.getElementType()); |
| // A null element type means an unsupported member (e.g. a _BitInt with |
| // byte-array storage); propagate the conversion failure. |
| if (!ty) |
| return {}; |
| return mlir::LLVM::LLVMArrayType::get(ty, type.getSize()); |
| }); |
| converter.addConversion([&](cir::VectorType type) -> mlir::Type { |
| // Vector-of-_BitInt memory layout is not modeled here (the element would |
| // stay at its literal width); report a conversion failure rather than emit |
| // an unvalidated vector. |
| if (auto intTy = mlir::dyn_cast<cir::IntType>(type.getElementType()); |
| intTy && intTy.isBitInt()) |
| return {}; |
| const mlir::Type ty = converter.convertType(type.getElementType()); |
| return mlir::VectorType::get(type.getSize(), ty, {type.getIsScalable()}); |
| }); |
| converter.addConversion([&](cir::BoolType type) -> mlir::Type { |
| return mlir::IntegerType::get(type.getContext(), 1, |
| mlir::IntegerType::Signless); |
| }); |
| converter.addConversion([&](cir::IntType type) -> mlir::Type { |
| // LLVM doesn't work with signed types, so we drop the CIR signs here. |
| return mlir::IntegerType::get(type.getContext(), type.getWidth()); |
| }); |
| converter.addConversion([&](cir::SingleType type) -> mlir::Type { |
| return mlir::Float32Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::DoubleType type) -> mlir::Type { |
| return mlir::Float64Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::FP80Type type) -> mlir::Type { |
| return mlir::Float80Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::FP128Type type) -> mlir::Type { |
| return mlir::Float128Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type { |
| return converter.convertType(type.getUnderlying()); |
| }); |
| converter.addConversion([&](cir::FP16Type type) -> mlir::Type { |
| return mlir::Float16Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::BF16Type type) -> mlir::Type { |
| return mlir::BFloat16Type::get(type.getContext()); |
| }); |
| converter.addConversion([&](cir::ComplexType type) -> mlir::Type { |
| // A complex type is lowered to an LLVM struct that contains the real and |
| // imaginary part as data fields. |
| mlir::Type elementTy = converter.convertType(type.getElementType()); |
| mlir::Type structFields[2] = {elementTy, elementTy}; |
| return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(), |
| structFields); |
| }); |
| converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> { |
| auto result = converter.convertType(type.getReturnType()); |
| llvm::SmallVector<mlir::Type> arguments; |
| arguments.reserve(type.getNumInputs()); |
| if (converter.convertTypes(type.getInputs(), arguments).failed()) |
| return std::nullopt; |
| auto varArg = type.isVarArg(); |
| return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg); |
| }); |
| converter.addConversion([&](cir::StructType type) -> mlir::Type { |
| llvm::SmallVector<mlir::Type> llvmMembers; |
| for (mlir::Type ty : type.getMembers()) { |
| mlir::Type memberTy = convertTypeForMemory(converter, dataLayout, ty); |
| // A null member means an unsupported type (e.g. a _BitInt with byte-array |
| // storage); propagate the conversion failure instead of building an |
| // invalid struct body. |
| if (!memberTy) |
| return {}; |
| llvmMembers.push_back(memberTy); |
| } |
| |
| mlir::LLVM::LLVMStructType llvmStruct; |
| if (type.getName()) { |
| llvmStruct = mlir::LLVM::LLVMStructType::getIdentified( |
| type.getContext(), type.getPrefixedName()); |
| if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed()) |
| llvm_unreachable("Failed to set body of record"); |
| } else { |
| llvmStruct = mlir::LLVM::LLVMStructType::getLiteral( |
| type.getContext(), llvmMembers, type.getPacked()); |
| } |
| return llvmStruct; |
| }); |
| // Unions are lowered as only the largest member. |
| converter.addConversion([&](cir::UnionType type) -> mlir::Type { |
| llvm::SmallVector<mlir::Type> llvmMembers; |
| if (!type.getMembers().empty()) |
| if (auto storage = type.getUnionStorageType(dataLayout)) { |
| mlir::Type storageTy = |
| convertTypeForMemory(converter, dataLayout, storage); |
| if (!storageTy) |
| return {}; |
| llvmMembers.push_back(storageTy); |
| } |
| if (mlir::Type pad = type.getPadding()) { |
| mlir::Type padTy = convertTypeForMemory(converter, dataLayout, pad); |
| if (!padTy) |
| return {}; |
| llvmMembers.push_back(padTy); |
| } |
| |
| mlir::LLVM::LLVMStructType llvmStruct; |
| if (type.getName()) { |
| llvmStruct = mlir::LLVM::LLVMStructType::getIdentified( |
| type.getContext(), type.getPrefixedName()); |
| if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed()) |
| llvm_unreachable("Failed to set body of record"); |
| } else { |
| llvmStruct = mlir::LLVM::LLVMStructType::getLiteral( |
| type.getContext(), llvmMembers, type.getPacked()); |
| } |
| return llvmStruct; |
| }); |
| converter.addConversion([&](cir::VoidType type) -> mlir::Type { |
| return mlir::LLVM::LLVMVoidType::get(type.getContext()); |
| }); |
| } |
| |
| static void buildCtorDtorList( |
| mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName, |
| llvm::function_ref<std::pair<StringRef, int>(mlir::Attribute)> createXtor) { |
| llvm::SmallVector<std::pair<StringRef, int>> globalXtors; |
| for (const mlir::NamedAttribute namedAttr : module->getAttrs()) { |
| if (namedAttr.getName() == globalXtorName) { |
| for (auto attr : mlir::cast<mlir::ArrayAttr>(namedAttr.getValue())) |
| globalXtors.emplace_back(createXtor(attr)); |
| break; |
| } |
| } |
| |
| if (globalXtors.empty()) |
| return; |
| |
| mlir::OpBuilder builder(module.getContext()); |
| builder.setInsertionPointToEnd(&module.getBodyRegion().back()); |
| |
| // Create a global array llvm.global_ctors with element type of |
| // struct { i32, ptr, ptr } |
| auto ctorPFTy = mlir::LLVM::LLVMPointerType::get(builder.getContext()); |
| llvm::SmallVector<mlir::Type> ctorStructFields; |
| ctorStructFields.push_back(builder.getI32Type()); |
| ctorStructFields.push_back(ctorPFTy); |
| ctorStructFields.push_back(ctorPFTy); |
| |
| auto ctorStructTy = mlir::LLVM::LLVMStructType::getLiteral( |
| builder.getContext(), ctorStructFields); |
| auto ctorStructArrayTy = |
| mlir::LLVM::LLVMArrayType::get(ctorStructTy, globalXtors.size()); |
| |
| mlir::Location loc = module.getLoc(); |
| auto newGlobalOp = mlir::LLVM::GlobalOp::create( |
| builder, loc, ctorStructArrayTy, /*constant=*/false, |
| mlir::LLVM::Linkage::Appending, llvmXtorName, mlir::Attribute()); |
| |
| builder.createBlock(&newGlobalOp.getRegion()); |
| builder.setInsertionPointToEnd(newGlobalOp.getInitializerBlock()); |
| |
| mlir::Value result = |
| mlir::LLVM::UndefOp::create(builder, loc, ctorStructArrayTy); |
| |
| for (auto [index, fn] : llvm::enumerate(globalXtors)) { |
| mlir::Value structInit = |
| mlir::LLVM::UndefOp::create(builder, loc, ctorStructTy); |
| mlir::Value initPriority = mlir::LLVM::ConstantOp::create( |
| builder, loc, ctorStructFields[0], fn.second); |
| mlir::Value initFuncAddr = mlir::LLVM::AddressOfOp::create( |
| builder, loc, ctorStructFields[1], fn.first); |
| mlir::Value initAssociate = |
| mlir::LLVM::ZeroOp::create(builder, loc, ctorStructFields[2]); |
| // Literal zero makes the InsertValueOp::create ambiguous. |
| llvm::SmallVector<int64_t> zero{0}; |
| structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit, |
| initPriority, zero); |
| structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit, |
| initFuncAddr, 1); |
| // TODO: handle associated data for initializers. |
| structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit, |
| initAssociate, 2); |
| result = mlir::LLVM::InsertValueOp::create(builder, loc, result, structInit, |
| index); |
| } |
| |
| mlir::LLVM::ReturnOp::create(builder, loc, result); |
| } |
| |
| mlir::LogicalResult CIRToLLVMObjSizeOpLowering::matchAndRewrite( |
| cir::ObjSizeOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type llvmResTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Location loc = op->getLoc(); |
| |
| mlir::IntegerType i1Ty = rewriter.getI1Type(); |
| |
| auto i1Val = [&rewriter, &loc, &i1Ty](bool val) { |
| return mlir::LLVM::ConstantOp::create(rewriter, loc, i1Ty, val); |
| }; |
| |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.objectsize", llvmResTy, |
| { |
| adaptor.getPtr(), |
| i1Val(op.getMin()), |
| i1Val(op.getNullunknown()), |
| i1Val(op.getDynamic()), |
| }); |
| |
| return mlir::LogicalResult::success(); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // @llvm.global.annotations emission |
| //===----------------------------------------------------------------------===// |
| |
| namespace { |
| constexpr StringRef llvmMetadataSectionName = "llvm.metadata"; |
| |
| /// Get-or-create a private constant string global in the llvm.metadata |
| /// section, deduplicated by string content. |
| mlir::LLVM::GlobalOp |
| getOrCreateAnnotationStringGlobal(mlir::OpBuilder &builder, mlir::Location loc, |
| mlir::ModuleOp module, llvm::StringRef str, |
| llvm::StringMap<mlir::LLVM::GlobalOp> &cache, |
| bool isArg) { |
| auto it = cache.find(str); |
| if (it != cache.end()) |
| return it->second; |
| |
| auto i8Ty = mlir::IntegerType::get(module.getContext(), 8); |
| auto arrayTy = mlir::LLVM::LLVMArrayType::get(i8Ty, str.size() + 1); |
| std::string name = ".str"; |
| if (!cache.empty()) |
| name += "." + std::to_string(cache.size()); |
| name += ".annotation"; |
| if (isArg) |
| name += ".arg"; |
| |
| mlir::LLVM::GlobalOp strGlobal = mlir::LLVM::GlobalOp::create( |
| builder, loc, arrayTy, /*isConstant=*/true, mlir::LLVM::Linkage::Private, |
| name, mlir::StringAttr::get(module.getContext(), std::string(str) + '\0'), |
| /*alignment=*/isArg ? 1 : 0); |
| if (!isArg) |
| strGlobal.setSection(llvmMetadataSectionName); |
| strGlobal.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global); |
| strGlobal.setDsoLocal(true); |
| cache[str] = strGlobal; |
| return strGlobal; |
| } |
| |
| /// Get-or-create a private constant struct holding the annotation arguments, |
| /// deduplicated by ArrayAttr identity. |
| mlir::LLVM::GlobalOp getOrCreateAnnotationArgsVar( |
| mlir::OpBuilder &builder, mlir::Location loc, mlir::ModuleOp module, |
| mlir::ArrayAttr argsAttr, |
| llvm::StringMap<mlir::LLVM::GlobalOp> &argStringCache, |
| llvm::MapVector<mlir::ArrayAttr, mlir::LLVM::GlobalOp> &argsCache) { |
| auto it = argsCache.find(argsAttr); |
| if (it != argsCache.end()) |
| return it->second; |
| |
| auto ptrTy = mlir::LLVM::LLVMPointerType::get(builder.getContext()); |
| |
| llvm::SmallVector<mlir::Type> fieldTypes; |
| for (mlir::Attribute arg : argsAttr) { |
| if (mlir::isa<mlir::StringAttr>(arg)) |
| fieldTypes.push_back(ptrTy); |
| else if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(arg)) |
| fieldTypes.push_back(intAttr.getType()); |
| else |
| llvm_unreachable("Unsupported annotation arg type"); |
| } |
| |
| auto structTy = |
| mlir::LLVM::LLVMStructType::getLiteral(builder.getContext(), fieldTypes); |
| std::string name = ".args"; |
| if (!argsCache.empty()) |
| name += "." + std::to_string(argsCache.size()); |
| name += ".annotation"; |
| |
| mlir::LLVM::GlobalOp argsGlobal = mlir::LLVM::GlobalOp::create( |
| builder, loc, structTy, /*isConstant=*/true, mlir::LLVM::Linkage::Private, |
| name, mlir::Attribute()); |
| argsGlobal.setSection(llvmMetadataSectionName); |
| argsGlobal.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Global); |
| argsGlobal.setDsoLocal(true); |
| |
| // Build the initializer block. |
| argsGlobal.getRegion().push_back(new mlir::Block()); |
| mlir::OpBuilder initBuilder(module.getContext()); |
| initBuilder.setInsertionPointToEnd(argsGlobal.getInitializerBlock()); |
| |
| mlir::Value structInit = |
| mlir::LLVM::UndefOp::create(initBuilder, loc, structTy); |
| for (auto [idx, arg] : llvm::enumerate(argsAttr)) { |
| if (auto strArg = mlir::dyn_cast<mlir::StringAttr>(arg)) { |
| mlir::LLVM::GlobalOp strGlobal = getOrCreateAnnotationStringGlobal( |
| builder, loc, module, strArg.getValue(), argStringCache, |
| /*isArg=*/true); |
| mlir::LLVM::AddressOfOp strAddr = mlir::LLVM::AddressOfOp::create( |
| initBuilder, loc, ptrTy, strGlobal.getSymName()); |
| structInit = mlir::LLVM::InsertValueOp::create(initBuilder, loc, |
| structInit, strAddr, idx); |
| } else if (auto intArg = mlir::dyn_cast<mlir::IntegerAttr>(arg)) { |
| mlir::LLVM::ConstantOp intConst = mlir::LLVM::ConstantOp::create( |
| initBuilder, loc, intArg.getType(), intArg.getValue()); |
| structInit = mlir::LLVM::InsertValueOp::create(initBuilder, loc, |
| structInit, intConst, idx); |
| } else { |
| llvm_unreachable("Unsupported annotation arg type"); |
| } |
| } |
| mlir::LLVM::ReturnOp::create(initBuilder, loc, structInit); |
| |
| argsCache[argsAttr] = argsGlobal; |
| return argsGlobal; |
| } |
| |
| /// Resolve a possibly-fused MLIR Location to a FileLineColLoc, returning |
| /// {filename, line}. Returns {empty, 0} if no usable file location is found. |
| std::pair<llvm::StringRef, unsigned> extractFileLine(mlir::Location loc) { |
| mlir::Location resolved = loc; |
| if (auto fused = mlir::dyn_cast<mlir::FusedLoc>(resolved)) { |
| if (!fused.getLocations().empty()) |
| resolved = fused.getLocations()[0]; |
| } |
| if (auto fl = mlir::dyn_cast<mlir::FileLineColLoc>(resolved)) |
| return {fl.getFilename().getValue(), fl.getLine()}; |
| return {"", 0}; |
| } |
| } // namespace |
| |
| void ConvertCIRToLLVMPass::collectGlobalAnnotations(mlir::ModuleOp module) { |
| auto handleArray = [&](mlir::StringAttr symName, mlir::ArrayAttr arr, |
| mlir::Location loc) { |
| if (!arr) |
| return; |
| for (mlir::Attribute a : arr) |
| if (auto annot = mlir::dyn_cast<cir::AnnotationAttr>(a)) |
| collectedAnnotations.emplace_back(symName, annot, loc); |
| }; |
| |
| // Walk in IR order: GlobalOps first (they appear before functions in the |
| // module body), then FuncOps. This matches OGCG's emission order. |
| module.walk([&](cir::GlobalOp op) { |
| handleArray(op.getSymNameAttr(), op.getAnnotationsAttr(), op.getLoc()); |
| }); |
| module.walk([&](cir::FuncOp op) { |
| handleArray(op.getSymNameAttr(), op.getAnnotationsAttr(), op.getLoc()); |
| }); |
| } |
| |
| void ConvertCIRToLLVMPass::buildGlobalAnnotationsVar(mlir::ModuleOp module) { |
| if (collectedAnnotations.empty()) |
| return; |
| |
| mlir::MLIRContext *ctx = module.getContext(); |
| mlir::OpBuilder builder(ctx); |
| builder.setInsertionPointToEnd(&module.getBodyRegion().back()); |
| |
| auto ptrTy = mlir::LLVM::LLVMPointerType::get(ctx); |
| auto i32Ty = builder.getI32Type(); |
| |
| // Each entry: { ptr, ptr, ptr, i32, ptr }. |
| auto entryTy = mlir::LLVM::LLVMStructType::getLiteral( |
| ctx, {ptrTy, ptrTy, ptrTy, i32Ty, ptrTy}); |
| auto arrayTy = |
| mlir::LLVM::LLVMArrayType::get(entryTy, collectedAnnotations.size()); |
| |
| mlir::Location moduleLoc = module.getLoc(); |
| auto annotationsGlobal = mlir::LLVM::GlobalOp::create( |
| builder, moduleLoc, arrayTy, /*isConstant=*/false, |
| mlir::LLVM::Linkage::Appending, "llvm.global.annotations", |
| mlir::Attribute()); |
| annotationsGlobal.setSection(llvmMetadataSectionName); |
| |
| // Strings/args constants must come *before* @llvm.global.annotations to |
| // match OGCG output order. Insert them just before the annotations global. |
| mlir::OpBuilder constsBuilder(ctx); |
| constsBuilder.setInsertionPoint(annotationsGlobal); |
| |
| llvm::StringMap<mlir::LLVM::GlobalOp> stringCache; |
| llvm::StringMap<mlir::LLVM::GlobalOp> argStringCache; |
| llvm::MapVector<mlir::ArrayAttr, mlir::LLVM::GlobalOp> argsCache; |
| |
| // Build the initializer block of @llvm.global.annotations. |
| annotationsGlobal.getRegion().push_back(new mlir::Block()); |
| mlir::OpBuilder initBuilder(ctx); |
| initBuilder.setInsertionPointToEnd(annotationsGlobal.getInitializerBlock()); |
| |
| mlir::Value arrayVal = |
| mlir::LLVM::UndefOp::create(initBuilder, moduleLoc, arrayTy); |
| |
| for (auto [idx, entry] : llvm::enumerate(collectedAnnotations)) { |
| mlir::Value entryVal = |
| mlir::LLVM::UndefOp::create(initBuilder, moduleLoc, entryTy); |
| |
| // Field 0: ptr to the annotated symbol. (Literal zero is ambiguous on |
| // InsertValueOp::create, wrap in a SmallVector.) |
| llvm::SmallVector<int64_t> zero{0}; |
| mlir::LLVM::AddressOfOp symAddr = mlir::LLVM::AddressOfOp::create( |
| initBuilder, moduleLoc, ptrTy, entry.symName.getValue()); |
| entryVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| entryVal, symAddr, zero); |
| |
| // Field 1: ptr to the annotation name string. |
| mlir::LLVM::GlobalOp nameGlobal = getOrCreateAnnotationStringGlobal( |
| constsBuilder, moduleLoc, module, entry.annotation.getName().getValue(), |
| stringCache, /*isArg=*/false); |
| mlir::LLVM::AddressOfOp nameAddr = mlir::LLVM::AddressOfOp::create( |
| initBuilder, moduleLoc, ptrTy, nameGlobal.getSymName()); |
| entryVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| entryVal, nameAddr, 1); |
| |
| // Fields 2 and 3: ptr to filename string and line number. |
| auto [filename, line] = extractFileLine(entry.loc); |
| mlir::LLVM::GlobalOp fileGlobal = getOrCreateAnnotationStringGlobal( |
| constsBuilder, moduleLoc, module, filename, stringCache, |
| /*isArg=*/false); |
| mlir::LLVM::AddressOfOp fileAddr = mlir::LLVM::AddressOfOp::create( |
| initBuilder, moduleLoc, ptrTy, fileGlobal.getSymName()); |
| entryVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| entryVal, fileAddr, 2); |
| mlir::LLVM::ConstantOp lineConst = |
| mlir::LLVM::ConstantOp::create(initBuilder, moduleLoc, i32Ty, line); |
| entryVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| entryVal, lineConst, 3); |
| |
| // Field 4: ptr to args, or null if none. |
| mlir::ArrayAttr args = entry.annotation.getArgs(); |
| mlir::Value argsField; |
| if (!args || args.empty()) { |
| argsField = mlir::LLVM::ZeroOp::create(initBuilder, moduleLoc, ptrTy); |
| } else { |
| mlir::LLVM::GlobalOp argsGlobal = getOrCreateAnnotationArgsVar( |
| constsBuilder, moduleLoc, module, args, argStringCache, argsCache); |
| argsField = mlir::LLVM::AddressOfOp::create(initBuilder, moduleLoc, ptrTy, |
| argsGlobal.getSymName()); |
| } |
| entryVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| entryVal, argsField, 4); |
| |
| arrayVal = mlir::LLVM::InsertValueOp::create(initBuilder, moduleLoc, |
| arrayVal, entryVal, idx); |
| } |
| |
| mlir::LLVM::ReturnOp::create(initBuilder, moduleLoc, arrayVal); |
| } |
| |
| void ConvertCIRToLLVMPass::resolveBlockAddressOp( |
| LLVMBlockAddressInfo &blockInfoAddr) { |
| |
| mlir::ModuleOp module = getOperation(); |
| mlir::OpBuilder opBuilder(module.getContext()); |
| for (auto &[blockAddOp, blockInfo] : |
| blockInfoAddr.getUnresolvedBlockAddress()) { |
| mlir::LLVM::BlockTagOp resolvedLabel = |
| blockInfoAddr.lookupBlockTag(blockInfo); |
| assert(resolvedLabel && "expected BlockTagOp to already be emitted"); |
| mlir::FlatSymbolRefAttr fnSym = blockInfo.getFunc(); |
| auto blkAddTag = mlir::LLVM::BlockAddressAttr::get( |
| opBuilder.getContext(), fnSym, resolvedLabel.getTagAttr()); |
| blockAddOp.setBlockAddrAttr(blkAddTag); |
| } |
| blockInfoAddr.clearUnresolvedMap(); |
| } |
| |
| void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) { |
| // Lower the module attributes to LLVM equivalents. |
| if (mlir::Attribute tripleAttr = |
| module->getAttr(cir::CIRDialect::getTripleAttrName())) |
| module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(), |
| tripleAttr); |
| |
| if (mlir::Attribute asmAttr = |
| module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName())) |
| module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(), |
| asmAttr); |
| } |
| |
| void ConvertCIRToLLVMPass::runOnOperation() { |
| llvm::TimeTraceScope scope("Convert CIR to LLVM Pass"); |
| |
| mlir::ModuleOp module = getOperation(); |
| mlir::DataLayout dl(module); |
| mlir::LLVMTypeConverter converter(&getContext()); |
| prepareTypeConverter(converter, dl); |
| |
| /// Tracks the state required to lower CIR `LabelOp` and `BlockAddressOp`. |
| /// Maps labels to their corresponding `BlockTagOp` and keeps bookkeeping |
| /// of unresolved `BlockAddressOp`s until they are matched with the |
| /// corresponding `BlockTagOp` in `resolveBlockAddressOp`. |
| LLVMBlockAddressInfo blockInfoAddr; |
| /// Cached symbol table collection used by call lowering patterns to avoid |
| /// repeated O(M) module-wide symbol scans for every call site. |
| mlir::SymbolTableCollection symbolTables; |
| mlir::RewritePatternSet patterns(&getContext()); |
| patterns.add<CIRToLLVMBlockAddressOpLowering, CIRToLLVMGlobalOpLowering, |
| CIRToLLVMLabelOpLowering>(converter, patterns.getContext(), dl, |
| symbolTables, blockInfoAddr); |
| |
| patterns.add< |
| #define GET_LLVM_LOWERING_PATTERNS_LIST |
| #include "clang/CIR/Dialect/IR/CIRLowering.inc" |
| #undef GET_LLVM_LOWERING_PATTERNS_LIST |
| >(converter, patterns.getContext(), dl, symbolTables); |
| |
| processCIRAttrs(module); |
| |
| // Collect annotation info from cir.func / cir.global before conversion; |
| // the annotations attribute is filtered out during FuncOp/GlobalOp lowering. |
| collectGlobalAnnotations(module); |
| |
| mlir::ConversionTarget target(getContext()); |
| target.addLegalOp<mlir::ModuleOp>(); |
| target.addLegalDialect<mlir::LLVM::LLVMDialect>(); |
| mlir::configureOpenMPToLLVMConversionLegality(target, converter); |
| target.addLegalDialect<mlir::omp::OpenMPDialect>(); |
| mlir::populateOpenMPToLLVMConversionPatterns(converter, patterns); |
| target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect, |
| mlir::func::FuncDialect>(); |
| |
| llvm::SmallVector<mlir::Operation *> ops; |
| ops.push_back(module); |
| cir::collectUnreachable(module, ops); |
| |
| if (failed(applyPartialConversion(ops, target, std::move(patterns)))) |
| signalPassFailure(); |
| |
| // Drop the cir.ptr-keyed data-layout entries: they drove pointer-width |
| // queries up to this point, but the LLVM IR exporter rejects CIR types. |
| if (auto dlSpec = mlir::dyn_cast_or_null<mlir::DataLayoutSpecAttr>( |
| module->getAttr(mlir::DLTIDialect::kDataLayoutAttrName))) { |
| llvm::SmallVector<mlir::DataLayoutEntryInterface> kept; |
| for (mlir::DataLayoutEntryInterface entry : dlSpec.getEntries()) { |
| if (entry.isTypeEntry() && |
| mlir::isa<cir::PointerType>(mlir::cast<mlir::Type>(entry.getKey()))) |
| continue; |
| kept.push_back(entry); |
| } |
| module->setAttr(mlir::DLTIDialect::kDataLayoutAttrName, |
| mlir::DataLayoutSpecAttr::get(module.getContext(), kept)); |
| } |
| |
| // Emit the llvm.global_ctors array. |
| buildCtorDtorList(module, cir::CIRDialect::getGlobalCtorsAttrName(), |
| "llvm.global_ctors", [](mlir::Attribute attr) { |
| auto ctorAttr = mlir::cast<cir::GlobalCtorAttr>(attr); |
| return std::make_pair(ctorAttr.getName(), |
| ctorAttr.getPriority()); |
| }); |
| // Emit the llvm.global_dtors array. |
| buildCtorDtorList(module, cir::CIRDialect::getGlobalDtorsAttrName(), |
| "llvm.global_dtors", [](mlir::Attribute attr) { |
| auto dtorAttr = mlir::cast<cir::GlobalDtorAttr>(attr); |
| return std::make_pair(dtorAttr.getName(), |
| dtorAttr.getPriority()); |
| }); |
| // Emit @llvm.global.annotations from the previously-collected entries. |
| buildGlobalAnnotationsVar(module); |
| |
| resolveBlockAddressOp(blockInfoAddr); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite( |
| cir::BrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(), |
| op.getDest()); |
| return mlir::LogicalResult::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite( |
| cir::GetMemberOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type llResTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Type pointee = op.getAddrTy().getPointee(); |
| |
| if (mlir::isa<cir::UnionType>(pointee)) { |
| // Union members share the address space, so we just need a bitcast to |
| // conform to type-checking. |
| rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy, |
| adaptor.getAddr()); |
| return mlir::success(); |
| } |
| |
| auto structTy = mlir::cast<cir::StructType>(pointee); |
| // Since the base address is a pointer to an aggregate, the first offset |
| // is always zero. The second offset tells us which member it will access. |
| llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{0, op.getIndex()}; |
| const mlir::Type elementTy = getTypeConverter()->convertType(structTy); |
| // Struct member accesses are always inbounds and nuw: the base pointer |
| // is valid and the member offset is a positive, constant offset within |
| // the struct layout, so it cannot wrap. This matches LLVM's |
| // IRBuilder::CreateStructGEP. |
| mlir::LLVM::GEPNoWrapFlags flags = |
| mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| op, llResTy, elementTy, adaptor.getAddr(), offset, flags); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMExtractMemberOpLowering::matchAndRewrite( |
| cir::ExtractMemberOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| std::int64_t indices[1] = {static_cast<std::int64_t>(op.getIndex())}; |
| |
| if (mlir::isa<cir::UnionType>(op.getRecord().getType())) { |
| op.emitError("cir.extract_member cannot extract member from a union"); |
| return mlir::failure(); |
| } |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>( |
| op, adaptor.getRecord(), indices); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMInsertMemberOpLowering::matchAndRewrite( |
| cir::InsertMemberOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| std::int64_t indecies[1] = {static_cast<std::int64_t>(op.getIndex())}; |
| |
| if (mlir::isa<cir::UnionType>(op.getRecord().getType())) { |
| op.emitError("cir.update_member cannot update member of a union"); |
| return mlir::failure(); |
| } |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>( |
| op, adaptor.getRecord(), adaptor.getValue(), indecies); |
| return mlir::success(); |
| } |
| |
| void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter, |
| mlir::SymbolTableCollection &symbolTables, |
| mlir::Operation *srcOp, llvm::StringRef fnName, |
| mlir::Type fnTy, |
| mlir::ArrayAttr argAttrs = nullptr, |
| mlir::ArrayAttr resAttrs = nullptr) { |
| mlir::ModuleOp modOp = srcOp->getParentOfType<mlir::ModuleOp>(); |
| mlir::Operation *sourceSymbol = symbolTables.lookupSymbolIn( |
| modOp, mlir::StringAttr::get(fnTy.getContext(), fnName)); |
| if (!sourceSymbol) { |
| mlir::OpBuilder::InsertionGuard guard(rewriter); |
| auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>(); |
| rewriter.setInsertionPoint(enclosingFnOp); |
| auto fn = |
| mlir::LLVM::LLVMFuncOp::create(rewriter, srcOp->getLoc(), fnName, fnTy); |
| if (argAttrs) |
| fn.setArgAttrsAttr(argAttrs); |
| if (resAttrs) |
| fn.setResAttrsAttr(resAttrs); |
| // Update the symbol table. |
| symbolTables.getSymbolTable(fn->getParentOp()).insert(fn); |
| } |
| } |
| |
| mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite( |
| cir::ThrowOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Location loc = op.getLoc(); |
| auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext()); |
| |
| if (op.rethrows()) { |
| auto funcTy = mlir::LLVM::LLVMFunctionType::get(voidTy, {}); |
| |
| // Get or create `declare void @__cxa_rethrow()` |
| const llvm::StringRef functionName = "__cxa_rethrow"; |
| createLLVMFuncOpIfNotExist(rewriter, symbolTables, op, functionName, |
| funcTy); |
| |
| auto cxaRethrow = mlir::LLVM::CallOp::create( |
| rewriter, loc, mlir::TypeRange{}, functionName); |
| |
| rewriter.replaceOp(op, cxaRethrow); |
| return mlir::success(); |
| } |
| |
| auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| auto fnTy = mlir::LLVM::LLVMFunctionType::get( |
| voidTy, {llvmPtrTy, llvmPtrTy, llvmPtrTy}); |
| |
| // Get or create `declare void @__cxa_throw(ptr, ptr, ptr)` |
| const llvm::StringRef fnName = "__cxa_throw"; |
| createLLVMFuncOpIfNotExist(rewriter, symbolTables, op, fnName, fnTy); |
| |
| mlir::Value typeInfo = mlir::LLVM::AddressOfOp::create( |
| rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), |
| adaptor.getTypeInfoAttr()); |
| |
| mlir::Value dtor; |
| if (op.getDtor()) { |
| dtor = mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy, |
| adaptor.getDtorAttr()); |
| } else { |
| dtor = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy); |
| } |
| |
| auto cxaThrowCall = mlir::LLVM::CallOp::create( |
| rewriter, loc, mlir::TypeRange{}, fnName, |
| mlir::ValueRange{adaptor.getExceptionPtr(), typeInfo, dtor}); |
| |
| rewriter.replaceOp(op, cxaThrowCall); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMAllocExceptionOpLowering::matchAndRewrite( |
| cir::AllocExceptionOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Get or create `declare ptr @__cxa_allocate_exception(i64)` |
| StringRef fnName = "__cxa_allocate_exception"; |
| auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| auto int64Ty = mlir::IntegerType::get(rewriter.getContext(), 64); |
| auto fnTy = mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {int64Ty}); |
| |
| createLLVMFuncOpIfNotExist(rewriter, symbolTables, op, fnName, fnTy); |
| auto exceptionSize = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), |
| adaptor.getSizeAttr()); |
| |
| auto allocaExceptionCall = mlir::LLVM::CallOp::create( |
| rewriter, op.getLoc(), mlir::TypeRange{llvmPtrTy}, fnName, |
| mlir::ValueRange{exceptionSize}); |
| |
| rewriter.replaceOp(op, allocaExceptionCall); |
| return mlir::success(); |
| } |
| |
| static mlir::LLVM::LLVMStructType |
| getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter) { |
| // Create the landing pad type: struct { ptr, i32 } |
| mlir::MLIRContext *ctx = rewriter.getContext(); |
| auto llvmPtr = mlir::LLVM::LLVMPointerType::get(ctx); |
| llvm::SmallVector<mlir::Type> structFields = {llvmPtr, rewriter.getI32Type()}; |
| return mlir::LLVM::LLVMStructType::getLiteral(ctx, structFields); |
| } |
| |
| mlir::LogicalResult CIRToLLVMEhInflightOpLowering::matchAndRewrite( |
| cir::EhInflightOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto llvmFn = op->getParentOfType<mlir::LLVM::LLVMFuncOp>(); |
| assert(llvmFn && "expected LLVM function parent"); |
| mlir::Block *entryBlock = &llvmFn.getRegion().front(); |
| assert(entryBlock->isEntryBlock()); |
| |
| mlir::ArrayAttr catchListAttr = op.getCatchTypeListAttr(); |
| mlir::SmallVector<mlir::Value> catchSymAddrs; |
| |
| auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| mlir::Location loc = op.getLoc(); |
| |
| // %landingpad = landingpad { ptr, i32 } |
| // Note that since llvm.landingpad has to be the first operation on the |
| // block, any needed value for its operands has to be added somewhere else. |
| if (catchListAttr) { |
| // catch ptr @_ZTIi |
| // catch ptr @_ZTIPKc |
| for (mlir::Attribute catchAttr : catchListAttr) { |
| auto symAttr = cast<mlir::FlatSymbolRefAttr>(catchAttr); |
| // Generate `llvm.mlir.addressof` for each symbol, and place those |
| // operations in the LLVM function entry basic block. |
| mlir::OpBuilder::InsertionGuard guard(rewriter); |
| rewriter.setInsertionPointToStart(entryBlock); |
| mlir::Value addrOp = mlir::LLVM::AddressOfOp::create( |
| rewriter, loc, llvmPtrTy, symAttr.getValue()); |
| catchSymAddrs.push_back(addrOp); |
| } |
| } |
| |
| // Emit a catch-all clause (catch ptr null) when: |
| // - The catch_all attribute is set (typed catches + catch-all), or |
| // - No typed catches and no cleanup (legacy pure catch-all form) |
| if (op.getCatchAll() || (!catchListAttr && !op.getCleanup())) { |
| mlir::OpBuilder::InsertionGuard guard(rewriter); |
| rewriter.setInsertionPointToStart(entryBlock); |
| mlir::Value nullOp = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy); |
| catchSymAddrs.push_back(nullOp); |
| } |
| |
| // %slot = extractvalue { ptr, i32 } %x, 0 |
| // %selector = extractvalue { ptr, i32 } %x, 1 |
| mlir::LLVM::LLVMStructType llvmLandingPadStructTy = |
| getLLVMLandingPadStructTy(rewriter); |
| auto landingPadOp = mlir::LLVM::LandingpadOp::create( |
| rewriter, loc, llvmLandingPadStructTy, catchSymAddrs); |
| |
| // The LLVM cleanup flag is only needed when there is no catch-all handler, |
| // since catch-all (catch ptr null) already ensures the personality function |
| // enters the landing pad for all exception types. |
| if (op.getCleanup() && !op.getCatchAll()) |
| landingPadOp.setCleanup(true); |
| |
| mlir::Value slot = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 0); |
| mlir::Value selector = |
| mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 1); |
| rewriter.replaceOp(op, mlir::ValueRange{slot, selector}); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMResumeFlatOpLowering::matchAndRewrite( |
| cir::ResumeFlatOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // %lpad.val = insertvalue { ptr, i32 } poison, ptr %exception_ptr, 0 |
| // %lpad.val2 = insertvalue { ptr, i32 } %lpad.val, i32 %selector, 1 |
| // resume { ptr, i32 } %lpad.val2 |
| mlir::Type llvmLandingPadStructTy = getLLVMLandingPadStructTy(rewriter); |
| mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, op.getLoc(), |
| llvmLandingPadStructTy); |
| |
| SmallVector<int64_t> slotIdx = {0}; |
| mlir::Value slot = mlir::LLVM::InsertValueOp::create( |
| rewriter, op.getLoc(), poison, adaptor.getExceptionPtr(), slotIdx); |
| |
| SmallVector<int64_t> selectorIdx = {1}; |
| mlir::Value selector = mlir::LLVM::InsertValueOp::create( |
| rewriter, op.getLoc(), slot, adaptor.getTypeId(), selectorIdx); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ResumeOp>(op, selector); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMEhTypeIdOpLowering::matchAndRewrite( |
| cir::EhTypeIdOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value addrOp = mlir::LLVM::AddressOfOp::create( |
| rewriter, op.getLoc(), |
| mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), |
| op.getTypeSymAttr()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::EhTypeidForOp>( |
| op, rewriter.getI32Type(), addrOp); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMEhSetjmpOpLowering::matchAndRewrite( |
| cir::EhSetjmpOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type returnType = typeConverter->convertType(op.getType()); |
| mlir::LLVM::CallIntrinsicOp newOp = |
| createCallLLVMIntrinsicOp(rewriter, op.getLoc(), "llvm.eh.sjlj.setjmp", |
| returnType, adaptor.getEnv()); |
| rewriter.replaceOp(op, newOp); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMEhLongjmpOpLowering::matchAndRewrite( |
| cir::EhLongjmpOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.eh.sjlj.longjmp", |
| /*resultTy=*/{}, adaptor.getOperands()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite( |
| cir::TrapOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Location loc = op->getLoc(); |
| rewriter.eraseOp(op); |
| |
| mlir::LLVM::Trap::create(rewriter, loc); |
| |
| // Note that the call to llvm.trap is not a terminator in LLVM dialect. |
| // So we must emit an additional llvm.unreachable to terminate the current |
| // block. |
| mlir::LLVM::UnreachableOp::create(rewriter, loc); |
| |
| return mlir::success(); |
| } |
| |
| static mlir::Value |
| getValueForVTableSymbol(mlir::Operation *op, |
| mlir::ConversionPatternRewriter &rewriter, |
| mlir::SymbolTableCollection &symbolTables, |
| const mlir::TypeConverter *converter, |
| mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) { |
| auto module = op->getParentOfType<mlir::ModuleOp>(); |
| mlir::Operation *symbol = symbolTables.lookupSymbolIn(module, nameAttr); |
| if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) { |
| eltType = llvmSymbol.getType(); |
| } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) { |
| eltType = converter->convertType(cirSymbol.getSymType()); |
| } else { |
| op->emitError() << "unexpected symbol type for " << symbol; |
| return {}; |
| } |
| |
| return mlir::LLVM::AddressOfOp::create( |
| rewriter, op->getLoc(), |
| mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue()); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite( |
| cir::VTableAddrPointOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::TypeConverter *converter = getTypeConverter(); |
| mlir::Type targetType = converter->convertType(op.getType()); |
| llvm::SmallVector<mlir::LLVM::GEPArg> offsets; |
| mlir::Type eltType; |
| mlir::Value symAddr = getValueForVTableSymbol( |
| op, rewriter, symbolTables, converter, op.getNameAttr(), eltType); |
| if (!symAddr) |
| return op.emitError() << "Unable to get value for vtable symbol"; |
| |
| offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{ |
| 0, op.getAddressPointAttr().getIndex(), |
| op.getAddressPointAttr().getOffset()}; |
| |
| assert(eltType && "Shouldn't ever be missing an eltType here"); |
| mlir::LLVM::GEPNoWrapFlags inboundsNuw = |
| mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType, |
| symAddr, offsets, inboundsNuw); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite( |
| cir::VTableGetVPtrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // cir.vtable.get_vptr is equivalent to a bitcast from the source object |
| // pointer to the vptr type. Since the LLVM dialect uses opaque pointers |
| // we can just replace uses of this operation with the original pointer. |
| mlir::Value srcVal = adaptor.getSrc(); |
| rewriter.replaceOp(op, srcVal); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite( |
| cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type targetType = getTypeConverter()->convertType(op.getType()); |
| auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| llvm::SmallVector<mlir::LLVM::GEPArg> offsets = |
| llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()}; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| op, targetType, eltType, adaptor.getVptr(), offsets, |
| mlir::LLVM::GEPNoWrapFlags::inbounds); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite( |
| cir::VTTAddrPointOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| const mlir::Type resultType = getTypeConverter()->convertType(op.getType()); |
| llvm::SmallVector<mlir::LLVM::GEPArg> offsets; |
| mlir::Type eltType; |
| mlir::Value llvmAddr = adaptor.getSymAddr(); |
| |
| if (op.getSymAddr()) { |
| if (op.getOffset() == 0) { |
| rewriter.replaceOp(op, {llvmAddr}); |
| return mlir::success(); |
| } |
| |
| offsets.push_back(adaptor.getOffset()); |
| eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| } else { |
| llvmAddr = |
| getValueForVTableSymbol(op, rewriter, symbolTables, getTypeConverter(), |
| op.getNameAttr(), eltType); |
| assert(eltType && "Shouldn't ever be missing an eltType here"); |
| offsets.push_back(0); |
| offsets.push_back(adaptor.getOffset()); |
| } |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| op, resultType, eltType, llvmAddr, offsets, |
| mlir::LLVM::GEPNoWrapFlags::inbounds); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite( |
| cir::VecCreateOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Start with an 'undef' value for the vector. Then 'insertelement' for |
| // each of the vector elements. |
| const auto vecTy = mlir::cast<cir::VectorType>(op.getType()); |
| const mlir::Type llvmTy = typeConverter->convertType(vecTy); |
| const mlir::Location loc = op.getLoc(); |
| mlir::Value result = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy); |
| assert(vecTy.getSize() == op.getElements().size() && |
| "cir.vec.create op count doesn't match vector type elements count"); |
| |
| for (uint64_t i = 0; i < vecTy.getSize(); ++i) { |
| const mlir::Value indexValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); |
| result = mlir::LLVM::InsertElementOp::create( |
| rewriter, loc, result, adaptor.getElements()[i], indexValue); |
| } |
| |
| rewriter.replaceOp(op, result); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecExtractOpLowering::matchAndRewrite( |
| cir::VecExtractOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>( |
| op, adaptor.getVec(), adaptor.getIndex()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite( |
| cir::VecInsertOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>( |
| op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite( |
| cir::VecCmpOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type elementType = elementTypeIfVector(op.getLhs().getType()); |
| mlir::Value bitResult; |
| if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) { |
| bitResult = mlir::LLVM::ICmpOp::create( |
| rewriter, op.getLoc(), |
| convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()), |
| adaptor.getLhs(), adaptor.getRhs()); |
| } else if (mlir::isa<cir::FPTypeInterface>(elementType)) { |
| if (cir::FenvAttr fenv = op.getFenvAttr()) { |
| auto i1VecTy = mlir::VectorType::get( |
| mlir::cast<cir::VectorType>(op.getLhs().getType()).getSize(), |
| rewriter.getI1Type()); |
| bitResult = createConstrainedFCmpCall(rewriter, op.getLoc(), |
| adaptor.getLhs(), adaptor.getRhs(), |
| op.getKind(), fenv, i1VecTy) |
| .getResult(0); |
| } else { |
| bitResult = mlir::LLVM::FCmpOp::create( |
| rewriter, op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()), |
| adaptor.getLhs(), adaptor.getRhs()); |
| } |
| } else { |
| return op.emitError() << "unsupported type for VecCmpOp: " << elementType; |
| } |
| |
| // LLVM IR vector comparison returns a vector of i1. This one-bit vector |
| // must be sign-extended to the correct result type, unless a vector of i1 is |
| // the type we need. |
| if (cast<cir::IntType>(cast<cir::VectorType>(op.getType()).getElementType()) |
| .getWidth() > 1) |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>( |
| op, typeConverter->convertType(op.getType()), bitResult); |
| else |
| rewriter.replaceOp(op, bitResult); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite( |
| cir::VecSplatOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Vector splat can be implemented with an `insertelement` and a |
| // `shufflevector`, which is better than an `insertelement` for each |
| // element in the vector. Start with an undef vector. Insert the value into |
| // the first element. Then use a `shufflevector` with a mask of all 0 to |
| // fill out the entire vector with that value. |
| cir::VectorType vecTy = op.getType(); |
| mlir::Type llvmTy = typeConverter->convertType(vecTy); |
| mlir::Location loc = op.getLoc(); |
| mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy); |
| |
| mlir::Value elementValue = adaptor.getValue(); |
| if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) { |
| // If the splat value is poison, then we can just use poison value |
| // for the entire vector. |
| rewriter.replaceOp(op, poison); |
| return mlir::success(); |
| } |
| |
| if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) { |
| if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) { |
| mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get( |
| mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>( |
| op, denseVec.getType(), denseVec); |
| return mlir::success(); |
| } |
| |
| if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) { |
| mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get( |
| mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>( |
| op, denseVec.getType(), denseVec); |
| return mlir::success(); |
| } |
| } |
| |
| mlir::Value indexValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), 0); |
| mlir::Value oneElement = mlir::LLVM::InsertElementOp::create( |
| rewriter, loc, poison, elementValue, indexValue); |
| SmallVector<int32_t> zeroValues(vecTy.getSize(), 0); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement, |
| poison, zeroValues); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite( |
| cir::VecShuffleOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices. |
| // Convert the ClangIR ArrayAttr of IntAttr constants into a |
| // SmallVector<int>. |
| SmallVector<int, 8> indices; |
| std::transform( |
| op.getIndices().begin(), op.getIndices().end(), |
| std::back_inserter(indices), [](mlir::Attribute intAttr) { |
| return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue(); |
| }); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>( |
| op, adaptor.getVec1(), adaptor.getVec2(), indices); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite( |
| cir::VecShuffleDynamicOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // LLVM IR does not have an operation that corresponds to this form of |
| // the built-in. |
| // __builtin_shufflevector(V, I) |
| // is implemented as this pseudocode, where the for loop is unrolled |
| // and N is the number of elements: |
| // |
| // result = undef |
| // maskbits = NextPowerOf2(N - 1) |
| // masked = I & maskbits |
| // for (i in 0 <= i < N) |
| // result[i] = V[masked[i]] |
| mlir::Location loc = op.getLoc(); |
| mlir::Value input = adaptor.getVec(); |
| mlir::Type llvmIndexVecType = |
| getTypeConverter()->convertType(op.getIndices().getType()); |
| mlir::Type llvmIndexType = getTypeConverter()->convertType( |
| elementTypeIfVector(op.getIndices().getType())); |
| uint64_t numElements = |
| mlir::cast<cir::VectorType>(op.getVec().getType()).getSize(); |
| |
| uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1; |
| mlir::Value maskValue = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, llvmIndexType, |
| rewriter.getIntegerAttr(llvmIndexType, maskBits)); |
| mlir::Value maskVector = |
| mlir::LLVM::UndefOp::create(rewriter, loc, llvmIndexVecType); |
| |
| for (uint64_t i = 0; i < numElements; ++i) { |
| mlir::Value idxValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); |
| maskVector = mlir::LLVM::InsertElementOp::create(rewriter, loc, maskVector, |
| maskValue, idxValue); |
| } |
| |
| mlir::Value maskedIndices = mlir::LLVM::AndOp::create( |
| rewriter, loc, llvmIndexVecType, adaptor.getIndices(), maskVector); |
| mlir::Value result = mlir::LLVM::UndefOp::create( |
| rewriter, loc, getTypeConverter()->convertType(op.getVec().getType())); |
| for (uint64_t i = 0; i < numElements; ++i) { |
| mlir::Value iValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i); |
| mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create( |
| rewriter, loc, maskedIndices, iValue); |
| mlir::Value valueAtIndex = |
| mlir::LLVM::ExtractElementOp::create(rewriter, loc, input, indexValue); |
| result = mlir::LLVM::InsertElementOp::create(rewriter, loc, result, |
| valueAtIndex, iValue); |
| } |
| rewriter.replaceOp(op, result); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite( |
| cir::VecTernaryOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| // Convert `cond` into a vector of i1, then use that in a `select` op. |
| mlir::Value bitVec = mlir::LLVM::ICmpOp::create( |
| rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(), |
| mlir::LLVM::ZeroOp::create( |
| rewriter, op.getCond().getLoc(), |
| typeConverter->convertType(op.getCond().getType()))); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>( |
| op, bitVec, adaptor.getLhs(), adaptor.getRhs()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite( |
| cir::ComplexAddOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value lhs = adaptor.getLhs(); |
| mlir::Value rhs = adaptor.getRhs(); |
| mlir::Location loc = op.getLoc(); |
| |
| auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType()); |
| mlir::Type complexElemTy = |
| getTypeConverter()->convertType(complexType.getElementType()); |
| auto lhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); |
| auto lhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); |
| auto rhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); |
| auto rhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); |
| |
| mlir::Value newReal; |
| mlir::Value newImag; |
| if (complexElemTy.isInteger()) { |
| newReal = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsReal, |
| rhsReal); |
| newImag = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsImag, |
| rhsImag); |
| } else { |
| assert(!cir::MissingFeatures::fastMathFlags()); |
| assert(!cir::MissingFeatures::fpConstraints()); |
| newReal = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsReal, |
| rhsReal); |
| newImag = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsImag, |
| rhsImag); |
| } |
| |
| mlir::Type complexLLVMTy = |
| getTypeConverter()->convertType(op.getResult().getType()); |
| auto initialComplex = |
| mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy); |
| |
| auto realComplex = mlir::LLVM::InsertValueOp::create( |
| rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0})); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>( |
| op, realComplex, newImag, ArrayRef(int64_t{1})); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite( |
| cir::ComplexCreateOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type complexLLVMTy = |
| getTypeConverter()->convertType(op.getResult().getType()); |
| auto initialComplex = |
| mlir::LLVM::UndefOp::create(rewriter, op->getLoc(), complexLLVMTy); |
| |
| auto realComplex = mlir::LLVM::InsertValueOp::create( |
| rewriter, op->getLoc(), initialComplex, adaptor.getReal(), |
| ArrayRef(int64_t{0})); |
| |
| auto complex = mlir::LLVM::InsertValueOp::create( |
| rewriter, op->getLoc(), realComplex, adaptor.getImag(), |
| ArrayRef(int64_t{1})); |
| |
| rewriter.replaceOp(op, complex); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite( |
| cir::ComplexRealOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Value operand = adaptor.getOperand(); |
| if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) { |
| operand = mlir::LLVM::ExtractValueOp::create( |
| rewriter, op.getLoc(), resultLLVMTy, operand, |
| llvm::ArrayRef<std::int64_t>{0}); |
| } |
| rewriter.replaceOp(op, operand); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite( |
| cir::ComplexSubOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Value lhs = adaptor.getLhs(); |
| mlir::Value rhs = adaptor.getRhs(); |
| mlir::Location loc = op.getLoc(); |
| |
| auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType()); |
| mlir::Type complexElemTy = |
| getTypeConverter()->convertType(complexType.getElementType()); |
| auto lhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0})); |
| auto lhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1})); |
| auto rhsReal = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0})); |
| auto rhsImag = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1})); |
| |
| mlir::Value newReal; |
| mlir::Value newImag; |
| if (complexElemTy.isInteger()) { |
| newReal = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsReal, |
| rhsReal); |
| newImag = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsImag, |
| rhsImag); |
| } else { |
| assert(!cir::MissingFeatures::fastMathFlags()); |
| assert(!cir::MissingFeatures::fpConstraints()); |
| newReal = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsReal, |
| rhsReal); |
| newImag = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsImag, |
| rhsImag); |
| } |
| |
| mlir::Type complexLLVMTy = |
| getTypeConverter()->convertType(op.getResult().getType()); |
| auto initialComplex = |
| mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy); |
| |
| auto realComplex = mlir::LLVM::InsertValueOp::create( |
| rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0})); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>( |
| op, realComplex, newImag, ArrayRef(int64_t{1})); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite( |
| cir::ComplexImagOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Value operand = adaptor.getOperand(); |
| mlir::Location loc = op.getLoc(); |
| |
| if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) { |
| operand = mlir::LLVM::ExtractValueOp::create( |
| rewriter, loc, resultLLVMTy, operand, llvm::ArrayRef<std::int64_t>{1}); |
| } else { |
| mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(resultLLVMTy); |
| operand = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, resultLLVMTy, zeroAttr); |
| } |
| |
| rewriter.replaceOp(op, operand); |
| return mlir::success(); |
| } |
| |
| mlir::IntegerType computeBitfieldIntType(mlir::Type storageType, |
| mlir::MLIRContext *context, |
| unsigned &storageSize) { |
| return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType) |
| .Case<cir::ArrayType>([&](cir::ArrayType atTy) { |
| storageSize = atTy.getSize() * 8; |
| return mlir::IntegerType::get(context, storageSize); |
| }) |
| .Case<cir::IntType>([&](cir::IntType intTy) { |
| storageSize = intTy.getWidth(); |
| return mlir::IntegerType::get(context, storageSize); |
| }) |
| .Default([](mlir::Type) -> mlir::IntegerType { |
| llvm_unreachable( |
| "Either ArrayType or IntType expected for bitfields storage"); |
| }); |
| } |
| |
| mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite( |
| cir::SetBitfieldOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::OpBuilder::InsertionGuard guard(rewriter); |
| rewriter.setInsertionPoint(op); |
| |
| cir::BitfieldInfoAttr info = op.getBitfieldInfo(); |
| uint64_t size = info.getSize(); |
| uint64_t offset = info.getOffset(); |
| mlir::Type storageType = info.getStorageType(); |
| mlir::MLIRContext *context = storageType.getContext(); |
| |
| unsigned storageSize = 0; |
| |
| mlir::IntegerType intType = |
| computeBitfieldIntType(storageType, context, storageSize); |
| |
| mlir::Value srcVal = createIntCast(rewriter, adaptor.getSrc(), intType); |
| unsigned srcWidth = storageSize; |
| mlir::Value resultVal = srcVal; |
| |
| if (storageSize != size) { |
| assert(storageSize > size && "Invalid bitfield size."); |
| |
| mlir::Value val = mlir::LLVM::LoadOp::create( |
| rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), |
| op.getIsVolatile()); |
| |
| srcVal = |
| createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size)); |
| resultVal = srcVal; |
| srcVal = createShL(rewriter, srcVal, offset); |
| |
| // Mask out the original value. |
| val = createAnd(rewriter, val, |
| ~llvm::APInt::getBitsSet(srcWidth, offset, offset + size)); |
| |
| // Or together the unchanged values and the source value. |
| srcVal = mlir::LLVM::OrOp::create(rewriter, op.getLoc(), val, srcVal); |
| } |
| |
| mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), srcVal, adaptor.getAddr(), |
| op.getAlignment(), op.getIsVolatile()); |
| |
| mlir::Type resultTy = getTypeConverter()->convertType(op.getType()); |
| |
| if (info.getIsSigned()) { |
| assert(size <= storageSize); |
| unsigned highBits = storageSize - size; |
| |
| if (highBits) { |
| resultVal = createShL(rewriter, resultVal, highBits); |
| resultVal = createAShR(rewriter, resultVal, highBits); |
| } |
| } |
| |
| resultVal = createIntCast(rewriter, resultVal, |
| mlir::cast<mlir::IntegerType>(resultTy), |
| info.getIsSigned()); |
| |
| rewriter.replaceOp(op, resultVal); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite( |
| cir::ComplexImagPtrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| cir::PointerType operandTy = op.getOperand().getType(); |
| mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Type elementLLVMTy = |
| getTypeConverter()->convertType(operandTy.getPointee()); |
| |
| mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}}; |
| mlir::LLVM::GEPNoWrapFlags inboundsNuw = |
| mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices, |
| inboundsNuw); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite( |
| cir::ComplexRealPtrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| cir::PointerType operandTy = op.getOperand().getType(); |
| mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Type elementLLVMTy = |
| getTypeConverter()->convertType(operandTy.getPointee()); |
| |
| mlir::LLVM::GEPArg gepIndices[2] = {0, 0}; |
| mlir::LLVM::GEPNoWrapFlags inboundsNuw = |
| mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw; |
| rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>( |
| op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices, |
| inboundsNuw); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite( |
| cir::GetBitfieldOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| mlir::OpBuilder::InsertionGuard guard(rewriter); |
| rewriter.setInsertionPoint(op); |
| |
| cir::BitfieldInfoAttr info = op.getBitfieldInfo(); |
| uint64_t size = info.getSize(); |
| uint64_t offset = info.getOffset(); |
| mlir::Type storageType = info.getStorageType(); |
| mlir::MLIRContext *context = storageType.getContext(); |
| unsigned storageSize = 0; |
| |
| mlir::IntegerType intType = |
| computeBitfieldIntType(storageType, context, storageSize); |
| |
| mlir::Value val = mlir::LLVM::LoadOp::create( |
| rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(), |
| op.getIsVolatile()); |
| val = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), intType, val); |
| |
| if (info.getIsSigned()) { |
| assert(static_cast<unsigned>(offset + size) <= storageSize); |
| unsigned highBits = storageSize - offset - size; |
| val = createShL(rewriter, val, highBits); |
| val = createAShR(rewriter, val, offset + highBits); |
| } else { |
| val = createLShR(rewriter, val, offset); |
| |
| if (static_cast<unsigned>(offset) + size < storageSize) |
| val = createAnd(rewriter, val, |
| llvm::APInt::getLowBitsSet(storageSize, size)); |
| } |
| |
| mlir::Type resTy = getTypeConverter()->convertType(op.getType()); |
| mlir::Value newOp = createIntCast( |
| rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned()); |
| rewriter.replaceOp(op, newOp); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite( |
| cir::InlineAsmOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type llResTy; |
| if (op.getNumResults()) |
| llResTy = getTypeConverter()->convertType(op.getType(0)); |
| |
| cir::AsmFlavor dialect = op.getAsmFlavor(); |
| mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att |
| ? mlir::LLVM::AsmDialect::AD_ATT |
| : mlir::LLVM::AsmDialect::AD_Intel; |
| |
| SmallVector<mlir::Attribute> opAttrs; |
| StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName(); |
| |
| // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we |
| // don't have the result (i.e. void type as a result of operation), the |
| // element type attribute will be attached to the whole instruction, but not |
| // to the operand |
| if (!op.getNumResults()) |
| opAttrs.push_back(mlir::Attribute()); |
| |
| SmallVector<mlir::Value> llvmOperands; |
| SmallVector<mlir::Value> cirOperands; |
| for (auto const &[llvmOp, cirOp] : |
| zip(adaptor.getAsmOperands(), op.getAsmOperands())) { |
| append_range(llvmOperands, llvmOp); |
| append_range(cirOperands, cirOp); |
| } |
| |
| // so far we infer the llvm dialect element type attr from |
| // CIR operand type. |
| for (auto const &[cirOpAttr, cirOp] : |
| zip(op.getOperandAttrs(), cirOperands)) { |
| if (!mlir::isa<mlir::UnitAttr>(cirOpAttr)) { |
| opAttrs.push_back(mlir::Attribute()); |
| continue; |
| } |
| |
| llvm::SmallVector<mlir::NamedAttribute, 1> attrs; |
| cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType()); |
| mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory( |
| *getTypeConverter(), dataLayout, typ.getPointee())); |
| |
| attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr)); |
| mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs); |
| opAttrs.push_back(newDict); |
| } |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>( |
| op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(), |
| op.getSideEffectsAttr(), |
| /*is_align_stack*/ mlir::UnitAttr(), |
| /*tail_call_kind*/ |
| mlir::LLVM::TailCallKindAttr::get( |
| getContext(), mlir::LLVM::tailcallkind::TailCallKind::None), |
| mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect), |
| rewriter.getArrayAttr(opAttrs)); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite( |
| cir::VAStartOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext()); |
| auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr, |
| adaptor.getArgList()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite( |
| cir::VAEndOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext()); |
| auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr, |
| adaptor.getArgList()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVACopyOpLowering::matchAndRewrite( |
| cir::VACopyOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext()); |
| auto dstList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr, |
| adaptor.getDstList()); |
| auto srcList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr, |
| adaptor.getSrcList()); |
| rewriter.replaceOpWithNewOp<mlir::LLVM::VaCopyOp>(op, dstList, srcList); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite( |
| cir::VAArgOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| assert(!cir::MissingFeatures::vaArgABILowering()); |
| auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext()); |
| auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr, |
| adaptor.getArgList()); |
| |
| mlir::Type llvmType = |
| getTypeConverter()->convertType(op->getResultTypes().front()); |
| if (!llvmType) |
| return mlir::failure(); |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMLabelOpLowering::matchAndRewrite( |
| cir::LabelOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::MLIRContext *ctx = rewriter.getContext(); |
| mlir::Block *block = op->getBlock(); |
| // A BlockTagOp cannot reside in the entry block. The address of the entry |
| // block cannot be taken |
| if (block->isEntryBlock()) { |
| mlir::Block *newBlock = |
| rewriter.splitBlock(op->getBlock(), mlir::Block::iterator(op)); |
| rewriter.setInsertionPointToEnd(block); |
| mlir::LLVM::BrOp::create(rewriter, op.getLoc(), newBlock); |
| } |
| auto tagAttr = |
| mlir::LLVM::BlockTagAttr::get(ctx, blockInfoAddr.getTagIndex()); |
| rewriter.setInsertionPoint(op); |
| |
| auto blockTagOp = |
| mlir::LLVM::BlockTagOp::create(rewriter, op->getLoc(), tagAttr); |
| mlir::LLVM::LLVMFuncOp func = op->getParentOfType<mlir::LLVM::LLVMFuncOp>(); |
| auto blockInfoAttr = |
| cir::BlockAddrInfoAttr::get(ctx, func.getSymName(), op.getLabel()); |
| blockInfoAddr.mapBlockTag(blockInfoAttr, blockTagOp); |
| rewriter.eraseOp(op); |
| |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMBlockAddressOpLowering::matchAndRewrite( |
| cir::BlockAddressOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::MLIRContext *ctx = rewriter.getContext(); |
| |
| mlir::LLVM::BlockTagOp matchLabel = |
| blockInfoAddr.lookupBlockTag(op.getBlockAddrInfoAttr()); |
| mlir::LLVM::BlockTagAttr tagAttr; |
| if (!matchLabel) |
| // If the BlockTagOp has not been emitted yet, use a placeholder. |
| // This will later be replaced with the correct tag index during |
| // `resolveBlockAddressOp`. |
| tagAttr = {}; |
| else |
| tagAttr = matchLabel.getTag(); |
| |
| auto blkAddr = mlir::LLVM::BlockAddressAttr::get( |
| rewriter.getContext(), op.getBlockAddrInfoAttr().getFunc(), tagAttr); |
| rewriter.setInsertionPoint(op); |
| auto newOp = mlir::LLVM::BlockAddressOp::create( |
| rewriter, op.getLoc(), mlir::LLVM::LLVMPointerType::get(ctx), blkAddr); |
| if (!matchLabel) |
| blockInfoAddr.addUnresolvedBlockAddress(newOp, op.getBlockAddrInfoAttr()); |
| rewriter.replaceOp(op, newOp); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMIndirectBrOpLowering::matchAndRewrite( |
| cir::IndirectBrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| mlir::Value targetAddr = adaptor.getAddr(); |
| |
| // If the poison attribute is set, use llvm.mlir.poison as the address. |
| // This happens when the block has no predecessors and is essentially |
| // unreachable. Do NOT erase the block argument directly, as that violates |
| // the MLIR dialect conversion framework contract (the framework tracks block |
| // arguments and will clean them up). A block with no predecessors simply |
| // produces no PHI node. |
| if (op.getPoison()) { |
| auto llvmPtrType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| targetAddr = |
| mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), llvmPtrType); |
| } |
| |
| rewriter.replaceOpWithNewOp<mlir::LLVM::IndirectBrOp>( |
| op, targetAddr, adaptor.getSuccOperands(), op.getSuccessors()); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMTokenNoneOpLowering::matchAndRewrite( |
| cir::TokenNoneOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroFreeOpLowering::matchAndRewrite( |
| cir::CoroFreeOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroEndOpLowering::matchAndRewrite( |
| cir::CoroEndOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroAllocOpLowering::matchAndRewrite( |
| cir::CoroAllocOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroBeginOpLowering::matchAndRewrite( |
| cir::CoroBeginOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroIdOpLowering::matchAndRewrite( |
| cir::CoroIdOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCoroSizeOpLowering::matchAndRewrite( |
| cir::CoroSizeOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| return mlir::failure(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMCpuIdOpLowering::matchAndRewrite( |
| cir::CpuIdOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| mlir::Type i32Ty = rewriter.getI32Type(); |
| mlir::Type i64Ty = rewriter.getI64Type(); |
| mlir::Type i32PtrTy = mlir::LLVM::LLVMPointerType::get(i32Ty.getContext(), 0); |
| |
| mlir::Type cpuidRetTy = mlir::LLVM::LLVMStructType::getLiteral( |
| rewriter.getContext(), {i32Ty, i32Ty, i32Ty, i32Ty}); |
| |
| mlir::Value functionId = adaptor.getFunctionId(); |
| mlir::Value subFunctionId = adaptor.getSubFunctionId(); |
| |
| StringRef asmString, constraints; |
| mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>(); |
| llvm::Triple triple( |
| mlir::cast<mlir::StringAttr>( |
| moduleOp->getAttr(cir::CIRDialect::getTripleAttrName())) |
| .getValue()); |
| if (triple.getArch() == llvm::Triple::x86) { |
| asmString = "cpuid"; |
| constraints = "={ax},={bx},={cx},={dx},{ax},{cx}"; |
| } else { |
| // x86-64 uses %rbx as the base register, so preserve it. |
| asmString = "xchgq %rbx, ${1:q}\n" |
| "cpuid\n" |
| "xchgq %rbx, ${1:q}"; |
| constraints = "={ax},=r,={cx},={dx},0,2"; |
| } |
| |
| mlir::Value inlineAsm = |
| mlir::LLVM::InlineAsmOp::create( |
| rewriter, op.getLoc(), cpuidRetTy, {functionId, subFunctionId}, |
| rewriter.getStringAttr(asmString), |
| rewriter.getStringAttr(constraints), |
| /*has_side_effects=*/mlir::UnitAttr{}, |
| /*is_align_stack=*/mlir::UnitAttr{}, |
| /*tail_call_kind=*/mlir::LLVM::TailCallKindAttr{}, |
| /*asm_dialect=*/mlir::LLVM::AsmDialectAttr{}, |
| /*operand_attrs=*/mlir::ArrayAttr{}) |
| .getResult(0); |
| |
| mlir::Value basePtr = adaptor.getCpuInfo(); |
| |
| mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>()); |
| unsigned alignment = layout.getTypeABIAlignment(i32Ty); |
| for (unsigned i = 0; i < 4; i++) { |
| mlir::Value extracted = |
| mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(), inlineAsm, i) |
| .getResult(); |
| mlir::Value index = mlir::LLVM::ConstantOp::create( |
| rewriter, op.getLoc(), i64Ty, rewriter.getI64IntegerAttr(i)); |
| llvm::SmallVector<mlir::Value, 1> gepIndices = {index}; |
| mlir::Value storePtr = mlir::LLVM::GEPOp::create( |
| rewriter, op.getLoc(), i32PtrTy, i32Ty, basePtr, |
| gepIndices, mlir::LLVM::GEPNoWrapFlags::none) |
| .getResult(); |
| mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), extracted, storePtr, |
| alignment); |
| } |
| |
| rewriter.eraseOp(op); |
| return mlir::success(); |
| } |
| |
| mlir::LogicalResult CIRToLLVMMemChrOpLowering::matchAndRewrite( |
| cir::MemChrOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext()); |
| mlir::Type srcTy = getTypeConverter()->convertType(op.getSrc().getType()); |
| mlir::Type patternTy = |
| getTypeConverter()->convertType(op.getPattern().getType()); |
| mlir::Type lenTy = getTypeConverter()->convertType(op.getLen().getType()); |
| auto fnTy = |
| mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {srcTy, patternTy, lenTy}, |
| /*isVarArg=*/false); |
| llvm::StringRef fnName = "memchr"; |
| |
| mlir::Builder b(rewriter.getContext()); |
| mlir::NamedAttribute noundefAttr = |
| b.getNamedAttr("llvm.noundef", b.getUnitAttr()); |
| mlir::DictionaryAttr noundefDict = mlir::DictionaryAttr::get( |
| rewriter.getContext(), llvm::ArrayRef(noundefAttr)); |
| SmallVector<mlir::Attribute> argAttrVec(3, noundefDict); |
| mlir::ArrayAttr argAttrs = |
| mlir::ArrayAttr::get(rewriter.getContext(), argAttrVec); |
| |
| createLLVMFuncOpIfNotExist(rewriter, symbolTables, op, fnName, fnTy, |
| argAttrs); |
| |
| mlir::LLVM::CallOp newCall = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>( |
| op, mlir::TypeRange{llvmPtrTy}, fnName, |
| mlir::ValueRange{adaptor.getSrc(), adaptor.getPattern(), |
| adaptor.getLen()}); |
| newCall.setArgAttrsAttr(argAttrs); |
| return mlir::success(); |
| } |
| |
| // Function to do the clear-padding operation. This is a faithful translation of |
| // CGBuiltin.cpp's ClearPadding function. |
| static void clearPadding(mlir::ConversionPatternRewriter &rewriter, |
| mlir::Location loc, mlir::Value inputPtr, |
| uint64_t baseAlignment, |
| cir::OffsetPairAttr paddingAttr) { |
| // FIXME(cir): Classic-codegen pulls this from the data layout, but MLIR |
| // DataLayout just assumes it is 8 everywhere (as does all our lowering). |
| // Clang doesn't support any non-8-bit-CHAR_BIT architectures, but keeping |
| // this separate so we can use it next time. |
| uint64_t charWidth = 8; |
| mlir::Type i8Ty = rewriter.getI8Type(); |
| mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(i8Ty.getContext()); |
| |
| auto startByte = paddingAttr.getStart() / charWidth; |
| auto startBit = paddingAttr.getStart() % charWidth; |
| auto endByte = paddingAttr.getEnd() / charWidth; |
| auto endBit = paddingAttr.getEnd() % charWidth; |
| |
| if (startByte == endByte) { |
| // Interval is within a single byte |
| auto index = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, rewriter.getI32Type(), startByte); |
| auto element = mlir::LLVM::GEPOp::create(rewriter, loc, ptrTy, i8Ty, |
| inputPtr, {index}); |
| |
| uint64_t adjustedAlignment = llvm::MinAlign(baseAlignment, startByte); |
| |
| auto value = mlir::LLVM::LoadOp::create(rewriter, loc, i8Ty, element, |
| adjustedAlignment); |
| |
| // Create mask to clear bits within the byte |
| // We want to clear bits from StartBit to EndBit-1 |
| uint8_t bitsToClear = ((1 << endBit) - 1) & ~((1 << startBit) - 1); |
| uint8_t bitsToKeep = ~bitsToClear; |
| auto maskValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, i8Ty, bitsToKeep); |
| auto newValue = mlir::LLVM::AndOp::create(rewriter, loc, value, maskValue); |
| |
| mlir::LLVM::StoreOp::create(rewriter, loc, newValue, element, |
| adjustedAlignment); |
| } else { |
| // Handle the start byte |
| if (startBit != 0) { |
| auto index = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, rewriter.getI32Type(), startByte); |
| auto element = mlir::LLVM::GEPOp::create(rewriter, loc, ptrTy, i8Ty, |
| inputPtr, {index}); |
| uint64_t adjustedAlignment = llvm::MinAlign(baseAlignment, startByte); |
| |
| auto value = mlir::LLVM::LoadOp::create(rewriter, loc, i8Ty, element, |
| adjustedAlignment); |
| |
| uint8_t bitsToClear = ((1 << (charWidth - startBit)) - 1) << startBit; |
| uint8_t bitsToKeep = ~bitsToClear; |
| auto maskValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, i8Ty, bitsToKeep); |
| auto newValue = |
| mlir::LLVM::AndOp::create(rewriter, loc, value, maskValue); |
| |
| mlir::LLVM::StoreOp::create(rewriter, loc, newValue, element, |
| adjustedAlignment); |
| ++startByte; |
| } |
| |
| // Handle full bytes in the middle |
| for (auto offset = startByte; offset < endByte; ++offset) { |
| auto index = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, rewriter.getI32Type(), offset); |
| auto element = mlir::LLVM::GEPOp::create(rewriter, loc, ptrTy, i8Ty, |
| inputPtr, {index}); |
| uint64_t adjustedAlignment = llvm::MinAlign(baseAlignment, offset); |
| |
| auto zero = mlir::LLVM::ConstantOp::create(rewriter, loc, i8Ty, 0); |
| mlir::LLVM::StoreOp::create(rewriter, loc, zero, element, |
| adjustedAlignment); |
| } |
| |
| // Handle the end byte |
| if (endBit != 0) { |
| auto index = mlir::LLVM::ConstantOp::create( |
| rewriter, loc, rewriter.getI32Type(), endByte); |
| auto element = mlir::LLVM::GEPOp::create(rewriter, loc, ptrTy, i8Ty, |
| inputPtr, {index}); |
| uint64_t adjustedAlignment = llvm::MinAlign(baseAlignment, endByte); |
| |
| auto value = mlir::LLVM::LoadOp::create(rewriter, loc, i8Ty, element, |
| adjustedAlignment); |
| |
| uint8_t bitsToClear = (1 << endBit) - 1; |
| uint8_t bitsToKeep = ~bitsToClear; |
| auto maskValue = |
| mlir::LLVM::ConstantOp::create(rewriter, loc, i8Ty, bitsToKeep); |
| auto newValue = |
| mlir::LLVM::AndOp::create(rewriter, loc, value, maskValue); |
| |
| mlir::LLVM::StoreOp::create(rewriter, loc, newValue, element, |
| adjustedAlignment); |
| } |
| } |
| } |
| |
| mlir::LogicalResult CIRToLLVMClearPaddingOpLowering::matchAndRewrite( |
| cir::ClearPaddingOp op, OpAdaptor adaptor, |
| mlir::ConversionPatternRewriter &rewriter) const { |
| |
| mlir::Value inputPtr = adaptor.getArg(); |
| for (mlir::Attribute attr : op.getPadding()) |
| clearPadding(rewriter, op.getLoc(), inputPtr, op.getAlignment(), |
| cast<cir::OffsetPairAttr>(attr)); |
| |
| rewriter.eraseOp(op); |
| return mlir::success(); |
| } |
| |
| std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() { |
| return std::make_unique<ConvertCIRToLLVMPass>(); |
| } |
| |
| void populateCIRToLLVMPasses(mlir::OpPassManager &pm, bool enableOpenMP) { |
| mlir::populateCIRPreLoweringPasses(pm); |
| if (enableOpenMP) |
| pm.addPass(mlir::omp::createMarkDeclareTargetPass()); |
| pm.addPass(createConvertCIRToLLVMPass()); |
| if (enableOpenMP) |
| pm.addPass(mlir::omp::createHostOpFilteringPass()); |
| } |
| |
| std::unique_ptr<llvm::Module> |
| lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx, |
| bool enableOpenMP, StringRef mlirSaveTempsOutFile, |
| llvm::vfs::FileSystem *fs) { |
| llvm::TimeTraceScope scope("lower from CIR to LLVM directly"); |
| |
| mlir::MLIRContext *mlirCtx = mlirModule.getContext(); |
| |
| mlir::PassManager pm(mlirCtx); |
| populateCIRToLLVMPasses(pm, enableOpenMP); |
| |
| (void)mlir::applyPassManagerCLOptions(pm); |
| |
| if (mlir::failed(pm.run(mlirModule))) { |
| // FIXME: Handle any errors where they occurs and return a nullptr here. |
| report_fatal_error( |
| "The pass manager failed to lower CIR to LLVMIR dialect!"); |
| } |
| |
| if (!mlirSaveTempsOutFile.empty()) { |
| std::error_code ec; |
| llvm::raw_fd_ostream out(mlirSaveTempsOutFile, ec); |
| if (!ec) |
| mlirModule->print(out); |
| } |
| |
| mlir::registerBuiltinDialectTranslation(*mlirCtx); |
| mlir::registerLLVMDialectTranslation(*mlirCtx); |
| mlir::registerOpenMPDialectTranslation(*mlirCtx); |
| mlir::registerCIRDialectTranslation(*mlirCtx); |
| |
| llvm::TimeTraceScope translateScope("translateModuleToLLVMIR"); |
| |
| StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule"); |
| std::unique_ptr<llvm::Module> llvmModule = mlir::translateModuleToLLVMIR( |
| mlirModule, llvmCtx, moduleName, /*disableVerification=*/false, fs); |
| |
| if (!llvmModule) { |
| // FIXME: Handle any errors where they occurs and return a nullptr here. |
| report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!"); |
| } |
| |
| return llvmModule; |
| } |
| } // namespace direct |
| } // namespace cir |
