clang/utils/TableGen/CIRLoweringEmitter.cpp - llvm-project - Git at Google


            
            //===-- CIRLoweringEmitter.cpp - Generate CIR lowering patterns -----------===//

            
            //

            
            // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

            
            // See https://llvm.org/LICENSE.txt for license information.

            
            // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

            
            //

            
            //===----------------------------------------------------------------------===//

            
            //

            
            // This TableGen backend emits CIR operation lowering patterns.

            
            //

            
            //===----------------------------------------------------------------------===//


            #include "TableGenBackends.h"

            
            #include "llvm/TableGen/Error.h"

            
            #include "llvm/TableGen/Record.h"

            
            #include "llvm/TableGen/TableGenBackend.h"

            
            #include <string>

            
            #include <utility>

            
            #include <vector>


            using namespace llvm;

            
            using namespace clang;


            namespace {

            
            std::vector<std::string> CXXABILoweringPatterns;

            
            std::vector<std::string> CXXABILoweringPatternsList;

            
            std::vector<std::string> CXXABILoweringAttrAlwaysLegal;

            
            std::vector<std::string> LLVMLoweringPatterns;

            
            std::vector<std::string> LLVMLoweringPatternsList;

            
            std::string CIRAttrToValueVisitFunc;

            
            std::vector<std::string> CIRAttrToValueVisitorCaseTypes;

            
            std::vector<std::string> CIRAttrToValueVisitorDecls;


            struct CustomLoweringCtor {

            
              struct Param {

            
                std::string Type;

            
                std::string Name;

            
              };


              std::vector<Param> Params;

            
            };


            // Adapted from mlir/lib/TableGen/Operator.cpp

            
            // Returns the C++ class name of the operation, which is the name of the

            
            // operation with the dialect prefix removed and the first underscore removed.

            
            // If the operation name starts with an underscore, the underscore is considered

            
            // part of the class name.

            
            std::string GetOpCppClassName(const Record *OpRecord) {

            
              StringRef Name = OpRecord->getName();

            
              StringRef Prefix;

            
              StringRef CppClassName;

            
              std::tie(Prefix, CppClassName) = Name.split('_');

            
              if (Prefix.empty()) {

            
                // Class name with a leading underscore and without dialect prefix

            
                return Name.str();

            
              }

            
              if (CppClassName.empty()) {

            
                // Class name without dialect prefix

            
                return Prefix.str();

            
              }


              return CppClassName.str();

            
            }


            // Returns the namespace-qualified C++ class name of an attribute. Unlike

            
            // operations, attributes carry the authoritative name in cppClassName, so the

            
            // def name is free to differ from it.

            
            std::string GetAttrCppClassRef(const Record *AttrRecord) {

            
              StringRef Ns =

            
                  AttrRecord->getValueAsDef("dialect")->getValueAsString("cppNamespace");

            
              Ns.consume_front("::");

            
              std::string CppClassRef = Ns.str();

            
              CppClassRef += "::";

            
              CppClassRef += AttrRecord->getValueAsString("cppClassName");

            
              return CppClassRef;

            
            }


            std::string GetOpABILoweringPatternName(llvm::StringRef OpName) {

            
              std::string Name = "CIR";

            
              Name += OpName;

            
              Name += "ABILowering";

            
              return Name;

            
            }


            std::string GetOpLLVMLoweringPatternName(llvm::StringRef OpName) {

            
              std::string Name = "CIRToLLVM";

            
              Name += OpName;

            
              Name += "Lowering";

            
              return Name;

            
            }

            
            std::optional<CustomLoweringCtor> parseCustomLoweringCtor(const Record *R) {

            
              if (!R)

            
                return std::nullopt;


              CustomLoweringCtor Ctor;

            
              const DagInit *Args = R->getValueAsDag("dagParams");


              for (const auto &[Arg, Name] : Args->getArgAndNames()) {

            
                Ctor.Params.push_back(

            
                    {Arg->getAsUnquotedString(), Name->getAsUnquotedString()});

            
              }


              return Ctor;

            
            }


            void emitCustomParamList(raw_ostream &Code,

            
                                     ArrayRef<CustomLoweringCtor::Param> Params) {

            
              for (const CustomLoweringCtor::Param &Param : Params) {

            
                Code << ", ";

            
                Code << Param.Type << " " << Param.Name;

            
              }

            
            }


            void emitCustomInitList(raw_ostream &Code,

            
                                    ArrayRef<CustomLoweringCtor::Param> Params) {

            
              for (const CustomLoweringCtor::Param &P : Params)

            
                Code << ", " << P.Name << "(" << P.Name << ")";

            
            }


            void GenerateABILoweringPattern(llvm::StringRef OpName,

            
                                            llvm::StringRef PatternName) {

            
              std::string CodeBuffer;

            
              llvm::raw_string_ostream Code(CodeBuffer);


              Code << "class " << PatternName

            
                   << " : public mlir::OpConversionPattern<cir::" << OpName << "> {\n";

            
              Code << "  [[maybe_unused]] mlir::DataLayout *dataLayout;\n";

            
              Code << "  [[maybe_unused]] cir::LowerModule *lowerModule;\n";

            
              Code << "\n";


              Code << "public:\n";

            
              Code << "  " << PatternName

            
                   << "(mlir::MLIRContext *context, const mlir::TypeConverter "

            
                      "&typeConverter, mlir::DataLayout &dataLayout, cir::LowerModule "

            
                      "&lowerModule)\n";

            
              Code << "    : OpConversionPattern<cir::" << OpName

            
                   << ">(typeConverter, context), dataLayout(&dataLayout), "

            
                      "lowerModule(&lowerModule) {}\n";

            
              Code << "\n";


              Code << "  mlir::LogicalResult matchAndRewrite(cir::" << OpName

            
                   << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "

            
                      "const override;\n";


              Code << "};\n";


              CXXABILoweringPatterns.push_back(std::move(CodeBuffer));

            
            }


            void GenerateLLVMLoweringPattern(llvm::StringRef OpName,

            
                                             llvm::StringRef PatternName, bool IsRecursive,

            
                                             llvm::StringRef ExtraDecl,

            
                                             const Record *CustomCtorRec,

            
                                             llvm::StringRef LLVMOp,

            
                                             llvm::StringRef ConstrainedLLVMIntrinsic,

            
                                             bool ConstrainedHasRoundingMode,

            
                                             bool HasFastMathFlags, bool HasZeroResult) {

            
              std::optional<CustomLoweringCtor> CustomCtor =

            
                  parseCustomLoweringCtor(CustomCtorRec);

            
              std::string CodeBuffer;

            
              llvm::raw_string_ostream Code(CodeBuffer);


              Code << "class " << PatternName

            
                   << " : public mlir::OpConversionPattern<cir::" << OpName << "> {\n";

            
              Code << "  [[maybe_unused]] mlir::DataLayout const &dataLayout;\n";

            
              Code << "  [[maybe_unused]] mlir::SymbolTableCollection &symbolTables;\n";


              if (CustomCtor) {

            
                for (const CustomLoweringCtor::Param &P : CustomCtor->Params)

            
                  Code << "  " << P.Type << " " << P.Name << ";\n";

            
              }


              Code << "\n";


              Code << "public:\n";

            
              Code << "  using mlir::OpConversionPattern<cir::" << OpName

            
                   << ">::OpConversionPattern;\n";


              // Constructor

            
              Code << "  " << PatternName

            
                   << "(const mlir::TypeConverter &typeConverter, "

            
                      "mlir::MLIRContext *context, const mlir::DataLayout &dataLayout, "

            
                      "mlir::SymbolTableCollection &symbolTables";


              if (CustomCtor)

            
                emitCustomParamList(Code, CustomCtor->Params);


              Code << ")\n";


              Code << "    : OpConversionPattern<cir::" << OpName

            
                   << ">(typeConverter, context), dataLayout(dataLayout), "

            
                      "symbolTables(symbolTables)";


              if (CustomCtor)

            
                emitCustomInitList(Code, CustomCtor->Params);


              Code << " {\n";


              if (IsRecursive)

            
                Code << "    setHasBoundedRewriteRecursion();\n";


              Code << "  }\n\n";


              if (!ConstrainedLLVMIntrinsic.empty()) {

            
                // Generate a matchAndRewrite body for a floating-point operation that

            
                // carries an optional `fenv` attribute. The shared `lowerConstrainableFPOp`

            
                // helper lowers to the plain `llvmOp` when no `fenv` attribute is present

            
                // and to a call to the constrained floating-point intrinsic when one is.

            
                assert(!LLVMOp.empty() && "constrainedLLVMIntrinsic requires llvmOp");

            
                Code

            
                    << "  mlir::LogicalResult matchAndRewrite(cir::" << OpName

            
                    << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "

            
                       "const override {\n";

            
                Code << "    return lowerConstrainableFPOp<mlir::LLVM::" << LLVMOp

            
                     << ">(\n";

            
                Code << "        op, adaptor.getOperands(), op.getFenvAttr(),\n";

            
                Code << "        "

            
                     << (HasFastMathFlags ? "op.getFastmathFlagsAttr()"

            
                                          : "cir::FastMathFlagsAttr{}")

            
                     << ",\n";

            
                Code << "        *getTypeConverter(), rewriter, \""

            
                     << ConstrainedLLVMIntrinsic << "\",\n";

            
                Code << "        /*hasRoundingMode=*/"

            
                     << (ConstrainedHasRoundingMode ? "true" : "false") << ");\n";

            
                Code << "  }\n";

            
              } else if (!LLVMOp.empty()) {

            
                // Generate the matchAndRewrite body automatically.

            
                Code

            
                    << "  mlir::LogicalResult matchAndRewrite(cir::" << OpName

            
                    << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "

            
                       "const override {\n";

            
                std::string Properties =

            
                    "cir::getDefaultProperties<mlir::LLVM::" + LLVMOp.str() +

            
                    ">(op.getContext())";

            
                if (HasZeroResult) {

            
                  Code << "    rewriter.replaceOpWithNewOp<mlir::LLVM::" << LLVMOp

            
                       << ">(op, mlir::TypeRange{}, adaptor.getOperands(), " << Properties

            
                       << ");\n";

            
                } else {

            
                  Code << "    mlir::Type resTy = "

            
                          "typeConverter->convertType(op.getType());\n";

            
                  Code << "    rewriter.replaceOpWithNewOp<mlir::LLVM::" << LLVMOp

            
                       << ">(op, mlir::TypeRange{resTy}, adaptor.getOperands(), "

            
                       << Properties << ");\n";

            
                }

            
                Code << "    return mlir::success();\n";

            
                Code << "  }\n";

            
              } else {

            
                Code

            
                    << "  mlir::LogicalResult matchAndRewrite(cir::" << OpName

            
                    << " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "

            
                       "const override;\n";

            
              }


              if (!ExtraDecl.empty()) {

            
                Code << "\nprivate:\n";

            
                Code << ExtraDecl << "\n";

            
              }


              Code << "};\n";


              LLVMLoweringPatterns.push_back(std::move(CodeBuffer));

            
            }


            void Generate(const Record *OpRecord) {

            
              std::string OpName = GetOpCppClassName(OpRecord);


              if (OpRecord->getValueAsBit("hasCXXABILowering")) {

            
                std::string PatternName = GetOpABILoweringPatternName(OpName);

            
                GenerateABILoweringPattern(OpName, PatternName);

            
                CXXABILoweringPatternsList.push_back(std::move(PatternName));

            
              }


              if (OpRecord->getValueAsBit("hasLLVMLowering")) {

            
                std::string PatternName = GetOpLLVMLoweringPatternName(OpName);

            
                bool IsRecursive = OpRecord->getValueAsBit("isLLVMLoweringRecursive");

            
                const Record *CustomCtor =

            
                    OpRecord->getValueAsOptionalDef("customLLVMLoweringConstructorDecl");

            
                llvm::StringRef ExtraDecl =

            
                    OpRecord->getValueAsString("extraLLVMLoweringPatternDecl");


                llvm::StringRef LLVMOp = OpRecord->getValueAsString("llvmOp");

            
                llvm::StringRef ConstrainedLLVMIntrinsic =

            
                    OpRecord->getValueAsString("constrainedLLVMIntrinsic");

            
                bool ConstrainedHasRoundingMode =

            
                    OpRecord->getValueAsBit("constrainedLLVMIntrinsicHasRoundingMode");

            
                const DagInit *ArgsDag = OpRecord->getValueAsDag("arguments");

            
                bool HasFastMathFlags =

            
                    llvm::any_of(ArgsDag->getArgNames(), [](const StringInit *Name) {

            
                      return Name && Name->getValue() == "fastmath_flags";

            
                    });


                if (!LLVMOp.empty() && CustomCtor)

            
                  PrintFatalError(OpRecord->getLoc(),

            
                                  "op '" + OpName +

            
                                      "' has both llvmOp and a custom lowering "

            
                                      "constructor, which is not supported");


                if (!ConstrainedLLVMIntrinsic.empty() && LLVMOp.empty())

            
                  PrintFatalError(OpRecord->getLoc(),

            
                                  "op '" + OpName +

            
                                      "' has constrainedLLVMIntrinsic set but no llvmOp, "

            
                                      "which is not supported");


                const DagInit *ResultsDag = OpRecord->getValueAsDag("results");

            
                bool IsZeroResult = ResultsDag->getNumArgs() == 0;

            
                GenerateLLVMLoweringPattern(OpName, PatternName, IsRecursive, ExtraDecl,

            
                                            CustomCtor, LLVMOp, ConstrainedLLVMIntrinsic,

            
                                            ConstrainedHasRoundingMode, HasFastMathFlags,

            
                                            IsZeroResult);

            
                // Only automatically register patterns that use the default constructor.

            
                // Patterns with a custom constructor must be manually registered by the

            
                // lowering pass.

            
                if (!CustomCtor)

            
                  LLVMLoweringPatternsList.push_back(std::move(PatternName));

            
              }

            
            }


            void GenerateCIREnumAttrs(const Record *Record) {

            
              // EnumAttr is in a separate hierarchy, so we have to set these separately, as

            
              // they never have an 'illegal' CXXABI type in them.

            
              CXXABILoweringAttrAlwaysLegal.push_back(GetAttrCppClassRef(Record));

            
            }


            void GenerateAttrToValueVisitor(const Record *Rec) {

            
              std::string CppClassRef = GetAttrCppClassRef(Rec);


              std::string CodeBuffer;

            
              llvm::raw_string_ostream Code(CodeBuffer);

            
              Code << "  mlir::Value visitCirAttr(" << CppClassRef << " attr);";

            
              CIRAttrToValueVisitorDecls.push_back(std::move(CodeBuffer));

            
              CIRAttrToValueVisitorCaseTypes.push_back(std::move(CppClassRef));

            
            }


            void GenerateAttrToValueVisitFunc() {

            
              std::string CodeBuffer;

            
              llvm::raw_string_ostream Code(CodeBuffer);

            
              Code << "  mlir::Value visit(mlir::Attribute attr) {\n"

            
                   << "    return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)\n"

            
                   << "        .Case<\n              "

            
                   << llvm::join(CIRAttrToValueVisitorCaseTypes, ",\n              ")

            
                   << ">(\n"

            
                   << "            [&](auto attrT) { return visitCirAttr(attrT); })\n"

            
                   << "        .Default([this](mlir::Attribute attr) {\n"

            
                   << "          mlir::emitError(parentOp->getLoc(), \"unsupported CIR "

            
                      "attribute in LLVM constant lowering\")\n"

            
                   << "              << attr;\n"

            
                   << "          return mlir::Value();\n"

            
                   << "        });\n"

            
                   << "  }\n";

            
              CIRAttrToValueVisitFunc = std::move(CodeBuffer);

            
            }


            void GenerateCIRAttrs(const Record *Record) {

            
              if (!Record->getValueAsBit("canHaveIllegalCXXABIType"))

            
                CXXABILoweringAttrAlwaysLegal.push_back(GetAttrCppClassRef(Record));

            
              if (Record->getValueAsBit("hasAttrToValueLowering"))

            
                GenerateAttrToValueVisitor(Record);

            
            }

            
            } // namespace


            void clang::EmitCIRLowering(const llvm::RecordKeeper &RK,

            
                                        llvm::raw_ostream &OS) {

            
              emitSourceFileHeader("Lowering patterns for CIR operations", OS);

            
              for (const auto *OpRecord : RK.getAllDerivedDefinitions("CIR_Op"))

            
                Generate(OpRecord);

            
              for (const auto *OpRecord : RK.getAllDerivedDefinitions("EnumAttr"))

            
                GenerateCIREnumAttrs(OpRecord);

            
              for (const auto *OpRecord : RK.getAllDerivedDefinitions("CIR_Attr"))

            
                GenerateCIRAttrs(OpRecord);

            
              GenerateAttrToValueVisitFunc();


              OS << "#ifdef GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS\n"

            
                 << CIRAttrToValueVisitFunc << "\n"

            
                 << llvm::join(CIRAttrToValueVisitorDecls, "\n") << "\n"

            
                 << "#endif // GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS\n\n";


              OS << "#ifdef GET_ABI_LOWERING_PATTERNS\n"

            
                 << llvm::join(CXXABILoweringPatterns, "\n") << "#endif\n\n";

            
              OS << "#ifdef GET_ABI_LOWERING_PATTERNS_LIST\n"

            
                 << llvm::join(CXXABILoweringPatternsList, ",\n") << "\n#endif\n\n";


              OS << "#ifdef GET_LLVM_LOWERING_PATTERNS\n"

            
                 << llvm::join(LLVMLoweringPatterns, "\n") << "#endif\n\n";

            
              OS << "#ifdef GET_LLVM_LOWERING_PATTERNS_LIST\n"

            
                 << llvm::join(LLVMLoweringPatternsList, ",\n") << "\n#endif\n\n";


              OS << "#ifdef CXX_ABI_ALWAYS_LEGAL_ATTRS\n"

            
                 << llvm::join(CXXABILoweringAttrAlwaysLegal, ",\n") << "\n#endif\n\n";

            
            }
	//===-- CIRLoweringEmitter.cpp - Generate CIR lowering patterns -----------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This TableGen backend emits CIR operation lowering patterns.
	//
	//===----------------------------------------------------------------------===//

	#include "TableGenBackends.h"
	#include "llvm/TableGen/Error.h"
	#include "llvm/TableGen/Record.h"
	#include "llvm/TableGen/TableGenBackend.h"
	#include <string>
	#include <utility>
	#include <vector>

	using namespace llvm;
	using namespace clang;

	namespace {
	std::vector<std::string> CXXABILoweringPatterns;
	std::vector<std::string> CXXABILoweringPatternsList;
	std::vector<std::string> CXXABILoweringAttrAlwaysLegal;
	std::vector<std::string> LLVMLoweringPatterns;
	std::vector<std::string> LLVMLoweringPatternsList;
	std::string CIRAttrToValueVisitFunc;
	std::vector<std::string> CIRAttrToValueVisitorCaseTypes;
	std::vector<std::string> CIRAttrToValueVisitorDecls;

	struct CustomLoweringCtor {
	struct Param {
	std::string Type;
	std::string Name;
	};

	std::vector<Param> Params;
	};

	// Adapted from mlir/lib/TableGen/Operator.cpp
	// Returns the C++ class name of the operation, which is the name of the
	// operation with the dialect prefix removed and the first underscore removed.
	// If the operation name starts with an underscore, the underscore is considered
	// part of the class name.
	std::string GetOpCppClassName(const Record *OpRecord) {
	StringRef Name = OpRecord->getName();
	StringRef Prefix;
	StringRef CppClassName;
	std::tie(Prefix, CppClassName) = Name.split('_');
	if (Prefix.empty()) {
	// Class name with a leading underscore and without dialect prefix
	return Name.str();
	}
	if (CppClassName.empty()) {
	// Class name without dialect prefix
	return Prefix.str();
	}

	return CppClassName.str();
	}

	// Returns the namespace-qualified C++ class name of an attribute. Unlike
	// operations, attributes carry the authoritative name in cppClassName, so the
	// def name is free to differ from it.
	std::string GetAttrCppClassRef(const Record *AttrRecord) {
	StringRef Ns =
	AttrRecord->getValueAsDef("dialect")->getValueAsString("cppNamespace");
	Ns.consume_front("::");
	std::string CppClassRef = Ns.str();
	CppClassRef += "::";
	CppClassRef += AttrRecord->getValueAsString("cppClassName");
	return CppClassRef;
	}

	std::string GetOpABILoweringPatternName(llvm::StringRef OpName) {
	std::string Name = "CIR";
	Name += OpName;
	Name += "ABILowering";
	return Name;
	}

	std::string GetOpLLVMLoweringPatternName(llvm::StringRef OpName) {
	std::string Name = "CIRToLLVM";
	Name += OpName;
	Name += "Lowering";
	return Name;
	}
	std::optional<CustomLoweringCtor> parseCustomLoweringCtor(const Record *R) {
	if (!R)
	return std::nullopt;

	CustomLoweringCtor Ctor;
	const DagInit *Args = R->getValueAsDag("dagParams");

	for (const auto &[Arg, Name] : Args->getArgAndNames()) {
	Ctor.Params.push_back(
	{Arg->getAsUnquotedString(), Name->getAsUnquotedString()});
	}

	return Ctor;
	}

	void emitCustomParamList(raw_ostream &Code,
	ArrayRef<CustomLoweringCtor::Param> Params) {
	for (const CustomLoweringCtor::Param &Param : Params) {
	Code << ", ";
	Code << Param.Type << " " << Param.Name;
	}
	}

	void emitCustomInitList(raw_ostream &Code,
	ArrayRef<CustomLoweringCtor::Param> Params) {
	for (const CustomLoweringCtor::Param &P : Params)
	Code << ", " << P.Name << "(" << P.Name << ")";
	}

	void GenerateABILoweringPattern(llvm::StringRef OpName,
	llvm::StringRef PatternName) {
	std::string CodeBuffer;
	llvm::raw_string_ostream Code(CodeBuffer);

	Code << "class " << PatternName
	<< " : public mlir::OpConversionPattern<cir::" << OpName << "> {\n";
	Code << " [[maybe_unused]] mlir::DataLayout *dataLayout;\n";
	Code << " [[maybe_unused]] cir::LowerModule *lowerModule;\n";
	Code << "\n";

	Code << "public:\n";
	Code << " " << PatternName
	<< "(mlir::MLIRContext *context, const mlir::TypeConverter "
	"&typeConverter, mlir::DataLayout &dataLayout, cir::LowerModule "
	"&lowerModule)\n";
	Code << " : OpConversionPattern<cir::" << OpName
	<< ">(typeConverter, context), dataLayout(&dataLayout), "
	"lowerModule(&lowerModule) {}\n";
	Code << "\n";

	Code << " mlir::LogicalResult matchAndRewrite(cir::" << OpName
	<< " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "
	"const override;\n";

	Code << "};\n";

	CXXABILoweringPatterns.push_back(std::move(CodeBuffer));
	}

	void GenerateLLVMLoweringPattern(llvm::StringRef OpName,
	llvm::StringRef PatternName, bool IsRecursive,
	llvm::StringRef ExtraDecl,
	const Record *CustomCtorRec,
	llvm::StringRef LLVMOp,
	llvm::StringRef ConstrainedLLVMIntrinsic,
	bool ConstrainedHasRoundingMode,
	bool HasFastMathFlags, bool HasZeroResult) {
	std::optional<CustomLoweringCtor> CustomCtor =
	parseCustomLoweringCtor(CustomCtorRec);
	std::string CodeBuffer;
	llvm::raw_string_ostream Code(CodeBuffer);

	Code << "class " << PatternName
	<< " : public mlir::OpConversionPattern<cir::" << OpName << "> {\n";
	Code << " [[maybe_unused]] mlir::DataLayout const &dataLayout;\n";
	Code << " [[maybe_unused]] mlir::SymbolTableCollection &symbolTables;\n";

	if (CustomCtor) {
	for (const CustomLoweringCtor::Param &P : CustomCtor->Params)
	Code << " " << P.Type << " " << P.Name << ";\n";
	}

	Code << "\n";

	Code << "public:\n";
	Code << " using mlir::OpConversionPattern<cir::" << OpName
	<< ">::OpConversionPattern;\n";

	// Constructor
	Code << " " << PatternName
	<< "(const mlir::TypeConverter &typeConverter, "
	"mlir::MLIRContext *context, const mlir::DataLayout &dataLayout, "
	"mlir::SymbolTableCollection &symbolTables";

	if (CustomCtor)
	emitCustomParamList(Code, CustomCtor->Params);

	Code << ")\n";

	Code << " : OpConversionPattern<cir::" << OpName
	<< ">(typeConverter, context), dataLayout(dataLayout), "
	"symbolTables(symbolTables)";

	if (CustomCtor)
	emitCustomInitList(Code, CustomCtor->Params);

	Code << " {\n";

	if (IsRecursive)
	Code << " setHasBoundedRewriteRecursion();\n";

	Code << " }\n\n";

	if (!ConstrainedLLVMIntrinsic.empty()) {
	// Generate a matchAndRewrite body for a floating-point operation that
	// carries an optional `fenv` attribute. The shared `lowerConstrainableFPOp`
	// helper lowers to the plain `llvmOp` when no `fenv` attribute is present
	// and to a call to the constrained floating-point intrinsic when one is.
	assert(!LLVMOp.empty() && "constrainedLLVMIntrinsic requires llvmOp");
	Code
	<< " mlir::LogicalResult matchAndRewrite(cir::" << OpName
	<< " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "
	"const override {\n";
	Code << " return lowerConstrainableFPOp<mlir::LLVM::" << LLVMOp
	<< ">(\n";
	Code << " op, adaptor.getOperands(), op.getFenvAttr(),\n";
	Code << " "
	<< (HasFastMathFlags ? "op.getFastmathFlagsAttr()"
	: "cir::FastMathFlagsAttr{}")
	<< ",\n";
	Code << " *getTypeConverter(), rewriter, \""
	<< ConstrainedLLVMIntrinsic << "\",\n";
	Code << " /hasRoundingMode=/"
	<< (ConstrainedHasRoundingMode ? "true" : "false") << ");\n";
	Code << " }\n";
	} else if (!LLVMOp.empty()) {
	// Generate the matchAndRewrite body automatically.
	Code
	<< " mlir::LogicalResult matchAndRewrite(cir::" << OpName
	<< " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "
	"const override {\n";
	std::string Properties =
	"cir::getDefaultProperties<mlir::LLVM::" + LLVMOp.str() +
	">(op.getContext())";
	if (HasZeroResult) {
	Code << " rewriter.replaceOpWithNewOp<mlir::LLVM::" << LLVMOp
	<< ">(op, mlir::TypeRange{}, adaptor.getOperands(), " << Properties
	<< ");\n";
	} else {
	Code << " mlir::Type resTy = "
	"typeConverter->convertType(op.getType());\n";
	Code << " rewriter.replaceOpWithNewOp<mlir::LLVM::" << LLVMOp
	<< ">(op, mlir::TypeRange{resTy}, adaptor.getOperands(), "
	<< Properties << ");\n";
	}
	Code << " return mlir::success();\n";
	Code << " }\n";
	} else {
	Code
	<< " mlir::LogicalResult matchAndRewrite(cir::" << OpName
	<< " op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) "
	"const override;\n";
	}

	if (!ExtraDecl.empty()) {
	Code << "\nprivate:\n";
	Code << ExtraDecl << "\n";
	}

	Code << "};\n";

	LLVMLoweringPatterns.push_back(std::move(CodeBuffer));
	}

	void Generate(const Record *OpRecord) {
	std::string OpName = GetOpCppClassName(OpRecord);

	if (OpRecord->getValueAsBit("hasCXXABILowering")) {
	std::string PatternName = GetOpABILoweringPatternName(OpName);
	GenerateABILoweringPattern(OpName, PatternName);
	CXXABILoweringPatternsList.push_back(std::move(PatternName));
	}

	if (OpRecord->getValueAsBit("hasLLVMLowering")) {
	std::string PatternName = GetOpLLVMLoweringPatternName(OpName);
	bool IsRecursive = OpRecord->getValueAsBit("isLLVMLoweringRecursive");
	const Record *CustomCtor =
	OpRecord->getValueAsOptionalDef("customLLVMLoweringConstructorDecl");
	llvm::StringRef ExtraDecl =
	OpRecord->getValueAsString("extraLLVMLoweringPatternDecl");

	llvm::StringRef LLVMOp = OpRecord->getValueAsString("llvmOp");
	llvm::StringRef ConstrainedLLVMIntrinsic =
	OpRecord->getValueAsString("constrainedLLVMIntrinsic");
	bool ConstrainedHasRoundingMode =
	OpRecord->getValueAsBit("constrainedLLVMIntrinsicHasRoundingMode");
	const DagInit *ArgsDag = OpRecord->getValueAsDag("arguments");
	bool HasFastMathFlags =
	llvm::any_of(ArgsDag->getArgNames(), [](const StringInit *Name) {
	return Name && Name->getValue() == "fastmath_flags";
	});

	if (!LLVMOp.empty() && CustomCtor)
	PrintFatalError(OpRecord->getLoc(),
	"op '" + OpName +
	"' has both llvmOp and a custom lowering "
	"constructor, which is not supported");

	if (!ConstrainedLLVMIntrinsic.empty() && LLVMOp.empty())
	PrintFatalError(OpRecord->getLoc(),
	"op '" + OpName +
	"' has constrainedLLVMIntrinsic set but no llvmOp, "
	"which is not supported");

	const DagInit *ResultsDag = OpRecord->getValueAsDag("results");
	bool IsZeroResult = ResultsDag->getNumArgs() == 0;
	GenerateLLVMLoweringPattern(OpName, PatternName, IsRecursive, ExtraDecl,
	CustomCtor, LLVMOp, ConstrainedLLVMIntrinsic,
	ConstrainedHasRoundingMode, HasFastMathFlags,
	IsZeroResult);
	// Only automatically register patterns that use the default constructor.
	// Patterns with a custom constructor must be manually registered by the
	// lowering pass.
	if (!CustomCtor)
	LLVMLoweringPatternsList.push_back(std::move(PatternName));
	}
	}

	void GenerateCIREnumAttrs(const Record *Record) {
	// EnumAttr is in a separate hierarchy, so we have to set these separately, as
	// they never have an 'illegal' CXXABI type in them.
	CXXABILoweringAttrAlwaysLegal.push_back(GetAttrCppClassRef(Record));
	}

	void GenerateAttrToValueVisitor(const Record *Rec) {
	std::string CppClassRef = GetAttrCppClassRef(Rec);

	std::string CodeBuffer;
	llvm::raw_string_ostream Code(CodeBuffer);
	Code << " mlir::Value visitCirAttr(" << CppClassRef << " attr);";
	CIRAttrToValueVisitorDecls.push_back(std::move(CodeBuffer));
	CIRAttrToValueVisitorCaseTypes.push_back(std::move(CppClassRef));
	}

	void GenerateAttrToValueVisitFunc() {
	std::string CodeBuffer;
	llvm::raw_string_ostream Code(CodeBuffer);
	Code << " mlir::Value visit(mlir::Attribute attr) {\n"
	<< " return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)\n"
	<< " .Case<\n "
	<< llvm::join(CIRAttrToValueVisitorCaseTypes, ",\n ")
	<< ">(\n"
	<< " [&](auto attrT) { return visitCirAttr(attrT); })\n"
	<< " .Default([this](mlir::Attribute attr) {\n"
	<< " mlir::emitError(parentOp->getLoc(), \"unsupported CIR "
	"attribute in LLVM constant lowering\")\n"
	<< " << attr;\n"
	<< " return mlir::Value();\n"
	<< " });\n"
	<< " }\n";
	CIRAttrToValueVisitFunc = std::move(CodeBuffer);
	}

	void GenerateCIRAttrs(const Record *Record) {
	if (!Record->getValueAsBit("canHaveIllegalCXXABIType"))
	CXXABILoweringAttrAlwaysLegal.push_back(GetAttrCppClassRef(Record));
	if (Record->getValueAsBit("hasAttrToValueLowering"))
	GenerateAttrToValueVisitor(Record);
	}
	} // namespace

	void clang::EmitCIRLowering(const llvm::RecordKeeper &RK,
	llvm::raw_ostream &OS) {
	emitSourceFileHeader("Lowering patterns for CIR operations", OS);
	for (const auto *OpRecord : RK.getAllDerivedDefinitions("CIR_Op"))
	Generate(OpRecord);
	for (const auto *OpRecord : RK.getAllDerivedDefinitions("EnumAttr"))
	GenerateCIREnumAttrs(OpRecord);
	for (const auto *OpRecord : RK.getAllDerivedDefinitions("CIR_Attr"))
	GenerateCIRAttrs(OpRecord);
	GenerateAttrToValueVisitFunc();

	OS << "#ifdef GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS\n"
	<< CIRAttrToValueVisitFunc << "\n"
	<< llvm::join(CIRAttrToValueVisitorDecls, "\n") << "\n"
	<< "#endif // GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS\n\n";

	OS << "#ifdef GET_ABI_LOWERING_PATTERNS\n"
	<< llvm::join(CXXABILoweringPatterns, "\n") << "#endif\n\n";
	OS << "#ifdef GET_ABI_LOWERING_PATTERNS_LIST\n"
	<< llvm::join(CXXABILoweringPatternsList, ",\n") << "\n#endif\n\n";

	OS << "#ifdef GET_LLVM_LOWERING_PATTERNS\n"
	<< llvm::join(LLVMLoweringPatterns, "\n") << "#endif\n\n";
	OS << "#ifdef GET_LLVM_LOWERING_PATTERNS_LIST\n"
	<< llvm::join(LLVMLoweringPatternsList, ",\n") << "\n#endif\n\n";

	OS << "#ifdef CXX_ABI_ALWAYS_LEGAL_ATTRS\n"
	<< llvm::join(CXXABILoweringAttrAlwaysLegal, ",\n") << "\n#endif\n\n";
	}