//===- OpenMPDialect.cpp - MLIR Dialect for OpenMP implementation ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the OpenMP dialect and its operations.
//
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/SymbolTable.h"
#include "mlir/Interfaces/FoldInterfaces.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/STLForwardCompat.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/ADT/bit.h"
#include "llvm/Support/InterleavedRange.h"
#include <cstddef>
#include <iterator>
#include <optional>
#include <variant>

#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"

using namespace mlir;
using namespace mlir::omp;

static ArrayAttr makeArrayAttr(MLIRContext *context,
                               llvm::ArrayRef<Attribute> attrs) {
  return attrs.empty() ? nullptr : ArrayAttr::get(context, attrs);
}

static DenseBoolArrayAttr
makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef<bool> boolArray) {
  return boolArray.empty() ? nullptr : DenseBoolArrayAttr::get(ctx, boolArray);
}

static DenseI64ArrayAttr
makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef<int64_t> intArray) {
  return intArray.empty() ? nullptr : DenseI64ArrayAttr::get(ctx, intArray);
}

namespace {
struct MemRefPointerLikeModel
    : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
                                            MemRefType> {
  Type getElementType(Type pointer) const {
    return llvm::cast<MemRefType>(pointer).getElementType();
  }
};

struct LLVMPointerPointerLikeModel
    : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
                                            LLVM::LLVMPointerType> {
  Type getElementType(Type pointer) const { return Type(); }
};
} // namespace

/// Generate a name of a canonical loop nest of the format
/// `<prefix>(_r<idx>_s<idx>)*`. Hereby, `_r<idx>` identifies the region
/// argument index of an operation that has multiple regions, if the operation
/// has multiple regions.
/// `_s<idx>` identifies the position of an operation within a region, where
/// only operations that may potentially contain loops ("container operations"
/// i.e. have region arguments) are counted. Again, it is omitted if there is
/// only one such operation in a region. If there are canonical loops nested
/// inside each other, also may also use the format `_d<num>` where <num> is the
/// nesting depth of the loop.
///
/// The generated name is a best-effort to make canonical loop unique within an
/// SSA namespace. This also means that regions with IsolatedFromAbove property
/// do not consider any parents or siblings.
static std::string generateLoopNestingName(StringRef prefix,
                                           CanonicalLoopOp op) {
  struct Component {
    /// If true, this component describes a region operand of an operation (the
    /// operand's owner) If false, this component describes an operation located
    /// in a parent region
    bool isRegionArgOfOp;
    bool skip = false;
    bool isUnique = false;

    size_t idx;
    Operation *op;
    Region *parentRegion;
    size_t loopDepth;

    Operation *&getOwnerOp() {
      assert(isRegionArgOfOp && "Must describe a region operand");
      return op;
    }
    size_t &getArgIdx() {
      assert(isRegionArgOfOp && "Must describe a region operand");
      return idx;
    }

    Operation *&getContainerOp() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return op;
    }
    size_t &getOpPos() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return idx;
    }
    bool isLoopOp() const {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return isa<CanonicalLoopOp>(op);
    }
    Region *&getParentRegion() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return parentRegion;
    }
    size_t &getLoopDepth() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return loopDepth;
    }

    void skipIf(bool v = true) { skip = skip || v; }
  };

  // List of ancestors, from inner to outer.
  // Alternates between
  //  * region argument of an operation
  //  * operation within a region
  SmallVector<Component> components;

  // Gather a list of parent regions and operations, and the position within
  // their parent
  Operation *o = op.getOperation();
  while (o) {
    // Operation within a region
    Region *r = o->getParentRegion();
    if (!r)
      break;

    llvm::ReversePostOrderTraversal<Block *> traversal(&r->getBlocks().front());
    size_t idx = 0;
    bool found = false;
    size_t sequentialIdx = -1;
    bool isOnlyContainerOp = true;
    for (Block *b : traversal) {
      for (Operation &op : *b) {
        if (&op == o && !found) {
          sequentialIdx = idx;
          found = true;
        }
        if (op.getNumRegions()) {
          idx += 1;
          if (idx > 1)
            isOnlyContainerOp = false;
        }
        if (found && !isOnlyContainerOp)
          break;
      }
    }

    Component &containerOpInRegion = components.emplace_back();
    containerOpInRegion.isRegionArgOfOp = false;
    containerOpInRegion.isUnique = isOnlyContainerOp;
    containerOpInRegion.getContainerOp() = o;
    containerOpInRegion.getOpPos() = sequentialIdx;
    containerOpInRegion.getParentRegion() = r;

    Operation *parent = r->getParentOp();

    // Region argument of an operation
    Component &regionArgOfOperation = components.emplace_back();
    regionArgOfOperation.isRegionArgOfOp = true;
    regionArgOfOperation.isUnique = true;
    regionArgOfOperation.getArgIdx() = 0;
    regionArgOfOperation.getOwnerOp() = parent;

    // The IsolatedFromAbove trait of the parent operation implies that each
    // individual region argument has its own separate namespace, so no
    // ambiguity.
    if (!parent || parent->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
      break;

    // Component only needed if operation has multiple region operands. Region
    // arguments may be optional, but we currently do not consider this.
    if (parent->getRegions().size() > 1) {
      auto getRegionIndex = [](Operation *o, Region *r) {
        for (auto [idx, region] : llvm::enumerate(o->getRegions())) {
          if (&region == r)
            return idx;
        }
        llvm_unreachable("Region not child of its parent operation");
      };
      regionArgOfOperation.isUnique = false;
      regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);
    }

    // next parent
    o = parent;
  }

  // Determine whether a region-argument component is not needed
  for (Component &c : components)
    c.skipIf(c.isRegionArgOfOp && c.isUnique);

  // Find runs of nested loops and determine each loop's depth in the loop nest
  size_t numSurroundingLoops = 0;
  for (Component &c : llvm::reverse(components)) {
    if (c.skip)
      continue;

    // non-skipped multi-argument operands interrupt the loop nest
    if (c.isRegionArgOfOp) {
      numSurroundingLoops = 0;
      continue;
    }

    // Multiple loops in a region means each of them is the outermost loop of a
    // new loop nest
    if (!c.isUnique)
      numSurroundingLoops = 0;

    c.getLoopDepth() = numSurroundingLoops;

    // Next loop is surrounded by one more loop
    if (isa<CanonicalLoopOp>(c.getContainerOp()))
      numSurroundingLoops += 1;
  }

  // In loop nests, skip all but the innermost loop that contains the depth
  // number
  bool isLoopNest = false;
  for (Component &c : components) {
    if (c.skip || c.isRegionArgOfOp)
      continue;

    if (!isLoopNest && c.getLoopDepth() >= 1) {
      // Innermost loop of a loop nest of at least two loops
      isLoopNest = true;
    } else if (isLoopNest) {
      // Non-innermost loop of a loop nest
      c.skipIf(c.isUnique);

      // If there is no surrounding loop left, this must have been the outermost
      // loop; leave loop-nest mode for the next iteration
      if (c.getLoopDepth() == 0)
        isLoopNest = false;
    }
  }

  // Skip non-loop unambiguous regions (but they should interrupt loop nests, so
  // we mark them as skipped only after computing loop nests)
  for (Component &c : components)
    c.skipIf(!c.isRegionArgOfOp && c.isUnique &&
             !isa<CanonicalLoopOp>(c.getContainerOp()));

  // Components can be skipped if they are already disambiguated by their parent
  // (or does not have a parent)
  bool newRegion = true;
  for (Component &c : llvm::reverse(components)) {
    c.skipIf(newRegion && c.isUnique);

    // non-skipped components disambiguate unique children
    if (!c.skip)
      newRegion = true;

    // ...except canonical loops that need a suffix for each nest
    if (!c.isRegionArgOfOp && c.getContainerOp())
      newRegion = false;
  }

  // Compile the nesting name string
  SmallString<64> Name{prefix};
  llvm::raw_svector_ostream NameOS(Name);
  for (auto &c : llvm::reverse(components)) {
    if (c.skip)
      continue;

    if (c.isRegionArgOfOp)
      NameOS << "_r" << c.getArgIdx();
    else if (c.getLoopDepth() >= 1)
      NameOS << "_d" << c.getLoopDepth();
    else
      NameOS << "_s" << c.getOpPos();
  }

  return NameOS.str().str();
}

void OpenMPDialect::initialize() {
  addOperations<
#define GET_OP_LIST
#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
      >();
  addAttributes<
#define GET_ATTRDEF_LIST
#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
      >();
  addTypes<
#define GET_TYPEDEF_LIST
#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
      >();

  declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();

  MemRefType::attachInterface<MemRefPointerLikeModel>(*getContext());
  LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
      *getContext());

  // Attach default offload module interface to module op to access
  // offload functionality through
  mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(
      *getContext());

  // Attach default declare target interfaces to operations which can be marked
  // as declare target (Global Operations and Functions/Subroutines in dialects
  // that Fortran (or other languages that lower to MLIR) translates too
  mlir::LLVM::GlobalOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::GlobalOp>>(
      *getContext());
  mlir::LLVM::LLVMFuncOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::LLVMFuncOp>>(
      *getContext());
  mlir::func::FuncOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::func::FuncOp>>(*getContext());
}

//===----------------------------------------------------------------------===//
// Parser and printer for Allocate Clause
//===----------------------------------------------------------------------===//

/// Parse an allocate clause with allocators and a list of operands with types.
///
/// allocate-operand-list :: = allocate-operand |
///                            allocator-operand `,` allocate-operand-list
/// allocate-operand :: = ssa-id-and-type -> ssa-id-and-type
/// ssa-id-and-type ::= ssa-id `:` type
static ParseResult parseAllocateAndAllocator(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocateVars,
    SmallVectorImpl<Type> &allocateTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocatorVars,
    SmallVectorImpl<Type> &allocatorTypes) {

  return parser.parseCommaSeparatedList([&]() {
    OpAsmParser::UnresolvedOperand operand;
    Type type;
    if (parser.parseOperand(operand) || parser.parseColonType(type))
      return failure();
    allocatorVars.push_back(operand);
    allocatorTypes.push_back(type);
    if (parser.parseArrow())
      return failure();
    if (parser.parseOperand(operand) || parser.parseColonType(type))
      return failure();

    allocateVars.push_back(operand);
    allocateTypes.push_back(type);
    return success();
  });
}

/// Print allocate clause
static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op,
                                      OperandRange allocateVars,
                                      TypeRange allocateTypes,
                                      OperandRange allocatorVars,
                                      TypeRange allocatorTypes) {
  for (unsigned i = 0; i < allocateVars.size(); ++i) {
    std::string separator = i == allocateVars.size() - 1 ? "" : ", ";
    p << allocatorVars[i] << " : " << allocatorTypes[i] << " -> ";
    p << allocateVars[i] << " : " << allocateTypes[i] << separator;
  }
}

//===----------------------------------------------------------------------===//
// Parser and printer for a clause attribute (StringEnumAttr)
//===----------------------------------------------------------------------===//

template <typename ClauseAttr>
static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) {
  using ClauseT = decltype(std::declval<ClauseAttr>().getValue());
  StringRef enumStr;
  SMLoc loc = parser.getCurrentLocation();
  if (parser.parseKeyword(&enumStr))
    return failure();
  if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {
    attr = ClauseAttr::get(parser.getContext(), *enumValue);
    return success();
  }
  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
}

template <typename ClauseAttr>
static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) {
  p << stringifyEnum(attr.getValue());
}

//===----------------------------------------------------------------------===//
// Parser and printer for Linear Clause
//===----------------------------------------------------------------------===//

/// linear ::= `linear` `(` linear-list `)`
/// linear-list := linear-val | linear-val linear-list
/// linear-val := ssa-id-and-type `=` ssa-id-and-type
///           | `val` `(` ssa-id-and-type `=` ssa-id-and-type `)`
///           | `ref` `(` ssa-id-and-type `=` ssa-id-and-type `)`
///           | `uval` `(` ssa-id-and-type `=` ssa-id-and-type `)`
static ParseResult parseLinearClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
    SmallVectorImpl<Type> &linearTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
    SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers) {
  SmallVector<Attribute> modifiers;
  auto result = parser.parseCommaSeparatedList([&]() {
    OpAsmParser::UnresolvedOperand var;
    Type type, stepType;
    OpAsmParser::UnresolvedOperand stepVar;

    std::optional<omp::LinearModifier> linearModifier;
    if (succeeded(parser.parseOptionalKeyword("val"))) {
      linearModifier = omp::LinearModifier::val;
    } else if (succeeded(parser.parseOptionalKeyword("ref"))) {
      linearModifier = omp::LinearModifier::ref;
    } else if (succeeded(parser.parseOptionalKeyword("uval"))) {
      linearModifier = omp::LinearModifier::uval;
    }

    bool hasLinearModifierParens = linearModifier.has_value();
    if (hasLinearModifierParens && parser.parseLParen())
      return failure();

    if (parser.parseOperand(var) || parser.parseColonType(type) ||
        parser.parseEqual() || parser.parseOperand(stepVar) ||
        parser.parseColonType(stepType))
      return failure();

    if (hasLinearModifierParens && parser.parseRParen())
      return failure();

    linearVars.push_back(var);
    linearTypes.push_back(type);
    linearStepVars.push_back(stepVar);
    linearStepTypes.push_back(stepType);
    if (linearModifier) {
      modifiers.push_back(
          omp::LinearModifierAttr::get(parser.getContext(), *linearModifier));
    } else {
      modifiers.push_back(UnitAttr::get(parser.getContext()));
    }
    return success();
  });
  if (failed(result))
    return failure();
  linearModifiers = ArrayAttr::get(parser.getContext(), modifiers);
  return success();
}

/// Print Linear Clause
static void printLinearClause(OpAsmPrinter &p, Operation *op,
                              ValueRange linearVars, TypeRange linearTypes,
                              ValueRange linearStepVars, TypeRange stepVarTypes,
                              ArrayAttr linearModifiers) {
  size_t linearVarsSize = linearVars.size();
  for (unsigned i = 0; i < linearVarsSize; ++i) {
    if (i != 0)
      p << ", ";
    // Print modifier keyword wrapper if present.
    Attribute modAttr = linearModifiers ? linearModifiers[i] : nullptr;
    auto mod = modAttr ? dyn_cast<omp::LinearModifierAttr>(modAttr) : nullptr;
    if (mod) {
      p << omp::stringifyLinearModifier(mod.getValue()) << "(";
    }
    p << linearVars[i] << " : " << linearTypes[i];
    p << " = " << linearStepVars[i] << " : " << stepVarTypes[i];
    if (mod)
      p << ")";
  }
}

//===----------------------------------------------------------------------===//
// Verifier for Linear modifier
//===----------------------------------------------------------------------===//

/// OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or
/// uval only on a declare simd directive."
/// Also verifies that modifier count matches variable count.
static LogicalResult
verifyLinearModifiers(Operation *op, std::optional<ArrayAttr> linearModifiers,
                      OperandRange linearVars, bool isDeclareSimd = false) {
  if (!linearModifiers)
    return success();
  if (linearModifiers->size() != linearVars.size())
    return op->emitOpError()
           << "expected as many linear modifiers as linear variables";
  if (!isDeclareSimd) {
    for (Attribute attr : *linearModifiers) {
      if (!attr)
        continue;
      auto modAttr = dyn_cast<omp::LinearModifierAttr>(attr);
      if (!modAttr)
        continue;
      omp::LinearModifier mod = modAttr.getValue();
      if (mod == omp::LinearModifier::ref || mod == omp::LinearModifier::uval)
        return op->emitOpError()
               << "linear modifier '" << omp::stringifyLinearModifier(mod)
               << "' may only be specified on a declare simd directive";
    }
  }
  return success();
}

//===----------------------------------------------------------------------===//
// Verifier for Nontemporal Clause
//===----------------------------------------------------------------------===//

static LogicalResult verifyNontemporalClause(Operation *op,
                                             OperandRange nontemporalVars) {

  // Check if each var is unique - OpenMP 5.0 -> 2.9.3.1 section
  DenseSet<Value> nontemporalItems;
  for (const auto &it : nontemporalVars)
    if (!nontemporalItems.insert(it).second)
      return op->emitOpError() << "nontemporal variable used more than once";

  return success();
}

//===----------------------------------------------------------------------===//
// Parser, verifier and printer for Aligned Clause
//===----------------------------------------------------------------------===//
static LogicalResult verifyAlignedClause(Operation *op,
                                         std::optional<ArrayAttr> alignments,
                                         OperandRange alignedVars) {
  // Check if number of alignment values equals to number of aligned variables
  if (!alignedVars.empty()) {
    if (!alignments || alignments->size() != alignedVars.size())
      return op->emitOpError()
             << "expected as many alignment values as aligned variables";
  } else {
    if (alignments)
      return op->emitOpError() << "unexpected alignment values attribute";
    return success();
  }

  // Check if each var is aligned only once - OpenMP 4.5 -> 2.8.1 section
  DenseSet<Value> alignedItems;
  for (auto it : alignedVars)
    if (!alignedItems.insert(it).second)
      return op->emitOpError() << "aligned variable used more than once";

  if (!alignments)
    return success();

  // Check if all alignment values are positive - OpenMP 4.5 -> 2.8.1 section
  for (unsigned i = 0; i < (*alignments).size(); ++i) {
    if (auto intAttr = llvm::dyn_cast<IntegerAttr>((*alignments)[i])) {
      if (intAttr.getValue().sle(0))
        return op->emitOpError() << "alignment should be greater than 0";
    } else {
      return op->emitOpError() << "expected integer alignment";
    }
  }

  return success();
}

/// aligned ::= `aligned` `(` aligned-list `)`
/// aligned-list := aligned-val | aligned-val aligned-list
/// aligned-val := ssa-id-and-type `->` alignment
static ParseResult
parseAlignedClause(OpAsmParser &parser,
                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &alignedVars,
                   SmallVectorImpl<Type> &alignedTypes,
                   ArrayAttr &alignmentsAttr) {
  SmallVector<Attribute> alignmentVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        if (parser.parseOperand(alignedVars.emplace_back()) ||
            parser.parseColonType(alignedTypes.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseAttribute(alignmentVec.emplace_back())) {
          return failure();
        }
        return success();
      })))
    return failure();
  SmallVector<Attribute> alignments(alignmentVec.begin(), alignmentVec.end());
  alignmentsAttr = ArrayAttr::get(parser.getContext(), alignments);
  return success();
}

/// Print Aligned Clause
static void printAlignedClause(OpAsmPrinter &p, Operation *op,
                               ValueRange alignedVars, TypeRange alignedTypes,
                               std::optional<ArrayAttr> alignments) {
  for (unsigned i = 0; i < alignedVars.size(); ++i) {
    if (i != 0)
      p << ", ";
    p << alignedVars[i] << " : " << alignedVars[i].getType();
    p << " -> " << (*alignments)[i];
  }
}

static LogicalResult verifyAllocateClause(
    Operation *op, ValueRange allocateVars, ValueRange allocatorVars,
    DenseI64ArrayAttr allocateAlignments,
    DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars = {},
    ArrayAttr privateSyms = nullptr, bool requirePrivateIndices = false) {
  if (allocateVars.size() != allocatorVars.size())
    return op->emitError(
        "expected equal sizes for allocate and allocator variables");

  if (allocateVars.empty()) {
    if (allocateAlignments)
      return op->emitError(
          "unexpected allocate alignments without allocate variables");
    if (allocatePrivateIndices)
      return op->emitError(
          "unexpected allocate private indices without allocate variables");
    return success();
  }

  if (allocateAlignments) {
    ArrayRef<int64_t> alignments = allocateAlignments.asArrayRef();
    if (alignments.size() != allocateVars.size())
      return op->emitError(
          "expected as many allocate alignments as allocate variables");
    for (int64_t alignment : alignments) {
      if (alignment < 0)
        return op->emitError("expected non-negative allocate alignments");
      if (alignment != 0 && (alignment & (alignment - 1)) != 0)
        return op->emitError(
            "expected positive allocate alignments to be powers of two");
    }
  }

  if (!allocatePrivateIndices) {
    if (requirePrivateIndices)
      return op->emitError(
          "expected an allocate private index for each allocate variable");
    return success();
  }

  ArrayRef<int64_t> indices = allocatePrivateIndices.asArrayRef();
  if (indices.size() != allocateVars.size())
    return op->emitError(
        "expected as many allocate private indices as allocate variables");

  DenseSet<int64_t> usedPrivateSlots;
  for (auto [allocateVar, privateIndex] :
       llvm::zip_equal(allocateVars, indices)) {
    if (privateIndex < 0 ||
        static_cast<uint64_t>(privateIndex) >= privateVars.size())
      return op->emitError("allocate private index is out of range");
    if (!usedPrivateSlots.insert(privateIndex).second)
      return op->emitError(
          "allocate private index refers to a private variable more than once");

    Value privateVar = privateVars[privateIndex];
    if (allocateVar.getType() != privateVar.getType())
      return op->emitError()
             << "type mismatch between allocate variable and private variable "
                "at index "
             << privateIndex;

    if (!privateSyms ||
        static_cast<uint64_t>(privateIndex) >= privateSyms.size())
      return op->emitError(
          "allocate private index does not have a privatizer symbol");

    auto privateSym = dyn_cast<SymbolRefAttr>(privateSyms[privateIndex]);
    if (!privateSym)
      return op->emitError(
          "allocate private index does not reference a privatizer symbol");
    PrivateClauseOp privatizer =
        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);
    if (!privatizer)
      return op->emitError() << "failed to lookup privatizer op with symbol: '"
                             << privateSym << "'";
    if (privatizer.getDataSharingType() != DataSharingClauseType::Private &&
        privatizer.getDataSharingType() != DataSharingClauseType::FirstPrivate)
      return op->emitError(
          "allocate private index must refer to private or firstprivate "
          "storage");
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Schedule Clause
//===----------------------------------------------------------------------===//

static ParseResult
verifyScheduleModifiers(OpAsmParser &parser,
                        SmallVectorImpl<SmallString<12>> &modifiers) {
  if (modifiers.size() > 2)
    return parser.emitError(parser.getNameLoc()) << " unexpected modifier(s)";
  for (const auto &mod : modifiers) {
    // Translate the string. If it has no value, then it was not a valid
    // modifier!
    auto symbol = symbolizeScheduleModifier(mod);
    if (!symbol)
      return parser.emitError(parser.getNameLoc())
             << " unknown modifier type: " << mod;
  }

  // If we have one modifier that is "simd", then stick a "none" modiifer in
  // index 0.
  if (modifiers.size() == 1) {
    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd) {
      modifiers.push_back(modifiers[0]);
      modifiers[0] = stringifyScheduleModifier(ScheduleModifier::none);
    }
  } else if (modifiers.size() == 2) {
    // If there are two modifier:
    // First modifier should not be simd, second one should be simd
    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd ||
        symbolizeScheduleModifier(modifiers[1]) != ScheduleModifier::simd)
      return parser.emitError(parser.getNameLoc())
             << " incorrect modifier order";
  }
  return success();
}

/// schedule ::= `schedule` `(` sched-list `)`
/// sched-list ::= sched-val | sched-val sched-list |
///                sched-val `,` sched-modifier
/// sched-val ::= sched-with-chunk | sched-wo-chunk
/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?
/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`
/// sched-wo-chunk ::=  `auto` | `runtime`
/// sched-modifier ::=  sched-mod-val | sched-mod-val `,` sched-mod-val
/// sched-mod-val ::=  `monotonic` | `nonmonotonic` | `simd` | `none`
static ParseResult
parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,
                    ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
                    std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
                    Type &chunkType) {
  StringRef keyword;
  if (parser.parseKeyword(&keyword))
    return failure();
  std::optional<mlir::omp::ClauseScheduleKind> schedule =
      symbolizeClauseScheduleKind(keyword);
  if (!schedule)
    return parser.emitError(parser.getNameLoc()) << " expected schedule kind";

  scheduleAttr = ClauseScheduleKindAttr::get(parser.getContext(), *schedule);
  switch (*schedule) {
  case ClauseScheduleKind::Static:
  case ClauseScheduleKind::Dynamic:
  case ClauseScheduleKind::Guided:
    if (succeeded(parser.parseOptionalEqual())) {
      chunkSize = OpAsmParser::UnresolvedOperand{};
      if (parser.parseOperand(*chunkSize) || parser.parseColonType(chunkType))
        return failure();
    } else {
      chunkSize = std::nullopt;
    }
    break;
  case ClauseScheduleKind::Auto:
  case ClauseScheduleKind::Runtime:
  case ClauseScheduleKind::Distribute:
    chunkSize = std::nullopt;
  }

  // If there is a comma, we have one or more modifiers..
  SmallVector<SmallString<12>> modifiers;
  while (succeeded(parser.parseOptionalComma())) {
    StringRef mod;
    if (parser.parseKeyword(&mod))
      return failure();
    modifiers.push_back(mod);
  }

  if (verifyScheduleModifiers(parser, modifiers))
    return failure();

  if (!modifiers.empty()) {
    SMLoc loc = parser.getCurrentLocation();
    if (std::optional<ScheduleModifier> mod =
            symbolizeScheduleModifier(modifiers[0])) {
      scheduleMod = ScheduleModifierAttr::get(parser.getContext(), *mod);
    } else {
      return parser.emitError(loc, "invalid schedule modifier");
    }
    // Only SIMD attribute is allowed here!
    if (modifiers.size() > 1) {
      assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);
      scheduleSimd = UnitAttr::get(parser.getBuilder().getContext());
    }
  }

  return success();
}

/// Print schedule clause
static void printScheduleClause(OpAsmPrinter &p, Operation *op,
                                ClauseScheduleKindAttr scheduleKind,
                                ScheduleModifierAttr scheduleMod,
                                UnitAttr scheduleSimd, Value scheduleChunk,
                                Type scheduleChunkType) {
  p << stringifyClauseScheduleKind(scheduleKind.getValue());
  if (scheduleChunk)
    p << " = " << scheduleChunk << " : " << scheduleChunk.getType();
  if (scheduleMod)
    p << ", " << stringifyScheduleModifier(scheduleMod.getValue());
  if (scheduleSimd)
    p << ", simd";
}

//===----------------------------------------------------------------------===//
// Parser and printer for Order Clause
//===----------------------------------------------------------------------===//

// order ::= `order` `(` [order-modiﬁer ':'] concurrent `)`
// order-modiﬁer ::= reproducible | unconstrained
static ParseResult parseOrderClause(OpAsmParser &parser,
                                    ClauseOrderKindAttr &order,
                                    OrderModifierAttr &orderMod) {
  StringRef enumStr;
  SMLoc loc = parser.getCurrentLocation();
  if (parser.parseKeyword(&enumStr))
    return failure();
  if (std::optional<OrderModifier> enumValue =
          symbolizeOrderModifier(enumStr)) {
    orderMod = OrderModifierAttr::get(parser.getContext(), *enumValue);
    if (parser.parseOptionalColon())
      return failure();
    loc = parser.getCurrentLocation();
    if (parser.parseKeyword(&enumStr))
      return failure();
  }
  if (std::optional<ClauseOrderKind> enumValue =
          symbolizeClauseOrderKind(enumStr)) {
    order = ClauseOrderKindAttr::get(parser.getContext(), *enumValue);
    return success();
  }
  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
}

static void printOrderClause(OpAsmPrinter &p, Operation *op,
                             ClauseOrderKindAttr order,
                             OrderModifierAttr orderMod) {
  if (orderMod)
    p << stringifyOrderModifier(orderMod.getValue()) << ":";
  if (order)
    p << stringifyClauseOrderKind(order.getValue());
}

template <typename ClauseTypeAttr, typename ClauseType>
static ParseResult
parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness,
                       std::optional<OpAsmParser::UnresolvedOperand> &operand,
                       Type &operandType,
                       std::optional<ClauseType> (*symbolizeClause)(StringRef),
                       StringRef clauseName) {
  StringRef enumStr;
  if (succeeded(parser.parseOptionalKeyword(&enumStr))) {
    if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {
      prescriptiveness = ClauseTypeAttr::get(parser.getContext(), *enumValue);
      if (parser.parseComma())
        return failure();
    } else {
      return parser.emitError(parser.getCurrentLocation())
             << "invalid " << clauseName << " modifier : '" << enumStr << "'";
      ;
    }
  }

  OpAsmParser::UnresolvedOperand var;
  if (succeeded(parser.parseOperand(var))) {
    operand = var;
  } else {
    return parser.emitError(parser.getCurrentLocation())
           << "expected " << clauseName << " operand";
  }

  if (operand.has_value()) {
    if (parser.parseColonType(operandType))
      return failure();
  }

  return success();
}

template <typename ClauseTypeAttr, typename ClauseType>
static void
printGranularityClause(OpAsmPrinter &p, Operation *op,
                       ClauseTypeAttr prescriptiveness, Value operand,
                       mlir::Type operandType,
                       StringRef (*stringifyClauseType)(ClauseType)) {

  if (prescriptiveness)
    p << stringifyClauseType(prescriptiveness.getValue()) << ", ";

  if (operand)
    p << operand << ": " << operandType;
}

//===----------------------------------------------------------------------===//
// Parser and printer for grainsize Clause
//===----------------------------------------------------------------------===//

// grainsize ::= `grainsize` `(` [strict ':'] grain-size `)`
static ParseResult
parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod,
                     std::optional<OpAsmParser::UnresolvedOperand> &grainsize,
                     Type &grainsizeType) {
  return parseGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
      parser, grainsizeMod, grainsize, grainsizeType,
      &symbolizeClauseGrainsizeType, "grainsize");
}

static void printGrainsizeClause(OpAsmPrinter &p, Operation *op,
                                 ClauseGrainsizeTypeAttr grainsizeMod,
                                 Value grainsize, mlir::Type grainsizeType) {
  printGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
      p, op, grainsizeMod, grainsize, grainsizeType,
      &stringifyClauseGrainsizeType);
}

//===----------------------------------------------------------------------===//
// Parser and printer for num_tasks Clause
//===----------------------------------------------------------------------===//

// numtask ::= `num_tasks` `(` [strict ':'] num-tasks `)`
static ParseResult
parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod,
                    std::optional<OpAsmParser::UnresolvedOperand> &numTasks,
                    Type &numTasksType) {
  return parseGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
      parser, numTasksMod, numTasks, numTasksType, &symbolizeClauseNumTasksType,
      "num_tasks");
}

static void printNumTasksClause(OpAsmPrinter &p, Operation *op,
                                ClauseNumTasksTypeAttr numTasksMod,
                                Value numTasks, mlir::Type numTasksType) {
  printGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
      p, op, numTasksMod, numTasks, numTasksType, &stringifyClauseNumTasksType);
}

//===----------------------------------------------------------------------===//
// Parser and printer for Heap Alloc Clause
//===----------------------------------------------------------------------===//

/// operation ::= $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
static ParseResult parseHeapAllocClause(
    OpAsmParser &parser, TypeAttr &inTypeAttr,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &typeparams,
    SmallVectorImpl<Type> &typeparamsTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &shape,
    SmallVectorImpl<Type> &shapeTypes) {
  mlir::Type inType;
  if (parser.parseType(inType))
    return mlir::failure();
  inTypeAttr = TypeAttr::get(inType);

  if (!parser.parseOptionalLParen()) {
    // parse the LEN params of the derived type. (<params> : <types>)
    if (parser.parseOperandList(typeparams, OpAsmParser::Delimiter::None) ||
        parser.parseColonTypeList(typeparamsTypes) || parser.parseRParen())
      return failure();
  }

  if (!parser.parseOptionalComma()) {
    // parse size to scale by, vector of n dimensions of type index
    if (parser.parseOperandList(shape, OpAsmParser::Delimiter::None))
      return failure();

    // TODO: This overrides the actual types of the operands, which might cause
    // issues when they don't match. At the moment this is done in place of
    // making the corresponding operand type `Variadic<Index>` because index
    // types are lowered to I64 prior to LLVM IR translation.
    shapeTypes.append(shape.size(), IndexType::get(parser.getContext()));
  }

  return success();
}

static void printHeapAllocClause(OpAsmPrinter &p, Operation *op,
                                 TypeAttr inType, ValueRange typeparams,
                                 TypeRange typeparamsTypes, ValueRange shape,
                                 TypeRange shapeTypes) {
  p << inType;
  if (!typeparams.empty()) {
    p << '(' << typeparams << " : " << typeparamsTypes << ')';
  }
  for (auto sh : shape) {
    p << ", ";
    p.printOperand(sh);
  }
}

//===----------------------------------------------------------------------===//
// Parser, printer and verify for dyn_groupprivate Clause
//===----------------------------------------------------------------------===//

static LogicalResult
verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup,
                            FallbackModifierAttr fallback,
                            Value dynGroupprivateSize) {
  if (!dynGroupprivateSize && (accessGroup || fallback))
    return op->emitOpError("dyn_groupprivate modifiers require a size operand");

  return success();
}

static ParseResult parseDynGroupprivateClause(
    OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr,
    FallbackModifierAttr &fallbackAttr,
    std::optional<OpAsmParser::UnresolvedOperand> &dynGroupprivateSize,
    Type &sizeType) {

  bool parsedAccessGroup = false;
  bool parsedFallback = false;
  bool parsedSize = false;

  return parser.parseCommaSeparatedList([&]() -> ParseResult {
    // Parse AccessGroupModifier.
    if (succeeded(parser.parseOptionalKeyword("cgroup"))) {
      if (parsedAccessGroup)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate access group modifier");
      accessGroupAttr = AccessGroupModifierAttr::get(
          parser.getContext(), AccessGroupModifier::cgroup);
      parsedAccessGroup = true;
      return success();
    }
    // Parse FallbackModifier.
    if (succeeded(parser.parseOptionalKeyword("fallback"))) {
      if (parsedFallback)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate fallback modifier");
      if (parser.parseLParen())
        return parser.emitError(parser.getCurrentLocation(),
                                "expected '(' after 'fallback'");
      llvm::StringRef fbKind;
      if (parser.parseKeyword(&fbKind))
        return parser.emitError(
            parser.getCurrentLocation(),
            "expected fallback modifier (abort/null/default_mem)");
      std::optional<FallbackModifier> fbEnum;
      if (fbKind == "abort")
        fbEnum = FallbackModifier::abort;
      else if (fbKind == "null")
        fbEnum = FallbackModifier::null;
      else if (fbKind == "default_mem")
        fbEnum = FallbackModifier::default_mem;
      else
        return parser.emitError(parser.getCurrentLocation(),
                                "invalid fallback modifier '" + fbKind + "'");
      fallbackAttr = FallbackModifierAttr::get(parser.getContext(), *fbEnum);
      if (parser.parseRParen())
        return parser.emitError(parser.getCurrentLocation(),
                                "expected ')' after fallback modifier");
      parsedFallback = true;
      return success();
    }
    // Parse size operand.
    OpAsmParser::UnresolvedOperand operand;
    if (succeeded(parser.parseOperand(operand))) {
      if (parsedSize)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate size operand");
      dynGroupprivateSize = operand;
      parsedSize = true;
      if (failed(parser.parseColon()) || failed(parser.parseType(sizeType)))
        return parser.emitError(parser.getCurrentLocation(),
                                "expected ':' and type after size operand");
      return success();
    }
    return parser.emitError(parser.getCurrentLocation(),
                            "expected dyn_groupprivate_size operand");
  });
}

static void printDynGroupprivateClause(OpAsmPrinter &printer, Operation *op,
                                       AccessGroupModifierAttr modifierFirst,
                                       FallbackModifierAttr modifierSecond,
                                       Value dynGroupprivateSize,
                                       Type sizeType) {

  bool needsComma = false;

  if (modifierFirst) {
    printer << modifierFirst.getValue();
    needsComma = true;
  }

  if (modifierSecond) {
    if (needsComma)
      printer << ", ";
    printer << "fallback(";
    printer << modifierSecond.getValue();
    printer << ")";
    needsComma = true;
  }

  if (dynGroupprivateSize) {
    if (needsComma)
      printer << ", ";
    printer << dynGroupprivateSize << " : " << sizeType;
  }
}

//===----------------------------------------------------------------------===//
// Parser and printer for in_reduction Clause
//===----------------------------------------------------------------------===//

/// Parses an `in_reduction` clause for an operation that does not give its
/// list items entry block arguments (e.g. `omp.target`). The expected format is
/// a comma-separated list of `[byref] @sym %var` followed by `: types`.
static ParseResult parseInReductionClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms) {
  SmallVector<SymbolRefAttr> symbolVec;
  SmallVector<bool> isByRefVec;

  if (parser.parseCommaSeparatedList([&]() {
        isByRefVec.push_back(parser.parseOptionalKeyword("byref").succeeded());
        if (parser.parseAttribute(symbolVec.emplace_back()) ||
            parser.parseOperand(inReductionVars.emplace_back()))
          return failure();
        return success();
      }))
    return failure();

  if (parser.parseColon())
    return failure();

  if (parser.parseCommaSeparatedList(
          [&]() { return parser.parseType(inReductionTypes.emplace_back()); }))
    return failure();

  if (inReductionVars.size() != inReductionTypes.size())
    return failure();

  inReductionByref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
  SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
  inReductionSyms = ArrayAttr::get(parser.getContext(), symbolAttrs);
  return success();
}

/// Prints an `in_reduction` clause for an operation that does not give its list
/// items entry block arguments (e.g. `omp.target`). Mirrors
/// `parseInReductionClause`.
static void printInReductionClause(OpAsmPrinter &p, Operation *op,
                                   ValueRange inReductionVars,
                                   TypeRange inReductionTypes,
                                   DenseBoolArrayAttr inReductionByref,
                                   ArrayAttr inReductionSyms) {
  MLIRContext *ctx = op->getContext();

  ArrayAttr syms = inReductionSyms;
  if (!syms) {
    SmallVector<Attribute> values(inReductionVars.size(), nullptr);
    syms = ArrayAttr::get(ctx, values);
  }

  DenseBoolArrayAttr byref = inReductionByref;
  if (!byref) {
    SmallVector<bool> values(inReductionVars.size(), false);
    byref = DenseBoolArrayAttr::get(ctx, values);
  }

  llvm::interleaveComma(
      llvm::zip_equal(inReductionVars, syms.getValue(), byref.asArrayRef()), p,
      [&p](auto t) {
        auto [var, sym, isByRef] = t;
        if (isByRef)
          p << "byref ";
        if (sym)
          p << sym << " ";
        p << var;
      });
  p << " : ";
  llvm::interleaveComma(inReductionTypes, p);
}

//===----------------------------------------------------------------------===//
// Parsers for operations including clauses that define entry block arguments.
//===----------------------------------------------------------------------===//

namespace {
struct MapParseArgs {
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  SmallVectorImpl<Type> &types;
  MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
               SmallVectorImpl<Type> &types)
      : vars(vars), types(types) {}
};
struct PrivateParseArgs {
  llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  llvm::SmallVectorImpl<Type> &types;
  ArrayAttr &syms;
  UnitAttr &needsBarrier;
  DenseI64ArrayAttr *mapIndices;
  PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
                   SmallVectorImpl<Type> &types, ArrayAttr &syms,
                   UnitAttr &needsBarrier,
                   DenseI64ArrayAttr *mapIndices = nullptr)
      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
        mapIndices(mapIndices) {}
};

struct ReductionParseArgs {
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  SmallVectorImpl<Type> &types;
  DenseBoolArrayAttr &byref;
  ArrayAttr &syms;
  ReductionModifierAttr *modifier;
  ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
                     SmallVectorImpl<Type> &types, DenseBoolArrayAttr &byref,
                     ArrayAttr &syms, ReductionModifierAttr *mod = nullptr)
      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
};

struct AllRegionParseArgs {
  std::optional<MapParseArgs> hasDeviceAddrArgs;
  std::optional<MapParseArgs> hostEvalArgs;
  std::optional<ReductionParseArgs> inReductionArgs;
  std::optional<MapParseArgs> mapArgs;
  std::optional<PrivateParseArgs> privateArgs;
  std::optional<ReductionParseArgs> reductionArgs;
  std::optional<ReductionParseArgs> taskReductionArgs;
  std::optional<MapParseArgs> useDeviceAddrArgs;
  std::optional<MapParseArgs> useDevicePtrArgs;
};
} // namespace

static inline constexpr StringRef getPrivateNeedsBarrierSpelling() {
  return "private_barrier";
}

static ParseResult parseClauseWithRegionArgs(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
    SmallVectorImpl<Type> &types,
    SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,
    ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,
    DenseBoolArrayAttr *byref = nullptr,
    ReductionModifierAttr *modifier = nullptr,
    UnitAttr *needsBarrier = nullptr) {
  SmallVector<SymbolRefAttr> symbolVec;
  SmallVector<int64_t> mapIndicesVec;
  SmallVector<bool> isByRefVec;
  unsigned regionArgOffset = regionPrivateArgs.size();

  if (parser.parseLParen())
    return failure();

  if (modifier && succeeded(parser.parseOptionalKeyword("mod"))) {
    StringRef enumStr;
    if (parser.parseColon() || parser.parseKeyword(&enumStr) ||
        parser.parseComma())
      return failure();
    std::optional<ReductionModifier> enumValue =
        symbolizeReductionModifier(enumStr);
    if (!enumValue.has_value())
      return failure();
    *modifier = ReductionModifierAttr::get(parser.getContext(), *enumValue);
    if (!*modifier)
      return failure();
  }

  if (parser.parseCommaSeparatedList([&]() {
        if (byref)
          isByRefVec.push_back(
              parser.parseOptionalKeyword("byref").succeeded());

        if (symbols && parser.parseAttribute(symbolVec.emplace_back()))
          return failure();

        if (parser.parseOperand(operands.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseArgument(regionPrivateArgs.emplace_back()))
          return failure();

        if (mapIndices) {
          if (parser.parseOptionalLSquare().succeeded()) {
            if (parser.parseKeyword("map_idx") || parser.parseEqual() ||
                parser.parseInteger(mapIndicesVec.emplace_back()) ||
                parser.parseRSquare())
              return failure();
          } else {
            mapIndicesVec.push_back(-1);
          }
        }

        return success();
      }))
    return failure();

  if (parser.parseColon())
    return failure();

  if (parser.parseCommaSeparatedList([&]() {
        if (parser.parseType(types.emplace_back()))
          return failure();

        return success();
      }))
    return failure();

  if (operands.size() != types.size())
    return failure();

  if (parser.parseRParen())
    return failure();

  if (needsBarrier) {
    if (parser.parseOptionalKeyword(getPrivateNeedsBarrierSpelling())
            .succeeded())
      *needsBarrier = mlir::UnitAttr::get(parser.getContext());
  }

  auto *argsBegin = regionPrivateArgs.begin();
  MutableArrayRef argsSubrange(argsBegin + regionArgOffset,
                               argsBegin + regionArgOffset + types.size());
  for (auto [prv, type] : llvm::zip_equal(argsSubrange, types)) {
    prv.type = type;
  }

  if (symbols) {
    SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
    *symbols = ArrayAttr::get(parser.getContext(), symbolAttrs);
  }

  if (!mapIndicesVec.empty())
    *mapIndices =
        mlir::DenseI64ArrayAttr::get(parser.getContext(), mapIndicesVec);

  if (byref)
    *byref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);

  return success();
}

static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<MapParseArgs> mapArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!mapArgs)
      return failure();

    if (failed(parseClauseWithRegionArgs(parser, mapArgs->vars, mapArgs->types,
                                         entryBlockArgs)))
      return failure();
  }
  return success();
}

static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!privateArgs)
      return failure();

    if (failed(parseClauseWithRegionArgs(
            parser, privateArgs->vars, privateArgs->types, entryBlockArgs,
            &privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
            /*modifier=*/nullptr, &privateArgs->needsBarrier)))
      return failure();
  }
  return success();
}

static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!reductionArgs)
      return failure();
    if (failed(parseClauseWithRegionArgs(
            parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,
            &reductionArgs->syms, /*mapIndices=*/nullptr, &reductionArgs->byref,
            reductionArgs->modifier)))
      return failure();
  }
  return success();
}

static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
                                       AllRegionParseArgs args) {
  llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "has_device_addr",
                                 args.hasDeviceAddrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `has_device_addr` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "host_eval",
                                 args.hostEvalArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `host_eval` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "in_reduction",
                                 args.inReductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `in_reduction` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",
                                 args.mapArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `map_entries` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "private",
                                 args.privateArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `private` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "reduction",
                                 args.reductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `reduction` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "task_reduction",
                                 args.taskReductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `task_reduction` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_addr",
                                 args.useDeviceAddrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `use_device_addr` format";

  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_ptr",
                                 args.useDevicePtrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `use_device_addr` format";

  return parser.parseRegion(region, entryBlockArgs);
}

// These parseXyz functions correspond to the custom<Xyz> definitions
// in the .td file(s).
static ParseResult parseTargetOpRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hasDeviceAddrVars,
    SmallVectorImpl<Type> &hasDeviceAddrTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hostEvalVars,
    SmallVectorImpl<Type> &hostEvalTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,
    SmallVectorImpl<Type> &mapTypes,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps) {
  AllRegionParseArgs args;
  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
  args.mapArgs.emplace(mapVars, mapTypes);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier, &privateMaps);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parseInReductionPrivateRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier) {
  AllRegionParseArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parseInReductionPrivateReductionRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
    ArrayAttr &reductionSyms) {
  AllRegionParseArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, &reductionMod);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parsePrivateRegion(
    OpAsmParser &parser, Region &region,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier) {
  AllRegionParseArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parsePrivateReductionRegion(
    OpAsmParser &parser, Region &region,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
    ArrayAttr &reductionSyms) {
  AllRegionParseArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, &reductionMod);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parseTaskReductionRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
    SmallVectorImpl<Type> &taskReductionTypes,
    DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) {
  AllRegionParseArgs args;
  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
                                 taskReductionByref, taskReductionSyms);
  return parseBlockArgRegion(parser, region, args);
}

static ParseResult parseUseDeviceAddrUseDevicePtrRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDeviceAddrVars,
    SmallVectorImpl<Type> &useDeviceAddrTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDevicePtrVars,
    SmallVectorImpl<Type> &useDevicePtrTypes) {
  AllRegionParseArgs args;
  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
  return parseBlockArgRegion(parser, region, args);
}

//===----------------------------------------------------------------------===//
// Printers for operations including clauses that define entry block arguments.
//===----------------------------------------------------------------------===//

namespace {
struct MapPrintArgs {
  ValueRange vars;
  TypeRange types;
  MapPrintArgs(ValueRange vars, TypeRange types) : vars(vars), types(types) {}
};
struct PrivatePrintArgs {
  ValueRange vars;
  TypeRange types;
  ArrayAttr syms;
  UnitAttr needsBarrier;
  DenseI64ArrayAttr mapIndices;
  PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,
                   UnitAttr needsBarrier, DenseI64ArrayAttr mapIndices)
      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
        mapIndices(mapIndices) {}
};
struct ReductionPrintArgs {
  ValueRange vars;
  TypeRange types;
  DenseBoolArrayAttr byref;
  ArrayAttr syms;
  ReductionModifierAttr modifier;
  ReductionPrintArgs(ValueRange vars, TypeRange types, DenseBoolArrayAttr byref,
                     ArrayAttr syms, ReductionModifierAttr mod = nullptr)
      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
};
struct AllRegionPrintArgs {
  std::optional<MapPrintArgs> hasDeviceAddrArgs;
  std::optional<MapPrintArgs> hostEvalArgs;
  std::optional<ReductionPrintArgs> inReductionArgs;
  std::optional<MapPrintArgs> mapArgs;
  std::optional<PrivatePrintArgs> privateArgs;
  std::optional<ReductionPrintArgs> reductionArgs;
  std::optional<ReductionPrintArgs> taskReductionArgs;
  std::optional<MapPrintArgs> useDeviceAddrArgs;
  std::optional<MapPrintArgs> useDevicePtrArgs;
};
} // namespace

static void printClauseWithRegionArgs(
    OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
    ValueRange argsSubrange, ValueRange operands, TypeRange types,
    ArrayAttr symbols = nullptr, DenseI64ArrayAttr mapIndices = nullptr,
    DenseBoolArrayAttr byref = nullptr,
    ReductionModifierAttr modifier = nullptr, UnitAttr needsBarrier = nullptr) {
  if (argsSubrange.empty())
    return;

  p << clauseName << "(";

  if (modifier)
    p << "mod: " << stringifyReductionModifier(modifier.getValue()) << ", ";

  if (!symbols) {
    llvm::SmallVector<Attribute> values(operands.size(), nullptr);
    symbols = ArrayAttr::get(ctx, values);
  }

  if (!mapIndices) {
    llvm::SmallVector<int64_t> values(operands.size(), -1);
    mapIndices = DenseI64ArrayAttr::get(ctx, values);
  }

  if (!byref) {
    mlir::SmallVector<bool> values(operands.size(), false);
    byref = DenseBoolArrayAttr::get(ctx, values);
  }

  llvm::interleaveComma(llvm::zip_equal(operands, argsSubrange, symbols,
                                        mapIndices.asArrayRef(),
                                        byref.asArrayRef()),
                        p, [&p](auto t) {
                          auto [op, arg, sym, map, isByRef] = t;
                          if (isByRef)
                            p << "byref ";
                          if (sym)
                            p << sym << " ";

                          p << op << " -> " << arg;

                          if (map != -1)
                            p << " [map_idx=" << map << "]";
                        });
  p << " : ";
  llvm::interleaveComma(types, p);
  p << ") ";

  if (needsBarrier)
    p << getPrivateNeedsBarrierSpelling() << " ";
}

static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
                                StringRef clauseName, ValueRange argsSubrange,
                                std::optional<MapPrintArgs> mapArgs) {
  if (mapArgs)
    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange, mapArgs->vars,
                              mapArgs->types);
}

static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
                                StringRef clauseName, ValueRange argsSubrange,
                                std::optional<PrivatePrintArgs> privateArgs) {
  if (privateArgs)
    printClauseWithRegionArgs(
        p, ctx, clauseName, argsSubrange, privateArgs->vars, privateArgs->types,
        privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
        /*modifier=*/nullptr, privateArgs->needsBarrier);
}

static void
printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
                    ValueRange argsSubrange,
                    std::optional<ReductionPrintArgs> reductionArgs) {
  if (reductionArgs)
    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
                              reductionArgs->vars, reductionArgs->types,
                              reductionArgs->syms, /*mapIndices=*/nullptr,
                              reductionArgs->byref, reductionArgs->modifier);
}

static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                const AllRegionPrintArgs &args) {
  auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
  MLIRContext *ctx = op->getContext();

  printBlockArgClause(p, ctx, "has_device_addr",
                      iface.getHasDeviceAddrBlockArgs(),
                      args.hasDeviceAddrArgs);
  printBlockArgClause(p, ctx, "host_eval", iface.getHostEvalBlockArgs(),
                      args.hostEvalArgs);
  printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
                      args.inReductionArgs);
  printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),
                      args.mapArgs);
  printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),
                      args.privateArgs);
  printBlockArgClause(p, ctx, "reduction", iface.getReductionBlockArgs(),
                      args.reductionArgs);
  printBlockArgClause(p, ctx, "task_reduction",
                      iface.getTaskReductionBlockArgs(),
                      args.taskReductionArgs);
  printBlockArgClause(p, ctx, "use_device_addr",
                      iface.getUseDeviceAddrBlockArgs(),
                      args.useDeviceAddrArgs);
  printBlockArgClause(p, ctx, "use_device_ptr",
                      iface.getUseDevicePtrBlockArgs(), args.useDevicePtrArgs);

  p.printRegion(region, /*printEntryBlockArgs=*/false);
}

// These parseXyz functions correspond to the custom<Xyz> definitions
// in the .td file(s).
static void printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                ValueRange hasDeviceAddrVars,
                                TypeRange hasDeviceAddrTypes,
                                ValueRange hostEvalVars,
                                TypeRange hostEvalTypes, ValueRange mapVars,
                                TypeRange mapTypes, ValueRange privateVars,
                                TypeRange privateTypes, ArrayAttr privateSyms,
                                UnitAttr privateNeedsBarrier,
                                DenseI64ArrayAttr privateMaps) {
  AllRegionPrintArgs args;
  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
  args.mapArgs.emplace(mapVars, mapTypes);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier, privateMaps);
  printBlockArgRegion(p, op, region, args);
}

static void printInReductionPrivateRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {
  AllRegionPrintArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  printBlockArgRegion(p, op, region, args);
}

static void printInReductionPrivateReductionRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
    ReductionModifierAttr reductionMod, ValueRange reductionVars,
    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
    ArrayAttr reductionSyms) {
  AllRegionPrintArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, reductionMod);
  printBlockArgRegion(p, op, region, args);
}

static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region,
                               ValueRange privateVars, TypeRange privateTypes,
                               ArrayAttr privateSyms,
                               UnitAttr privateNeedsBarrier) {
  AllRegionPrintArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  printBlockArgRegion(p, op, region, args);
}

static void printPrivateReductionRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars,
    TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
    ReductionModifierAttr reductionMod, ValueRange reductionVars,
    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
    ArrayAttr reductionSyms) {
  AllRegionPrintArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, reductionMod);
  printBlockArgRegion(p, op, region, args);
}

static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
                                     Region &region,
                                     ValueRange taskReductionVars,
                                     TypeRange taskReductionTypes,
                                     DenseBoolArrayAttr taskReductionByref,
                                     ArrayAttr taskReductionSyms) {
  AllRegionPrintArgs args;
  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
                                 taskReductionByref, taskReductionSyms);
  printBlockArgRegion(p, op, region, args);
}

static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op,
                                                 Region &region,
                                                 ValueRange useDeviceAddrVars,
                                                 TypeRange useDeviceAddrTypes,
                                                 ValueRange useDevicePtrVars,
                                                 TypeRange useDevicePtrTypes) {
  AllRegionPrintArgs args;
  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
  printBlockArgRegion(p, op, region, args);
}

template <typename ParsePrefixFn>
static ParseResult parseSplitIteratedList(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
    SmallVectorImpl<Type> &iteratedTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &plainVars,
    SmallVectorImpl<Type> &plainTypes, ParsePrefixFn &&parsePrefix) {

  return parser.parseCommaSeparatedList([&]() -> ParseResult {
    if (failed(parsePrefix()))
      return failure();

    OpAsmParser::UnresolvedOperand v;
    Type ty;
    if (parser.parseOperand(v) || parser.parseColonType(ty))
      return failure();

    if (llvm::isa<mlir::omp::IteratedType>(ty)) {
      iteratedVars.push_back(v);
      iteratedTypes.push_back(ty);
    } else {
      plainVars.push_back(v);
      plainTypes.push_back(ty);
    }
    return success();
  });
}

template <typename PrintPrefixFn>
static void printSplitIteratedList(OpAsmPrinter &p, ValueRange iteratedVars,
                                   TypeRange iteratedTypes,
                                   ValueRange plainVars, TypeRange plainTypes,
                                   PrintPrefixFn &&printPrefixForPlain,
                                   PrintPrefixFn &&printPrefixForIterated) {

  bool first = true;
  auto emit = [&](Value v, Type t, auto &&printPrefix) {
    if (!first)
      p << ", ";
    printPrefix(v, t);
    p << v << " : " << t;
    first = false;
  };

  for (unsigned i = 0; i < iteratedVars.size(); ++i)
    emit(iteratedVars[i], iteratedTypes[i], printPrefixForIterated);
  for (unsigned i = 0; i < plainVars.size(); ++i)
    emit(plainVars[i], plainTypes[i], printPrefixForPlain);
}

/// Verifies Reduction Clause
static LogicalResult
verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,
                       OperandRange reductionVars,
                       std::optional<ArrayRef<bool>> reductionByref) {
  if (!reductionVars.empty()) {
    if (!reductionSyms || reductionSyms->size() != reductionVars.size())
      return op->emitOpError()
             << "expected as many reduction symbol references "
                "as reduction variables";
    if (reductionByref && reductionByref->size() != reductionVars.size())
      return op->emitError() << "expected as many reduction variable by "
                                "reference attributes as reduction variables";
  } else {
    if (reductionSyms)
      return op->emitOpError() << "unexpected reduction symbol references";
    return success();
  }

  // TODO: The followings should be done in
  // SymbolUserOpInterface::verifySymbolUses.
  DenseSet<Value> accumulators;
  for (auto args : llvm::zip(reductionVars, *reductionSyms)) {
    Value accum = std::get<0>(args);

    if (!accumulators.insert(accum).second)
      return op->emitOpError() << "accumulator variable used more than once";

    Type varType = accum.getType();
    auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));
    auto decl =
        SymbolTable::lookupNearestSymbolFrom<DeclareReductionOp>(op, symbolRef);
    if (!decl)
      return op->emitOpError() << "expected symbol reference " << symbolRef
                               << " to point to a reduction declaration";

    if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)
      return op->emitOpError()
             << "expected accumulator (" << varType
             << ") to be the same type as reduction declaration ("
             << decl.getAccumulatorType() << ")";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Copyprivate
//===----------------------------------------------------------------------===//

/// copyprivate-entry-list ::= copyprivate-entry
///                          | copyprivate-entry-list `,` copyprivate-entry
/// copyprivate-entry ::= ssa-id `->` symbol-ref `:` type
static ParseResult parseCopyprivate(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &copyprivateVars,
    SmallVectorImpl<Type> &copyprivateTypes, ArrayAttr &copyprivateSyms) {
  SmallVector<SymbolRefAttr> symsVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        if (parser.parseOperand(copyprivateVars.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseAttribute(symsVec.emplace_back()) ||
            parser.parseColonType(copyprivateTypes.emplace_back()))
          return failure();
        return success();
      })))
    return failure();
  SmallVector<Attribute> syms(symsVec.begin(), symsVec.end());
  copyprivateSyms = ArrayAttr::get(parser.getContext(), syms);
  return success();
}

/// Print Copyprivate clause
static void printCopyprivate(OpAsmPrinter &p, Operation *op,
                             OperandRange copyprivateVars,
                             TypeRange copyprivateTypes,
                             std::optional<ArrayAttr> copyprivateSyms) {
  if (!copyprivateSyms.has_value())
    return;
  llvm::interleaveComma(
      llvm::zip(copyprivateVars, *copyprivateSyms, copyprivateTypes), p,
      [&](const auto &args) {
        p << std::get<0>(args) << " -> " << std::get<1>(args) << " : "
          << std::get<2>(args);
      });
}

/// Verifies CopyPrivate Clause
static LogicalResult
verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars,
                         std::optional<ArrayAttr> copyprivateSyms) {
  size_t copyprivateSymsSize =
      copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;
  if (copyprivateSymsSize != copyprivateVars.size())
    return op->emitOpError() << "inconsistent number of copyprivate vars (= "
                             << copyprivateVars.size()
                             << ") and functions (= " << copyprivateSymsSize
                             << "), both must be equal";
  if (!copyprivateSyms.has_value())
    return success();

  for (auto copyprivateVarAndSym :
       llvm::zip(copyprivateVars, *copyprivateSyms)) {
    auto symbolRef =
        llvm::cast<SymbolRefAttr>(std::get<1>(copyprivateVarAndSym));
    std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>
        funcOp;
    if (mlir::func::FuncOp mlirFuncOp =
            SymbolTable::lookupNearestSymbolFrom<mlir::func::FuncOp>(op,
                                                                     symbolRef))
      funcOp = mlirFuncOp;
    else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =
                 SymbolTable::lookupNearestSymbolFrom<mlir::LLVM::LLVMFuncOp>(
                     op, symbolRef))
      funcOp = llvmFuncOp;

    auto getNumArguments = [&] {
      return std::visit([](auto &f) { return f.getNumArguments(); }, *funcOp);
    };

    auto getArgumentType = [&](unsigned i) {
      return std::visit([i](auto &f) { return f.getArgumentTypes()[i]; },
                        *funcOp);
    };

    if (!funcOp)
      return op->emitOpError() << "expected symbol reference " << symbolRef
                               << " to point to a copy function";

    if (getNumArguments() != 2)
      return op->emitOpError()
             << "expected copy function " << symbolRef << " to have 2 operands";

    Type argTy = getArgumentType(0);
    if (argTy != getArgumentType(1))
      return op->emitOpError() << "expected copy function " << symbolRef
                               << " arguments to have the same type";

    Type varType = std::get<0>(copyprivateVarAndSym).getType();
    if (argTy != varType)
      return op->emitOpError()
             << "expected copy function arguments' type (" << argTy
             << ") to be the same as copyprivate variable's type (" << varType
             << ")";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Parser, printer and verifier for DependVarList
//===----------------------------------------------------------------------===//

/// depend-entry-list ::= depend-entry
///                     | depend-entry-list `,` depend-entry
/// depend-entry ::= depend-kind `->` ssa-id `:` type
///                | depend-kind `->` ssa-id `:` iterated-type
static ParseResult parseDependVarList(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dependVars,
    SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
    SmallVectorImpl<Type> &iteratedTypes, ArrayAttr &iteratedKinds) {
  SmallVector<ClauseTaskDependAttr> kindsVec;
  SmallVector<ClauseTaskDependAttr> iterKindsVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        StringRef keyword;
        OpAsmParser::UnresolvedOperand operand;
        Type ty;
        if (parser.parseKeyword(&keyword) || parser.parseArrow() ||
            parser.parseOperand(operand) || parser.parseColonType(ty))
          return failure();
        std::optional<ClauseTaskDepend> keywordDepend =
            symbolizeClauseTaskDepend(keyword);
        if (!keywordDepend)
          return failure();
        auto kindAttr =
            ClauseTaskDependAttr::get(parser.getContext(), *keywordDepend);
        if (llvm::isa<mlir::omp::IteratedType>(ty)) {
          iteratedVars.push_back(operand);
          iteratedTypes.push_back(ty);
          iterKindsVec.push_back(kindAttr);
        } else {
          dependVars.push_back(operand);
          dependTypes.push_back(ty);
          kindsVec.push_back(kindAttr);
        }
        return success();
      })))
    return failure();
  SmallVector<Attribute> kinds(kindsVec.begin(), kindsVec.end());
  dependKinds = ArrayAttr::get(parser.getContext(), kinds);
  SmallVector<Attribute> iterKinds(iterKindsVec.begin(), iterKindsVec.end());
  iteratedKinds = ArrayAttr::get(parser.getContext(), iterKinds);
  return success();
}

/// Print Depend clause
static void printDependVarList(OpAsmPrinter &p, Operation *op,
                               OperandRange dependVars, TypeRange dependTypes,
                               std::optional<ArrayAttr> dependKinds,
                               OperandRange iteratedVars,
                               TypeRange iteratedTypes,
                               std::optional<ArrayAttr> iteratedKinds) {
  bool first = true;
  auto printEntries = [&](OperandRange vars, TypeRange types,
                          std::optional<ArrayAttr> kinds) {
    for (unsigned i = 0, e = vars.size(); i < e; ++i) {
      if (!first)
        p << ", ";
      p << stringifyClauseTaskDepend(
               llvm::cast<mlir::omp::ClauseTaskDependAttr>((*kinds)[i])
                   .getValue())
        << " -> " << vars[i] << " : " << types[i];
      first = false;
    }
  };
  printEntries(dependVars, dependTypes, dependKinds);
  printEntries(iteratedVars, iteratedTypes, iteratedKinds);
}

/// Verifies Depend clause
static LogicalResult verifyDependVarList(Operation *op,
                                         std::optional<ArrayAttr> dependKinds,
                                         OperandRange dependVars,
                                         std::optional<ArrayAttr> iteratedKinds,
                                         OperandRange iteratedVars) {
  if (!dependVars.empty()) {
    if (!dependKinds || dependKinds->size() != dependVars.size())
      return op->emitOpError() << "expected as many depend values"
                                  " as depend variables";
  } else {
    if (dependKinds && !dependKinds->empty())
      return op->emitOpError() << "unexpected depend values";
  }

  if (!iteratedVars.empty()) {
    if (!iteratedKinds || iteratedKinds->size() != iteratedVars.size())
      return op->emitOpError() << "expected as many depend iterated values"
                                  " as depend iterated variables";
  } else {
    if (iteratedKinds && !iteratedKinds->empty())
      return op->emitOpError() << "unexpected depend iterated values";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Synchronization Hint (2.17.12)
//===----------------------------------------------------------------------===//

/// Parses a Synchronization Hint clause. The value of hint is an integer
/// which is a combination of different hints from `omp_sync_hint_t`.
///
/// hint-clause = `hint` `(` hint-value `)`
static ParseResult parseSynchronizationHint(OpAsmParser &parser,
                                            IntegerAttr &hintAttr) {
  StringRef hintKeyword;
  int64_t hint = 0;
  if (succeeded(parser.parseOptionalKeyword("none"))) {
    hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), 0);
    return success();
  }
  auto parseKeyword = [&]() -> ParseResult {
    if (failed(parser.parseKeyword(&hintKeyword)))
      return failure();
    if (hintKeyword == "uncontended")
      hint |= 1;
    else if (hintKeyword == "contended")
      hint |= 2;
    else if (hintKeyword == "nonspeculative")
      hint |= 4;
    else if (hintKeyword == "speculative")
      hint |= 8;
    else
      return parser.emitError(parser.getCurrentLocation())
             << hintKeyword << " is not a valid hint";
    return success();
  };
  if (parser.parseCommaSeparatedList(parseKeyword))
    return failure();
  hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), hint);
  return success();
}

/// Prints a Synchronization Hint clause
static void printSynchronizationHint(OpAsmPrinter &p, Operation *op,
                                     IntegerAttr hintAttr) {
  int64_t hint = hintAttr.getInt();

  if (hint == 0) {
    p << "none";
    return;
  }

  // Helper function to get n-th bit from the right end of `value`
  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };

  bool uncontended = bitn(hint, 0);
  bool contended = bitn(hint, 1);
  bool nonspeculative = bitn(hint, 2);
  bool speculative = bitn(hint, 3);

  SmallVector<StringRef> hints;
  if (uncontended)
    hints.push_back("uncontended");
  if (contended)
    hints.push_back("contended");
  if (nonspeculative)
    hints.push_back("nonspeculative");
  if (speculative)
    hints.push_back("speculative");

  llvm::interleaveComma(hints, p);
}

/// Verifies a synchronization hint clause
static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) {

  // Helper function to get n-th bit from the right end of `value`
  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };

  bool uncontended = bitn(hint, 0);
  bool contended = bitn(hint, 1);
  bool nonspeculative = bitn(hint, 2);
  bool speculative = bitn(hint, 3);

  if (uncontended && contended)
    return op->emitOpError() << "the hints omp_sync_hint_uncontended and "
                                "omp_sync_hint_contended cannot be combined";
  if (nonspeculative && speculative)
    return op->emitOpError() << "the hints omp_sync_hint_nonspeculative and "
                                "omp_sync_hint_speculative cannot be combined.";
  return success();
}

//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Target
//===----------------------------------------------------------------------===//

// Helper function to get bitwise AND of `value` and 'flag' then return it as a
// boolean
static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag) {
  return (value & flag) == flag;
}

/// Parses a map_entries map type from a string format back into its numeric
/// value.
///
/// map-clause = `map_clauses (  ( `(` `always, `? `implicit, `? `ompx_hold, `?
/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )
static ParseResult parseMapClause(OpAsmParser &parser,
                                  ClauseMapFlagsAttr &mapType) {
  ClauseMapFlags mapTypeBits = ClauseMapFlags::none;
  // This simply verifies the correct keyword is read in, the
  // keyword itself is stored inside of the operation
  auto parseTypeAndMod = [&]() -> ParseResult {
    StringRef mapTypeMod;
    if (parser.parseKeyword(&mapTypeMod))
      return failure();

    if (mapTypeMod == "always")
      mapTypeBits |= ClauseMapFlags::always;

    if (mapTypeMod == "implicit")
      mapTypeBits |= ClauseMapFlags::implicit;

    if (mapTypeMod == "ompx_hold")
      mapTypeBits |= ClauseMapFlags::ompx_hold;

    if (mapTypeMod == "close")
      mapTypeBits |= ClauseMapFlags::close;

    if (mapTypeMod == "present")
      mapTypeBits |= ClauseMapFlags::present;

    if (mapTypeMod == "to")
      mapTypeBits |= ClauseMapFlags::to;

    if (mapTypeMod == "from")
      mapTypeBits |= ClauseMapFlags::from;

    if (mapTypeMod == "tofrom")
      mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;

    if (mapTypeMod == "delete")
      mapTypeBits |= ClauseMapFlags::del;

    if (mapTypeMod == "storage")
      mapTypeBits |= ClauseMapFlags::storage;

    if (mapTypeMod == "return_param")
      mapTypeBits |= ClauseMapFlags::return_param;

    if (mapTypeMod == "private")
      mapTypeBits |= ClauseMapFlags::priv;

    if (mapTypeMod == "literal")
      mapTypeBits |= ClauseMapFlags::literal;

    if (mapTypeMod == "attach")
      mapTypeBits |= ClauseMapFlags::attach;

    if (mapTypeMod == "attach_always")
      mapTypeBits |= ClauseMapFlags::attach_always;

    if (mapTypeMod == "attach_never")
      mapTypeBits |= ClauseMapFlags::attach_never;

    if (mapTypeMod == "attach_auto")
      mapTypeBits |= ClauseMapFlags::attach_auto;

    if (mapTypeMod == "ref_ptr")
      mapTypeBits |= ClauseMapFlags::ref_ptr;

    if (mapTypeMod == "ref_ptee")
      mapTypeBits |= ClauseMapFlags::ref_ptee;

    if (mapTypeMod == "is_device_ptr")
      mapTypeBits |= ClauseMapFlags::is_device_ptr;

    return success();
  };

  if (parser.parseCommaSeparatedList(parseTypeAndMod))
    return failure();

  mapType =
      parser.getBuilder().getAttr<mlir::omp::ClauseMapFlagsAttr>(mapTypeBits);

  return success();
}

/// Prints a map_entries map type from its numeric value out into its string
/// format.
static void printMapClause(OpAsmPrinter &p, Operation *op,
                           ClauseMapFlagsAttr mapType) {
  llvm::SmallVector<std::string, 4> mapTypeStrs;
  ClauseMapFlags mapFlags = mapType.getValue();

  // handling of always, close, present placed at the beginning of the string
  // to aid readability
  if (mapTypeToBool(mapFlags, ClauseMapFlags::always))
    mapTypeStrs.push_back("always");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::implicit))
    mapTypeStrs.push_back("implicit");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ompx_hold))
    mapTypeStrs.push_back("ompx_hold");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::close))
    mapTypeStrs.push_back("close");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::present))
    mapTypeStrs.push_back("present");

  // special handling of to/from/tofrom/delete and release/alloc, release +
  // alloc are the abscense of one of the other flags, whereas tofrom requires
  // both the to and from flag to be set.
  bool to = mapTypeToBool(mapFlags, ClauseMapFlags::to);
  bool from = mapTypeToBool(mapFlags, ClauseMapFlags::from);

  if (to && from)
    mapTypeStrs.push_back("tofrom");
  else if (from)
    mapTypeStrs.push_back("from");
  else if (to)
    mapTypeStrs.push_back("to");

  if (mapTypeToBool(mapFlags, ClauseMapFlags::del))
    mapTypeStrs.push_back("delete");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::return_param))
    mapTypeStrs.push_back("return_param");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::storage))
    mapTypeStrs.push_back("storage");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::priv))
    mapTypeStrs.push_back("private");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::literal))
    mapTypeStrs.push_back("literal");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach))
    mapTypeStrs.push_back("attach");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_always))
    mapTypeStrs.push_back("attach_always");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_never))
    mapTypeStrs.push_back("attach_never");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_auto))
    mapTypeStrs.push_back("attach_auto");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptr))
    mapTypeStrs.push_back("ref_ptr");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptee))
    mapTypeStrs.push_back("ref_ptee");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::is_device_ptr))
    mapTypeStrs.push_back("is_device_ptr");
  if (mapFlags == ClauseMapFlags::none)
    mapTypeStrs.push_back("none");

  for (unsigned int i = 0; i < mapTypeStrs.size(); ++i) {
    p << mapTypeStrs[i];
    if (i + 1 < mapTypeStrs.size()) {
      p << ", ";
    }
  }
}

static ParseResult parseMembersIndex(OpAsmParser &parser,
                                     ArrayAttr &membersIdx) {
  SmallVector<Attribute> values, memberIdxs;

  auto parseIndices = [&]() -> ParseResult {
    int64_t value;
    if (parser.parseInteger(value))
      return failure();
    values.push_back(IntegerAttr::get(parser.getBuilder().getIntegerType(64),
                                      APInt(64, value, /*isSigned=*/false)));
    return success();
  };

  do {
    if (failed(parser.parseLSquare()))
      return failure();

    if (parser.parseCommaSeparatedList(parseIndices))
      return failure();

    if (failed(parser.parseRSquare()))
      return failure();

    memberIdxs.push_back(ArrayAttr::get(parser.getContext(), values));
    values.clear();
  } while (succeeded(parser.parseOptionalComma()));

  if (!memberIdxs.empty())
    membersIdx = ArrayAttr::get(parser.getContext(), memberIdxs);

  return success();
}

static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
                              ArrayAttr membersIdx) {
  if (!membersIdx)
    return;

  llvm::interleaveComma(membersIdx, p, [&p](Attribute v) {
    p << "[";
    auto memberIdx = cast<ArrayAttr>(v);
    llvm::interleaveComma(memberIdx.getValue(), p, [&p](Attribute v2) {
      p << cast<IntegerAttr>(v2).getInt();
    });
    p << "]";
  });
}

static void printCaptureType(OpAsmPrinter &p, Operation *op,
                             VariableCaptureKindAttr mapCaptureType) {
  std::string typeCapStr;
  llvm::raw_string_ostream typeCap(typeCapStr);
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)
    typeCap << "ByRef";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)
    typeCap << "ByCopy";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)
    typeCap << "VLAType";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
    typeCap << "This";
  p << typeCapStr;
}

static ParseResult parseCaptureType(OpAsmParser &parser,
                                    VariableCaptureKindAttr &mapCaptureType) {
  StringRef mapCaptureKey;
  if (parser.parseKeyword(&mapCaptureKey))
    return failure();

  if (mapCaptureKey == "This")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::This);
  if (mapCaptureKey == "ByRef")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::ByRef);
  if (mapCaptureKey == "ByCopy")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::ByCopy);
  if (mapCaptureKey == "VLAType")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::VLAType);

  return success();
}

static LogicalResult verifyMapInfoForMapClause(
    Operation *op, mlir::omp::MapInfoOp mapInfoOp,
    llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> &updateToVars,
    llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>>
        &updateFromVars) {
  mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();

  bool to = mapTypeToBool(mapTypeBits, ClauseMapFlags::to);
  bool from = mapTypeToBool(mapTypeBits, ClauseMapFlags::from);
  bool del = mapTypeToBool(mapTypeBits, ClauseMapFlags::del);

  bool always = mapTypeToBool(mapTypeBits, ClauseMapFlags::always);
  bool close = mapTypeToBool(mapTypeBits, ClauseMapFlags::close);
  bool implicit = mapTypeToBool(mapTypeBits, ClauseMapFlags::implicit);
  bool attach = mapTypeToBool(mapTypeBits, ClauseMapFlags::attach);

  if ((isa<TargetDataOp>(op) || isa<TargetOp>(op)) && del)
    return emitError(op->getLoc(),
                     "to, from, tofrom and alloc map types are permitted");

  if (isa<TargetEnterDataOp>(op) && (from || del))
    return emitError(op->getLoc(), "to and alloc map types are permitted");

  if (isa<TargetExitDataOp>(op) && to)
    return emitError(op->getLoc(),
                     "from, release and delete map types are permitted");

  if (isa<TargetUpdateOp>(op)) {
    if (del) {
      return emitError(op->getLoc(),
                       "at least one of to or from map types must be "
                       "specified, other map types are not permitted");
    }

    if (!to && !from && !attach) {
      return emitError(op->getLoc(),
                       "at least one of to or from or attach map types must be "
                       "specified, other map types are not permitted");
    }

    auto updateVar = mapInfoOp.getVarPtr();

    if ((to && from) || (to && updateFromVars.contains(updateVar)) ||
        (from && updateToVars.contains(updateVar))) {
      return emitError(
          op->getLoc(),
          "either to or from map types can be specified, not both");
    }

    if (always || close || implicit) {
      return emitError(
          op->getLoc(),
          "present, mapper and iterator map type modifiers are permitted");
    }

    // It's possible we have an attach map, in which case if there is no to
    // or from tied to it, we skip insertion.
    if (to || from) {
      to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);
    }
  }

  if ((mapInfoOp.getVarPtrPtr() && !mapInfoOp.getVarPtrPtrType()) ||
      (!mapInfoOp.getVarPtrPtr() && mapInfoOp.getVarPtrPtrType())) {
    return emitError(op->getLoc(),
                     "if varPtrPtr or varPtrPtrType is specified, then both "
                     "must be present");
  }

  return success();
}

static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars,
                                     OperandRange mapIterated) {
  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateToVars;
  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateFromVars;

  for (auto mapOp : mapVars) {
    if (!mapOp.getDefiningOp())
      return emitError(op->getLoc(), "missing map operation");

    if (auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {
      if (failed(verifyMapInfoForMapClause(op, mapInfoOp, updateToVars,
                                           updateFromVars)))
        return failure();
    } else if (!isa<DeclareMapperInfoOp>(op)) {
      return emitError(op->getLoc(),
                       "map argument is not a map entry operation");
    }
  }

  // Verify iterated map entries.
  for (auto iterVal : mapIterated) {
    auto iterOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
    if (!iterOp)
      return op->emitOpError() << "'map_iterated' arguments must be defined by "
                                  "'omp.iterator' ops";

    // Check that the iterator body yields a value defined by omp.map.info.
    auto yieldOp =
        cast<mlir::omp::YieldOp>(iterOp.getRegion().front().getTerminator());
    auto yieldedMapInfo =
        yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>();
    if (!yieldedMapInfo)
      return op->emitOpError() << "'map_iterated' iterator body must yield "
                                  "a value defined by 'omp.map.info'";

    if (failed(verifyMapInfoForMapClause(op, yieldedMapInfo, updateToVars,
                                         updateFromVars)))
      return failure();
  }

  return success();
}

template <typename OpType>
static LogicalResult verifyPrivateVarList(OpType &op);

static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {
  std::optional<DenseI64ArrayAttr> privateMapIndices =
      targetOp.getPrivateMapsAttr();

  // None of the private operands are mapped.
  if (!privateMapIndices.has_value() || !privateMapIndices.value())
    return success();

  OperandRange privateVars = targetOp.getPrivateVars();

  if (privateMapIndices.value().size() !=
      static_cast<int64_t>(privateVars.size()))
    return emitError(targetOp.getLoc(), "sizes of `private` operand range and "
                                        "`private_maps` attribute mismatch");

  return success();
}

//===----------------------------------------------------------------------===//
// MapInfoOp
//===----------------------------------------------------------------------===//

static LogicalResult verifyMapInfoDefinedArgs(Operation *op,
                                              StringRef clauseName,
                                              OperandRange vars) {
  for (Value var : vars)
    if (!llvm::isa_and_present<MapInfoOp>(var.getDefiningOp()))
      return op->emitOpError()
             << "'" << clauseName
             << "' arguments must be defined by 'omp.map.info' ops";
  return success();
}

LogicalResult MapInfoOp::verify() {
  if (getMapperId() &&
      !SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
          *this, getMapperIdAttr())) {
    return emitError("invalid mapper id");
  }

  if (failed(verifyMapInfoDefinedArgs(*this, "members", getMembers())))
    return failure();

  return success();
}

//===----------------------------------------------------------------------===//
// TargetDataOp
//===----------------------------------------------------------------------===//

void TargetDataOp::build(OpBuilder &builder, OperationState &state,
                         const TargetDataOperands &clauses) {
  TargetDataOp::build(builder, state, clauses.device, clauses.ifExpr,
                      clauses.mapVars, clauses.mapIterated,
                      clauses.useDeviceAddrVars, clauses.useDevicePtrVars);
}

LogicalResult TargetDataOp::verify() {
  if (getMapVars().empty() && getMapIterated().empty() &&
      getUseDevicePtrVars().empty() && getUseDeviceAddrVars().empty()) {
    return ::emitError(this->getLoc(),
                       "At least one of map, use_device_ptr_vars, or "
                       "use_device_addr_vars operand must be present");
  }

  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_ptr",
                                      getUseDevicePtrVars())))
    return failure();

  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_addr",
                                      getUseDeviceAddrVars())))
    return failure();

  return verifyMapClause(*this, getMapVars(), getMapIterated());
}

//===----------------------------------------------------------------------===//
// TargetEnterDataOp
//===----------------------------------------------------------------------===//

void TargetEnterDataOp::build(
    OpBuilder &builder, OperationState &state,
    const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetEnterDataOp::build(
      builder, state, makeArrayAttr(ctx, clauses.dependKinds),
      clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
      clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
      clauses.mapIterated, clauses.nowait);
}

LogicalResult TargetEnterDataOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}

//===----------------------------------------------------------------------===//
// TargetExitDataOp
//===----------------------------------------------------------------------===//

void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,
                             const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetExitDataOp::build(
      builder, state, makeArrayAttr(ctx, clauses.dependKinds),
      clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
      clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
      clauses.mapIterated, clauses.nowait);
}

LogicalResult TargetExitDataOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}

//===----------------------------------------------------------------------===//
// TargetUpdateOp
//===----------------------------------------------------------------------===//

void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,
                           const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetUpdateOp::build(builder, state, makeArrayAttr(ctx, clauses.dependKinds),
                        clauses.dependVars,
                        makeArrayAttr(ctx, clauses.dependIteratedKinds),
                        clauses.dependIterated, clauses.device, clauses.ifExpr,
                        clauses.mapVars, clauses.mapIterated, clauses.nowait);
}

LogicalResult TargetUpdateOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}

//===----------------------------------------------------------------------===//
// TargetOp
//===----------------------------------------------------------------------===//

void TargetOp::build(OpBuilder &builder, OperationState &state,
                     const TargetExtOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
      makeArrayAttr(ctx, clauses.dependIteratedKinds), clauses.dependIterated,
      clauses.device, clauses.dynGroupprivateAccessGroup,
      clauses.dynGroupprivateFallback, clauses.dynGroupprivateSize,
      clauses.hasDeviceAddrVars, clauses.hostEvalVars, clauses.ifExpr,
      clauses.inReductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.isDevicePtrVars,
      clauses.mapVars, clauses.mapIterated, clauses.nowait, clauses.privateVars,
      makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
      clauses.threadLimitVars, /*private_maps=*/nullptr, clauses.kernelType);
}

bool TargetOp::hasHostEvalTripCount() {
  TargetExecMode mode = getKernelType();
  if (mode == TargetExecMode::spmd || mode == TargetExecMode::spmd_no_loop)
    return true;

  if (mode == TargetExecMode::bare)
    return false;

  // If it represents a `target teams distribute` construct, also evaluate the
  // `distribute` trip count on the host.
  Operation *capturedOp =
      cast<ComposableOpInterface>(getOperation()).findCapturedOp();
  if (auto loopNestOp = dyn_cast_if_present<LoopNestOp>(capturedOp)) {
    SmallVector<LoopWrapperInterface> loopWrappers;
    loopNestOp.gatherWrappers(loopWrappers);

    LoopWrapperInterface *innermostWrapper = loopWrappers.begin();
    if (isa<SimdOp>(innermostWrapper))
      innermostWrapper = std::next(innermostWrapper);

    auto numWrappers = std::distance(innermostWrapper, loopWrappers.end());
    if (numWrappers != 1)
      return false;

    if (!isa<DistributeOp>(innermostWrapper))
      return false;

    Operation *parentOp = innermostWrapper->getOperation()->getParentOp();
    if (isa_and_present<TeamsOp>(parentOp) &&
        parentOp->getParentOp() == getOperation())
      return true;
  }

  return false;
}

/// An `omp.target` `in_reduction` operand is captured by a `map_entries` entry
/// when the entry's `MapInfoOp` var_ptr is the same SSA value, or another
/// result of the same defining op. At this stage, exact identity can only be
/// required for block arguments, which have no defining op. Flang emits
/// `hlfir.declare` #0 for the `in_reduction` operand and #1 for the map
/// `var_ptr`; these collapse to the same value after lowering, but that cannot
/// be enforced here.
static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr) {
  if (mapVarPtr == inReductionVar)
    return true;
  Operation *def = inReductionVar.getDefiningOp();
  return def && mapVarPtr.getDefiningOp() == def;
}

LogicalResult TargetOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (getKernelType() == TargetExecMode::bare && !isCombined())
    return emitOpError() << "bare kernel requires 'omp.combined'";

  if (failed(verifyDependVarList(*this, getDependKinds(), getDependVars(),
                                 getDependIteratedKinds(),
                                 getDependIterated())))
    return failure();

  if (failed(verifyMapInfoDefinedArgs(*this, "has_device_addr",
                                      getHasDeviceAddrVars())))
    return failure();

  if (failed(verifyMapClause(*this, getMapVars(), getMapIterated())))
    return failure();

  if (failed(verifyDynGroupprivateClause(
          *this, getDynGroupprivateAccessGroupAttr(),
          getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
    return failure();

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  if (failed(verifyReductionVarList(*this, getInReductionSyms(),
                                    getInReductionVars(),
                                    getInReductionByref())))
    return failure();

  // An `in_reduction` operand on `omp.target` has no dedicated entry block
  // argument; inside the region it is accessed through the block argument of a
  // matching `map_entries` entry, and the host rewrites that map argument to
  // the reduction-private storage. Require every `in_reduction` operand to be
  // captured by at least one `map_entries` entry.
  for (Value inReductionVar : getInReductionVars()) {
    bool captured = false;
    for (Value mapVar : getMapVars()) {
      auto mapInfo = mapVar.getDefiningOp<MapInfoOp>();
      if (targetInReductionCapturedBy(inReductionVar, mapInfo.getVarPtr())) {
        captured = true;
        break;
      }
    }
    if (!captured)
      return emitOpError() << "in_reduction variable must be captured by a "
                              "matching map_entries entry";
  }

  return verifyPrivateVarsMapping(*this);
}

LogicalResult TargetOp::verifyRegions() {
  auto teamsOps = getOps<TeamsOp>();
  auto numNestedTeams = std::distance(teamsOps.begin(), teamsOps.end());
  if (numNestedTeams > 1)
    return emitError("target containing multiple 'omp.teams' nested ops");

  if (numNestedTeams == 0) {
    switch (getKernelType()) {
    case TargetExecMode::bare:
      return emitOpError()
             << "bare kernel must contain a nested 'omp.teams' operation";
    case TargetExecMode::spmd_no_loop:
      return emitOpError() << "spmd_no_loop kernel must contain a nested "
                              "'omp.teams' operation";
    default:
      break;
    }
  }

  Operation *capturedOp =
      cast<ComposableOpInterface>(getOperation()).findCapturedOp();
  if ((getKernelType() == TargetExecMode::spmd ||
       getKernelType() == TargetExecMode::spmd_no_loop) &&
      !isa_and_present<LoopNestOp>(capturedOp))
    return emitOpError()
           << "SPMD kernel must capture an 'omp.loop_nest' operation";

  bool isTargetDevice = false;
  if (auto offloadMod = (*this)->getParentOfType<OffloadModuleInterface>())
    if (offloadMod.getIsTargetDevice())
      isTargetDevice = true;

  // Check that host_eval values are only used in legal ways.
  llvm::ArrayRef<BlockArgument> hostEvalBlockArgs =
      cast<BlockArgOpenMPOpInterface>(getOperation()).getHostEvalBlockArgs();

  bool hostEvalTripCount = hasHostEvalTripCount();
  for (Value hostEvalArg : hostEvalBlockArgs) {
    for (Operation *user : hostEvalArg.getUsers()) {
      if (auto teamsOp = dyn_cast<TeamsOp>(user)) {
        // Check if used in num_teams_lower or any of num_teams_upper_vars
        if (hostEvalArg == teamsOp.getNumTeamsLower() ||
            llvm::is_contained(teamsOp.getNumTeamsUpperVars(), hostEvalArg) ||
            llvm::is_contained(teamsOp.getThreadLimitVars(), hostEvalArg))
          continue;

        return emitOpError() << "host_eval argument only legal as 'num_teams' "
                                "and 'thread_limit' in 'omp.teams'";
      }
      if (auto parallelOp = dyn_cast<ParallelOp>(user)) {
        if (llvm::is_contained(parallelOp.getNumThreadsVars(), hostEvalArg))
          continue;

        return emitOpError()
               << "host_eval argument only legal as 'num_threads' in "
                  "'omp.parallel'";
      }
      if (auto loopNestOp = dyn_cast<LoopNestOp>(user)) {
        if (hostEvalTripCount &&
            (llvm::is_contained(loopNestOp.getLoopLowerBounds(), hostEvalArg) ||
             llvm::is_contained(loopNestOp.getLoopUpperBounds(), hostEvalArg) ||
             llvm::is_contained(loopNestOp.getLoopSteps(), hostEvalArg)))
          continue;

        return emitOpError() << "host_eval argument only legal as loop bounds "
                                "and steps in 'omp.loop_nest' when trip count "
                                "must be evaluated in the host";
      }

      return emitOpError() << "host_eval argument illegal use in '"
                           << user->getName() << "' operation";
    }
  }

  if (hostEvalTripCount && !isTargetDevice) {
    auto loopOp = cast<LoopNestOp>(capturedOp);
    for (auto arg : llvm::concat<Value>(loopOp.getLoopLowerBounds(),
                                        loopOp.getLoopUpperBounds(),
                                        loopOp.getLoopSteps())) {
      if (!llvm::is_contained(hostEvalBlockArgs, arg))
        return emitOpError() << "nested 'omp.loop_nest' bounds expected to "
                                "be host-evaluated";
    }
  }

  return success();
}

//===----------------------------------------------------------------------===//
// ParallelOp
//===----------------------------------------------------------------------===//

void ParallelOp::build(OpBuilder &builder, OperationState &state,
                       ArrayRef<NamedAttribute> attributes) {
  ParallelOp::build(builder, state, /*allocate_vars=*/ValueRange(),
                    /*allocator_vars=*/ValueRange(),
                    /*allocate_alignments=*/nullptr,
                    /*allocate_private_indices=*/nullptr, /*if_expr=*/nullptr,
                    /*num_threads_vars=*/ValueRange(),
                    /*private_vars=*/ValueRange(),
                    /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,
                    /*proc_bind_kind=*/nullptr,
                    /*reduction_mod =*/nullptr, /*reduction_vars=*/ValueRange(),
                    /*reduction_byref=*/nullptr, /*reduction_syms=*/nullptr);
  state.addAttributes(attributes);
}

void ParallelOp::build(OpBuilder &builder, OperationState &state,
                       const ParallelOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  ParallelOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                    makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                    makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                    clauses.ifExpr, clauses.numThreadsVars, clauses.privateVars,
                    makeArrayAttr(ctx, clauses.privateSyms),
                    clauses.privateNeedsBarrier, clauses.procBindKind,
                    clauses.reductionMod, clauses.reductionVars,
                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                    makeArrayAttr(ctx, clauses.reductionSyms));
}

template <typename OpType>
static LogicalResult verifyPrivateVarList(OpType &op) {
  auto privateVars = op.getPrivateVars();
  auto privateSyms = op.getPrivateSymsAttr();

  if (privateVars.empty() && (privateSyms == nullptr || privateSyms.empty()))
    return success();

  auto numPrivateVars = privateVars.size();
  auto numPrivateSyms = (privateSyms == nullptr) ? 0 : privateSyms.size();

  if (numPrivateVars != numPrivateSyms)
    return op.emitError() << "inconsistent number of private variables and "
                             "privatizer op symbols, private vars: "
                          << numPrivateVars
                          << " vs. privatizer op symbols: " << numPrivateSyms;

  for (auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {
    Type varType = std::get<0>(privateVarInfo).getType();
    SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));
    PrivateClauseOp privatizerOp =
        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);

    if (privatizerOp == nullptr)
      return op.emitError() << "failed to lookup privatizer op with symbol: '"
                            << privateSym << "'";

    Type privatizerType = privatizerOp.getArgType();

    if (privatizerType && (varType != privatizerType))
      return op.emitError()
             << "type mismatch between a "
             << (privatizerOp.getDataSharingType() ==
                         DataSharingClauseType::Private
                     ? "private"
                     : "firstprivate")
             << " variable and its privatizer op, var type: " << varType
             << " vs. privatizer op type: " << privatizerType;
  }

  return success();
}

LogicalResult ParallelOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr(),
          /*requirePrivateIndices=*/true)))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

LogicalResult ParallelOp::verifyRegions() {
  auto distChildOps = getOps<DistributeOp>();
  int numDistChildOps = std::distance(distChildOps.begin(), distChildOps.end());
  if (numDistChildOps > 1)
    return emitError()
           << "multiple 'omp.distribute' nested inside of 'omp.parallel'";

  if (numDistChildOps == 1) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite operation";

    auto *ompDialect = getContext()->getLoadedDialect<OpenMPDialect>();
    Operation &distributeOp = **distChildOps.begin();
    for (Operation &childOp : getOps()) {
      if (&childOp == &distributeOp || ompDialect != childOp.getDialect())
        continue;

      if (!childOp.hasTrait<OpTrait::IsTerminator>())
        return emitError() << "unexpected OpenMP operation inside of composite "
                              "'omp.parallel': "
                           << childOp.getName();
    }
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite operation";
  }
  return success();
}

//===----------------------------------------------------------------------===//
// TeamsOp
//===----------------------------------------------------------------------===//

static bool opInGlobalImplicitParallelRegion(Operation *op) {
  while ((op = op->getParentOp()))
    if (isa<OpenMPDialect>(op->getDialect()))
      return false;
  return true;
}

void TeamsOp::build(OpBuilder &builder, OperationState &state,
                    const TeamsOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  TeamsOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      clauses.dynGroupprivateAccessGroup, clauses.dynGroupprivateFallback,
      clauses.dynGroupprivateSize, clauses.ifExpr, clauses.numTeamsLower,
      clauses.numTeamsUpperVars, /*private_vars=*/{}, /*private_syms=*/nullptr,
      /*private_needs_barrier=*/nullptr, clauses.reductionMod,
      clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadLimitVars);
}

// Verify num_teams clause
static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower,
                                          OperandRange numTeamsUpperVars) {
  // If lower is specified, upper must have exactly one value
  if (numTeamsLower) {
    if (numTeamsUpperVars.size() != 1)
      return op->emitError(
          "expected exactly one num_teams upper bound when lower bound is "
          "specified");
    if (numTeamsLower.getType() != numTeamsUpperVars[0].getType())
      return op->emitError(
          "expected num_teams upper bound and lower bound to be "
          "the same type");
  }

  return success();
}

LogicalResult TeamsOp::verify() {
  // Check parent region
  // TODO If nested inside of a target region, also check that it does not
  // contain any statements, declarations or directives other than this
  // omp.teams construct. The issue is how to support the initialization of
  // this operation's own arguments (allow SSA values across omp.target?).
  Operation *op = getOperation();
  auto parentTarget = llvm::dyn_cast_if_present<TargetOp>(op->getParentOp());
  if (!parentTarget && !opInGlobalImplicitParallelRegion(op))
    return emitError("expected to be nested inside of omp.target or not nested "
                     "in any OpenMP dialect operations");

  // Check for num_teams clause restrictions
  if (failed(verifyNumTeamsClause(op, this->getNumTeamsLower(),
                                  this->getNumTeamsUpperVars())))
    return failure();

  if (parentTarget &&
      parentTarget.getKernelType() == TargetExecMode::spmd_no_loop &&
      (getNumTeamsLower() || !getNumTeamsUpperVars().empty()))
    return emitOpError() << "'num_teams' not allowed in SPMD-no-loop kernels";

  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (failed(verifyDynGroupprivateClause(
          op, getDynGroupprivateAccessGroupAttr(),
          getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
    return failure();

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

//===----------------------------------------------------------------------===//
// SectionOp
//===----------------------------------------------------------------------===//

OperandRange SectionOp::getPrivateVars() {
  return getParentOp().getPrivateVars();
}

OperandRange SectionOp::getReductionVars() {
  return getParentOp().getReductionVars();
}

//===----------------------------------------------------------------------===//
// SectionsOp
//===----------------------------------------------------------------------===//

void SectionsOp::build(OpBuilder &builder, OperationState &state,
                       const SectionsOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  SectionsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                    makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                    makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                    clauses.nowait, /*private_vars=*/{},
                    /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,
                    clauses.reductionMod, clauses.reductionVars,
                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                    makeArrayAttr(ctx, clauses.reductionSyms));
}

LogicalResult SectionsOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";

  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

LogicalResult SectionsOp::verifyRegions() {
  for (auto &inst : *getRegion().begin()) {
    if (!(isa<SectionOp>(inst) || isa<TerminatorOp>(inst))) {
      return emitOpError()
             << "expected omp.section op or terminator op inside region";
    }
  }

  return success();
}

//===----------------------------------------------------------------------===//
// ScopeOp
//===----------------------------------------------------------------------===//

void ScopeOp::build(OpBuilder &builder, OperationState &state,
                    const ScopeOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  ScopeOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                 clauses.nowait, clauses.privateVars,
                 makeArrayAttr(ctx, clauses.privateSyms),
                 clauses.privateNeedsBarrier, clauses.reductionMod,
                 clauses.reductionVars,
                 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                 makeArrayAttr(ctx, clauses.reductionSyms));
}

LogicalResult ScopeOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

//===----------------------------------------------------------------------===//
// SingleOp
//===----------------------------------------------------------------------===//

void SingleOp::build(OpBuilder &builder, OperationState &state,
                     const SingleOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  SingleOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                  makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                  makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                  clauses.copyprivateVars,
                  makeArrayAttr(ctx, clauses.copyprivateSyms), clauses.nowait,
                  /*private_vars=*/{}, /*private_syms=*/nullptr,
                  /*private_needs_barrier=*/nullptr);
}

LogicalResult SingleOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  return verifyCopyprivateVarList(*this, getCopyprivateVars(),
                                  getCopyprivateSyms());
}

//===----------------------------------------------------------------------===//
// WorkshareOp
//===----------------------------------------------------------------------===//

void WorkshareOp::build(OpBuilder &builder, OperationState &state,
                        const WorkshareOperands &clauses) {
  WorkshareOp::build(builder, state, clauses.nowait);
}

LogicalResult WorkshareOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";

  return success();
}

//===----------------------------------------------------------------------===//
// WorkshareLoopWrapperOp
//===----------------------------------------------------------------------===//

LogicalResult WorkshareLoopWrapperOp::verifyRegions() {
  if (isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
      getNestedWrapper())
    return emitOpError() << "expected to be a standalone loop wrapper";

  return success();
}

//===----------------------------------------------------------------------===//
// LoopWrapperInterface
//===----------------------------------------------------------------------===//

LogicalResult LoopWrapperInterface::verifyImpl() {
  Operation *op = this->getOperation();
  if (!op->hasTrait<OpTrait::NoTerminator>() ||
      !op->hasTrait<OpTrait::SingleBlock>())
    return emitOpError() << "loop wrapper must also have the `NoTerminator` "
                            "and `SingleBlock` traits";

  if (op->getNumRegions() != 1)
    return emitOpError() << "loop wrapper does not contain exactly one region";

  Region &region = op->getRegion(0);
  if (range_size(region.getOps()) != 1)
    return emitOpError()
           << "loop wrapper does not contain exactly one nested op";

  Operation &firstOp = *region.op_begin();
  if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))
    return emitOpError() << "nested in loop wrapper is not another loop "
                            "wrapper or `omp.loop_nest`";

  return success();
}

//===----------------------------------------------------------------------===//
// ComposableOpInterface
//===----------------------------------------------------------------------===//

Operation *ComposableOpInterface::findCapturedOp() {
  Operation *op = this->getOperation();

  // Handle the composite case by returning the wrapped omp.loop_nest.
  if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(op))
    return wrapperOp.getWrappedLoop();

  // Do not look further if this op is not combined with any of its children.
  // Need to check for composite for the omp.parallel case, which is not a loop
  // wrapper itself.
  if (!isCombined() && !isComposite())
    return op;

  Region &region = op->getRegion(0);
  for (Operation &nestedOp : region.getOps()) {
    if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(&nestedOp))
      return wrapperOp.getWrappedLoop();

    if (auto composableOp = dyn_cast<ComposableOpInterface>(&nestedOp))
      return composableOp.findCapturedOp();
  }

  // This can only be reached if the op has an omp.combined attribute but the
  // corresponding nested composable op has been deleted. In that case, it's
  // correct to return this operation.
  return op;
}

LogicalResult ComposableOpInterface::verifyImpl() {
  Operation *op = this->getOperation();

  if (op->getNumRegions() != 1)
    return emitOpError() << "composable ops must have a single region";

  if (isComposite() && !isa<LoopWrapperInterface, ParallelOp>(op))
    return emitOpError() << "non-loop wrapper cannot be composite";

  // If combined, must have exactly one eligible nested op (composable or loop
  // wrapper).
  if (isCombined()) {
    Operation *nestedOp = nullptr;
    auto count = llvm::count_if(
        op->getRegion(0).getOps(), [&nestedOp](mlir::Operation &op) {
          if (isa<ComposableOpInterface, LoopWrapperInterface>(op)) {
            nestedOp = &op;
            return true;
          }
          return false;
        });

    // Make an exception for ops marked as omp.combined with no eligible nested
    // ops: this situation should be disallowed, but it can be reached if an
    // MLIR optimization pass find that the child operation has no side effects
    // (many ComposableOpInterface ops have RecursiveMemoryEffects), so it gets
    // deleted without updating the parent's attribute.
    //
    // Since there's a well defined way of handling that situation (treat it as
    // non-combined), we relax the requirement here. Ensuring the parent is
    // updated every time a pass that can potentially remove a child composable
    // op runs is less preferable as a solution.
    if (count == 0)
      return success();

    if (count > 1)
      return emitOpError()
             << "multiple eligible child ops found in combined op";

    // This operation cannot be combined if its captured nested op can be
    // executed more than once (i.e. its block's successors can reach it) or if
    // it's not guaranteed to be executed before all exits of the region (i.e.
    // it doesn't dominate all blocks with no successors reachable from the
    // entry block).
    DominanceInfo domInfo;
    Block *parentBlock = nestedOp->getBlock();

    for (Block *successor : parentBlock->getSuccessors())
      if (successor->isReachable(parentBlock))
        return emitOpError() << "nested combined child op is part of a loop";

    for (Block &block : op->getRegion(0))
      if (domInfo.isReachableFromEntry(&block) && block.hasNoSuccessors() &&
          !domInfo.dominates(parentBlock, &block))
        return emitOpError()
               << "nested combined child op doesn't unconditionally execute";
  }
  return success();
}

//===----------------------------------------------------------------------===//
// LoopOp
//===----------------------------------------------------------------------===//

void LoopOp::build(OpBuilder &builder, OperationState &state,
                   const LoopOperands &clauses) {
  MLIRContext *ctx = builder.getContext();

  LoopOp::build(builder, state, clauses.bindKind, clauses.privateVars,
                makeArrayAttr(ctx, clauses.privateSyms),
                clauses.privateNeedsBarrier, clauses.order, clauses.orderMod,
                clauses.reductionMod, clauses.reductionVars,
                makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                makeArrayAttr(ctx, clauses.reductionSyms));
}

LogicalResult LoopOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

LogicalResult LoopOp::verifyRegions() {
  if (llvm::isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
      getNestedWrapper())
    return emitOpError() << "expected to be a standalone loop wrapper";

  return success();
}

//===----------------------------------------------------------------------===//
// WsloopOp
//===----------------------------------------------------------------------===//

void WsloopOp::build(OpBuilder &builder, OperationState &state,
                     ArrayRef<NamedAttribute> attributes) {
  build(builder, state, /*allocate_vars=*/{}, /*allocator_vars=*/{},
        /*allocate_alignments=*/nullptr,
        /*allocate_private_indices=*/nullptr,
        /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
        /*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
        /*nowait=*/false, /*order=*/nullptr, /*order_mod=*/nullptr,
        /*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
        /*private_needs_barrier=*/false,
        /*reduction_mod=*/nullptr, /*reduction_vars=*/ValueRange(),
        /*reduction_byref=*/nullptr,
        /*reduction_syms=*/nullptr, /*schedule_kind=*/nullptr,
        /*schedule_chunk=*/nullptr, /*schedule_mod=*/nullptr,
        /*schedule_simd=*/false);
  state.addAttributes(attributes);
}

void WsloopOp::build(OpBuilder &builder, OperationState &state,
                     const WsloopOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  WsloopOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
      clauses.linearModifiers, clauses.nowait, clauses.order, clauses.orderMod,
      clauses.ordered, clauses.privateVars,
      makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
      clauses.reductionMod, clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
      clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
}

LogicalResult WsloopOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (failed(
          verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
    return failure();
  if (getLinearVars().size() &&
      getLinearVarTypes().value().size() != getLinearVars().size())
    return emitError() << "Ill-formed type attributes for linear variables";

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}

LogicalResult WsloopOp::verifyRegions() {
  bool isCompositeChildLeaf =
      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());

  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";

    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after DO/FOR.
    if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrapper is 'omp.simd'";

  } else if (isComposite() && !isCompositeChildLeaf) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  } else if (!isComposite() && isCompositeChildLeaf) {
    return emitError()
           << "'omp.composite' attribute missing from composite wrapper";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Simd construct [2.9.3.1]
//===----------------------------------------------------------------------===//

void SimdOp::build(OpBuilder &builder, OperationState &state,
                   const SimdOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  SimdOp::build(builder, state, clauses.alignedVars,
                makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
                clauses.linearVars, clauses.linearStepVars,
                clauses.linearVarTypes, clauses.linearModifiers,
                clauses.nontemporalVars, clauses.order, clauses.orderMod,
                clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
                clauses.privateNeedsBarrier, clauses.reductionMod,
                clauses.reductionVars,
                makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
                clauses.simdlen);
}

LogicalResult SimdOp::verify() {
  if (getSimdlen().has_value() && getSafelen().has_value() &&
      getSimdlen().value() > getSafelen().value())
    return emitOpError()
           << "simdlen clause and safelen clause are both present, but the "
              "simdlen value is not less than or equal to safelen value";

  if (verifyAlignedClause(*this, getAlignments(), getAlignedVars()).failed())
    return failure();

  if (verifyNontemporalClause(*this, getNontemporalVars()).failed())
    return failure();

  if (failed(
          verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
    return failure();

  bool isCompositeChildLeaf =
      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());

  if (!isComposite() && isCompositeChildLeaf)
    return emitError()
           << "'omp.composite' attribute missing from composite wrapper";

  if (isComposite() && !isCompositeChildLeaf)
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";

  // Firstprivate is not allowed for SIMD in the standard. Check that none of
  // the private decls are for firstprivate.
  std::optional<ArrayAttr> privateSyms = getPrivateSyms();
  if (privateSyms) {
    for (const Attribute &sym : *privateSyms) {
      auto symRef = cast<SymbolRefAttr>(sym);
      omp::PrivateClauseOp privatizer =
          SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
              getOperation(), symRef);
      if (!privatizer)
        return emitError() << "Cannot find privatizer '" << symRef << "'";
      if (privatizer.getDataSharingType() ==
          DataSharingClauseType::FirstPrivate)
        return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
    }
  }

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  if (getLinearVars().size() &&
      getLinearVarTypes().value().size() != getLinearVars().size())
    return emitError() << "Ill-formed type attributes for linear variables";

  llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
  llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
  // TODO Check lastprivate vars when their support is added to SimdOp.
  for (Value var : getLinearVars()) {
    if (privateVars.contains(var) || reductionVars.contains(var))
      return emitOpError()
             << "linear variables cannot appear in other data-sharing clauses";
  }

  return success();
}

LogicalResult SimdOp::verifyRegions() {
  if (getNestedWrapper())
    return emitOpError() << "must wrap an 'omp.loop_nest' directly";

  return success();
}

//===----------------------------------------------------------------------===//
// Distribute construct [2.9.4.1]
//===----------------------------------------------------------------------===//

void DistributeOp::build(OpBuilder &builder, OperationState &state,
                         const DistributeOperands &clauses) {
  DistributeOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(builder.getContext(), clauses.allocateAlignments),
      makeDenseI64ArrayAttr(builder.getContext(),
                            clauses.allocatePrivateIndices),
      clauses.distScheduleStatic, clauses.distScheduleChunkSize, clauses.order,
      clauses.orderMod, clauses.privateVars,
      makeArrayAttr(builder.getContext(), clauses.privateSyms),
      clauses.privateNeedsBarrier);
}

LogicalResult DistributeOp::verify() {
  if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())
    return emitOpError() << "chunk size set without "
                            "dist_schedule_static being present";

  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return success();
}

LogicalResult DistributeOp::verifyRegions() {
  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";
    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after DISTRIBUTE.
    if (isa<WsloopOp>(nested)) {
      Operation *parentOp = (*this)->getParentOp();
      if (!llvm::dyn_cast_if_present<ParallelOp>(parentOp) ||
          !cast<ComposableOpInterface>(parentOp).isComposite()) {
        return emitError() << "an 'omp.wsloop' nested wrapper is only allowed "
                              "when a composite 'omp.parallel' is the direct "
                              "parent";
      }
    } else if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrappers are 'omp.simd' and "
                            "'omp.wsloop'";
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// DeclareMapperOp / DeclareMapperInfoOp
//===----------------------------------------------------------------------===//

void DeclareMapperInfoOp::build(OpBuilder &builder, OperationState &state,
                                const DeclareMapperInfoOperands &clauses) {
  DeclareMapperInfoOp::build(builder, state, clauses.mapVars,
                             clauses.mapIterated);
}

LogicalResult DeclareMapperInfoOp::verify() {
  return verifyMapClause(*this, getMapVars(), getMapIterated());
}

LogicalResult DeclareMapperOp::verifyRegions() {
  if (!llvm::isa_and_present<DeclareMapperInfoOp>(
          getRegion().getBlocks().front().getTerminator()))
    return emitOpError() << "expected terminator to be a DeclareMapperInfoOp";

  return success();
}

//===----------------------------------------------------------------------===//
// DeclareReductionOp
//===----------------------------------------------------------------------===//

LogicalResult DeclareReductionOp::verifyRegions() {
  if (!getAllocRegion().empty()) {
    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
      if (yieldOp.getResults().size() != 1 ||
          yieldOp.getResults().getTypes()[0] != getType())
        return emitOpError() << "expects alloc region to yield a value "
                                "of the reduction type";
    }
  }

  if (getInitializerRegion().empty())
    return emitOpError() << "expects non-empty initializer region";
  Block &initializerEntryBlock = getInitializerRegion().front();

  if (initializerEntryBlock.getNumArguments() == 1) {
    if (!getAllocRegion().empty())
      return emitOpError() << "expects two arguments to the initializer region "
                              "when an allocation region is used";
  } else if (initializerEntryBlock.getNumArguments() == 2) {
    if (getAllocRegion().empty())
      return emitOpError() << "expects one argument to the initializer region "
                              "when no allocation region is used";
  } else {
    return emitOpError()
           << "expects one or two arguments to the initializer region";
  }

  for (mlir::Value arg : initializerEntryBlock.getArguments())
    if (arg.getType() != getType())
      return emitOpError() << "expects initializer region argument to match "
                              "the reduction type";

  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
    if (yieldOp.getResults().size() != 1 ||
        yieldOp.getResults().getTypes()[0] != getType())
      return emitOpError() << "expects initializer region to yield a value "
                              "of the reduction type";
  }

  if (getReductionRegion().empty())
    return emitOpError() << "expects non-empty reduction region";
  Block &reductionEntryBlock = getReductionRegion().front();
  if (reductionEntryBlock.getNumArguments() != 2 ||
      reductionEntryBlock.getArgumentTypes()[0] !=
          reductionEntryBlock.getArgumentTypes()[1] ||
      reductionEntryBlock.getArgumentTypes()[0] != getType())
    return emitOpError() << "expects reduction region with two arguments of "
                            "the reduction type";
  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
    if (yieldOp.getResults().size() != 1 ||
        yieldOp.getResults().getTypes()[0] != getType())
      return emitOpError() << "expects reduction region to yield a value "
                              "of the reduction type";
  }

  if (!getAtomicReductionRegion().empty()) {
    Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
    if (atomicReductionEntryBlock.getNumArguments() != 2 ||
        atomicReductionEntryBlock.getArgumentTypes()[0] !=
            atomicReductionEntryBlock.getArgumentTypes()[1])
      return emitOpError() << "expects atomic reduction region with two "
                              "arguments of the same type";
    auto ptrType = llvm::dyn_cast<PointerLikeType>(
        atomicReductionEntryBlock.getArgumentTypes()[0]);
    if (!ptrType ||
        (ptrType.getElementType() && ptrType.getElementType() != getType()))
      return emitOpError() << "expects atomic reduction region arguments to "
                              "be accumulators containing the reduction type";
  }

  if (getCleanupRegion().empty())
    return success();
  Block &cleanupEntryBlock = getCleanupRegion().front();
  if (cleanupEntryBlock.getNumArguments() != 1 ||
      cleanupEntryBlock.getArgument(0).getType() != getType())
    return emitOpError() << "expects cleanup region with one argument "
                            "of the reduction type";

  return success();
}

//===----------------------------------------------------------------------===//
// TaskOp
//===----------------------------------------------------------------------===//

void TaskOp::build(OpBuilder &builder, OperationState &state,
                   const TaskOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskOp::build(
      builder, state, clauses.iterated, clauses.affinityVars,
      clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
      makeArrayAttr(ctx, clauses.dependIteratedKinds), clauses.dependIterated,
      clauses.final, clauses.ifExpr, clauses.inReductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
      clauses.priority, /*private_vars=*/clauses.privateVars,
      /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
      clauses.privateNeedsBarrier, clauses.untied, clauses.eventHandle);
}

LogicalResult TaskOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  if (failed(verifyDependVars))
    return verifyDependVars;

  if (failed(verifyPrivateVarList(*this)))
    return failure();

  return verifyReductionVarList(*this, getInReductionSyms(),
                                getInReductionVars(), getInReductionByref());
}

//===----------------------------------------------------------------------===//
// TaskgroupOp
//===----------------------------------------------------------------------===//

void TaskgroupOp::build(OpBuilder &builder, OperationState &state,
                        const TaskgroupOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskgroupOp::build(builder, state, clauses.allocateVars,
                     clauses.allocatorVars,
                     makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                     makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                     clauses.taskReductionVars,
                     makeDenseBoolArrayAttr(ctx, clauses.taskReductionByref),
                     makeArrayAttr(ctx, clauses.taskReductionSyms));
}

LogicalResult TaskgroupOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr())))
    return failure();

  return verifyReductionVarList(*this, getTaskReductionSyms(),
                                getTaskReductionVars(),
                                getTaskReductionByref());
}

//===----------------------------------------------------------------------===//
// TaskloopContextOp
//===----------------------------------------------------------------------===//

void TaskloopContextOp::build(OpBuilder &builder, OperationState &state,
                              const TaskloopContextOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskloopContextOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices), clauses.final,
      clauses.grainsizeMod, clauses.grainsize, clauses.ifExpr,
      clauses.inReductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
      clauses.nogroup, clauses.numTasksMod, clauses.numTasks, clauses.priority,
      /*private_vars=*/clauses.privateVars,
      /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
      clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.untied);
  state.addAttribute("omp.combined", UnitAttr::get(ctx));
}

TaskloopWrapperOp TaskloopContextOp::getLoopOp() {
  return cast<TaskloopWrapperOp>(
      *llvm::find_if(getRegion().front(), [](mlir::Operation &op) {
        return isa<TaskloopWrapperOp>(op);
      }));
}

LogicalResult TaskloopContextOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();

  if (failed(verifyReductionVarList(*this, getReductionSyms(),
                                    getReductionVars(), getReductionByref())) ||
      failed(verifyReductionVarList(*this, getInReductionSyms(),
                                    getInReductionVars(),
                                    getInReductionByref())))
    return failure();

  if (!getReductionVars().empty() && getNogroup())
    return emitError("if a reduction clause is present on the taskloop "
                     "directive, the nogroup clause must not be specified");
  for (auto var : getReductionVars()) {
    if (llvm::is_contained(getInReductionVars(), var))
      return emitError("the same list item cannot appear in both a reduction "
                       "and an in_reduction clause");
  }

  if (getGrainsize() && getNumTasks()) {
    return emitError(
        "the grainsize clause and num_tasks clause are mutually exclusive and "
        "may not appear on the same taskloop directive");
  }

  // Without this restriction, any compound construct including `taskloop` would
  // fail to correctly identify the whole chain of operations (see
  // ComposableOpInterface::findCapturedOp()), as well as failing to do so even
  // for standalone `taskloop` constructs.
  if (!isCombined())
    return emitOpError("must always contain the 'omp.combined' attribute");

  return success();
}

LogicalResult TaskloopContextOp::verifyRegions() {
  Region &region = getRegion();
  auto loopWrapperIt = llvm::find_if(region.front(), [](mlir::Operation &op) {
    return isa<TaskloopWrapperOp>(op);
  });
  if (loopWrapperIt == region.front().end())
    return emitOpError()
           << "expected a TaskloopWrapperOp directly nested in the region";

  auto loopWrapperOp = cast<TaskloopWrapperOp>(*loopWrapperIt);
  auto loopNestOp = dyn_cast<LoopNestOp>(loopWrapperOp.getWrappedLoop());
  // This will fail the verifier for TaskloopWrapperOp and print an error
  // message there.
  if (!loopNestOp)
    return failure();

  std::function<bool(Value)> isValidBoundValue = [&](Value value) -> bool {
    Region *valueRegion = value.getParentRegion();
    // A loop bound value defined outside of the taskloop context region is
    // valid. A region is considered an ancestor of itself.
    if (!region.isAncestor(valueRegion))
      return true;

    Operation *defOp = value.getDefiningOp();
    if (!defOp || defOp->getNumRegions() != 0 || !isPure(defOp))
      return false;

    return llvm::all_of(defOp->getOperands(), isValidBoundValue);
  };
  auto hasUnsupportedTaskloopLocalBound = [&](OperandRange range) -> bool {
    return llvm::any_of(range,
                        [&](Value value) { return !isValidBoundValue(value); });
  };

  if (hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopLowerBounds()) ||
      hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopUpperBounds()) ||
      hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopSteps())) {
    return emitOpError()
           << "expects loop bounds and steps to be defined outside of the "
              "taskloop.context region or by pure, regionless operations "
              "that do not depend on block arguments";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// TaskloopWrapperOp
//===----------------------------------------------------------------------===//

void TaskloopWrapperOp::build(OpBuilder &builder, OperationState &state,
                              const TaskloopWrapperOperands &clauses) {
  TaskloopWrapperOp::build(builder, state);
}

TaskloopContextOp TaskloopWrapperOp::getTaskloopContext() {
  return dyn_cast<TaskloopContextOp>(getOperation()->getParentOp());
}

LogicalResult TaskloopWrapperOp::verify() {
  TaskloopContextOp context = getTaskloopContext();
  if (!context)
    return emitOpError() << "expected to be nested in a taskloop context op";
  return success();
}

LogicalResult TaskloopWrapperOp::verifyRegions() {
  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";

    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after TASKLOOP.
    if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrapper is 'omp.simd'";
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  }

  return success();
}

//===----------------------------------------------------------------------===//
// LoopNestOp
//===----------------------------------------------------------------------===//

ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {
  // Parse an opening `(` followed by induction variables followed by `)`
  SmallVector<OpAsmParser::Argument> ivs;
  SmallVector<OpAsmParser::UnresolvedOperand> lbs, ubs;
  Type loopVarType;
  if (parser.parseArgumentList(ivs, OpAsmParser::Delimiter::Paren) ||
      parser.parseColonType(loopVarType) ||
      // Parse loop bounds.
      parser.parseEqual() ||
      parser.parseOperandList(lbs, ivs.size(), OpAsmParser::Delimiter::Paren) ||
      parser.parseKeyword("to") ||
      parser.parseOperandList(ubs, ivs.size(), OpAsmParser::Delimiter::Paren))
    return failure();

  for (auto &iv : ivs)
    iv.type = loopVarType;

  auto *ctx = parser.getBuilder().getContext();
  // Parse "inclusive" flag.
  if (succeeded(parser.parseOptionalKeyword("inclusive")))
    result.addAttribute("loop_inclusive", UnitAttr::get(ctx));

  // Parse step values.
  SmallVector<OpAsmParser::UnresolvedOperand> steps;
  if (parser.parseKeyword("step") ||
      parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren))
    return failure();

  // Parse collapse
  int64_t value = 0;
  if (!parser.parseOptionalKeyword("collapse") &&
      (parser.parseLParen() || parser.parseInteger(value) ||
       parser.parseRParen()))
    return failure();
  if (value > 1)
    result.addAttribute(
        "collapse_num_loops",
        IntegerAttr::get(parser.getBuilder().getI64Type(), value));

  // Parse tiles
  SmallVector<int64_t> tiles;
  auto parseTiles = [&]() -> ParseResult {
    int64_t tile;
    if (parser.parseInteger(tile))
      return failure();
    tiles.push_back(tile);
    return success();
  };

  if (!parser.parseOptionalKeyword("tiles") &&
      (parser.parseLParen() || parser.parseCommaSeparatedList(parseTiles) ||
       parser.parseRParen()))
    return failure();

  if (tiles.size() > 0)
    result.addAttribute("tile_sizes", DenseI64ArrayAttr::get(ctx, tiles));

  // Parse the body.
  Region *region = result.addRegion();
  if (parser.parseRegion(*region, ivs))
    return failure();

  // Resolve operands.
  if (parser.resolveOperands(lbs, loopVarType, result.operands) ||
      parser.resolveOperands(ubs, loopVarType, result.operands) ||
      parser.resolveOperands(steps, loopVarType, result.operands))
    return failure();

  // Parse the optional attribute list.
  return parser.parseOptionalAttrDict(result.attributes);
}

void LoopNestOp::print(OpAsmPrinter &p) {
  Region &region = getRegion();
  auto args = region.getArguments();
  p << " (" << args << ") : " << args[0].getType() << " = ("
    << getLoopLowerBounds() << ") to (" << getLoopUpperBounds() << ") ";
  if (getLoopInclusive())
    p << "inclusive ";
  p << "step (" << getLoopSteps() << ") ";
  if (int64_t numCollapse = getCollapseNumLoops())
    if (numCollapse > 1)
      p << "collapse(" << numCollapse << ") ";

  if (const auto tiles = getTileSizes())
    p << "tiles(" << tiles.value() << ") ";

  p.printRegion(region, /*printEntryBlockArgs=*/false);
}

void LoopNestOp::build(OpBuilder &builder, OperationState &state,
                       const LoopNestOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  LoopNestOp::build(builder, state, clauses.collapseNumLoops,
                    clauses.loopLowerBounds, clauses.loopUpperBounds,
                    clauses.loopSteps, clauses.loopInclusive,
                    makeDenseI64ArrayAttr(ctx, clauses.tileSizes));
}

LogicalResult LoopNestOp::verify() {
  if (getLoopLowerBounds().empty())
    return emitOpError() << "must represent at least one loop";

  if (getLoopLowerBounds().size() != getIVs().size())
    return emitOpError() << "number of range arguments and IVs do not match";

  for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), getIVs())) {
    if (lb.getType() != iv.getType())
      return emitOpError()
             << "range argument type does not match corresponding IV type";
  }

  uint64_t numIVs = getIVs().size();

  if (const auto &numCollapse = getCollapseNumLoops())
    if (numCollapse > numIVs)
      return emitOpError()
             << "collapse value is larger than the number of loops";

  if (const auto &tiles = getTileSizes())
    if (tiles.value().size() > numIVs)
      return emitOpError() << "too few canonical loops for tile dimensions";

  if (!llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp()))
    return emitOpError() << "expects parent op to be a loop wrapper";

  return success();
}

void LoopNestOp::gatherWrappers(
    SmallVectorImpl<LoopWrapperInterface> &wrappers) {
  Operation *parent = (*this)->getParentOp();
  while (auto wrapper =
             llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {
    wrappers.push_back(wrapper);
    parent = parent->getParentOp();
  }
}

//===----------------------------------------------------------------------===//
// OpenMP canonical loop handling
//===----------------------------------------------------------------------===//

std::tuple<NewCliOp, OpOperand *, OpOperand *>
mlir::omp ::decodeCli(Value cli) {

  // Defining a CLI for a generated loop is optional; if there is none then
  // there is no followup-tranformation
  if (!cli)
    return {{}, nullptr, nullptr};

  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
         "Unexpected type of cli");

  NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());
  OpOperand *gen = nullptr;
  OpOperand *cons = nullptr;
  for (OpOperand &use : cli.getUses()) {
    auto op = cast<LoopTransformationInterface>(use.getOwner());

    unsigned opnum = use.getOperandNumber();
    if (op.isGeneratee(opnum)) {
      assert(!gen && "Each CLI may have at most one def");
      gen = &use;
    } else if (op.isApplyee(opnum)) {
      assert(!cons && "Each CLI may have at most one consumer");
      cons = &use;
    } else {
      llvm_unreachable("Unexpected operand for a CLI");
    }
  }

  return {create, gen, cons};
}

ClauseProcBindKind
mlir::omp::convertProcBindKind(llvm::omp::ProcBindKind kind) {
  switch (kind) {
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
    return ClauseProcBindKind::Close;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
    return ClauseProcBindKind::Master;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
    return ClauseProcBindKind::Primary;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
    return ClauseProcBindKind::Spread;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
    break;
  }
  llvm_unreachable("unexpected proc-bind kind");
}

void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
                     ::mlir::OperationState &odsState) {
  odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
}

void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
  Value result = getResult();
  auto [newCli, gen, cons] = decodeCli(result);

  // Structured binding `gen` cannot be captured in lambdas before C++20
  OpOperand *generator = gen;

  // Derive the CLI variable name from its generator:
  //  * "canonloop" for omp.canonical_loop
  //  * custom name for loop transformation generatees
  //  * "cli" as fallback if no generator
  //  * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
  //  at that level
  //  * "_s<idx>" suffix for operations with multiple regions, where <idx> is
  //  the index of that region
  std::string cliName{"cli"};
  if (gen) {
    cliName =
        TypeSwitch<Operation *, std::string>(gen->getOwner())
            .Case([&](CanonicalLoopOp op) {
              return generateLoopNestingName("canonloop", op);
            })
            .Case([&](UnrollHeuristicOp op) -> std::string {
              llvm_unreachable("heuristic unrolling does not generate a loop");
            })
            .Case([&](FuseOp op) -> std::string {
              unsigned opnum = generator->getOperandNumber();
              // The position of the first loop to be fused is the same position
              // as the resulting fused loop
              if (op.getFirst().has_value() && opnum != op.getFirst().value())
                return "canonloop_fuse";
              else
                return "fused";
            })
            .Case([&](TileOp op) -> std::string {
              auto [generateesFirst, generateesCount] =
                  op.getGenerateesODSOperandIndexAndLength();
              unsigned firstGrid = generateesFirst;
              unsigned firstIntratile = generateesFirst + generateesCount / 2;
              unsigned end = generateesFirst + generateesCount;
              unsigned opnum = generator->getOperandNumber();
              // In the OpenMP apply and looprange clauses, indices are 1-based
              if (firstGrid <= opnum && opnum < firstIntratile) {
                unsigned gridnum = opnum - firstGrid + 1;
                return ("grid" + Twine(gridnum)).str();
              }
              if (firstIntratile <= opnum && opnum < end) {
                unsigned intratilenum = opnum - firstIntratile + 1;
                return ("intratile" + Twine(intratilenum)).str();
              }
              llvm_unreachable("Unexpected generatee argument");
            })
            .DefaultUnreachable("TODO: Custom name for this operation");
  }

  setNameFn(result, cliName);
}

LogicalResult NewCliOp::verify() {
  Value cli = getResult();

  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
         "Unexpected type of cli");

  // Check that the CLI is used in at most generator and one consumer
  OpOperand *gen = nullptr;
  OpOperand *cons = nullptr;
  for (mlir::OpOperand &use : cli.getUses()) {
    auto op = cast<mlir::omp::LoopTransformationInterface>(use.getOwner());

    unsigned opnum = use.getOperandNumber();
    if (op.isGeneratee(opnum)) {
      if (gen) {
        InFlightDiagnostic error =
            emitOpError("CLI must have at most one generator");
        error.attachNote(gen->getOwner()->getLoc())
            .append("first generator here:");
        error.attachNote(use.getOwner()->getLoc())
            .append("second generator here:");
        return error;
      }

      gen = &use;
    } else if (op.isApplyee(opnum)) {
      if (cons) {
        InFlightDiagnostic error =
            emitOpError("CLI must have at most one consumer");
        error.attachNote(cons->getOwner()->getLoc())
            .append("first consumer here:")
            .appendOp(*cons->getOwner(),
                      OpPrintingFlags().printGenericOpForm());
        error.attachNote(use.getOwner()->getLoc())
            .append("second consumer here:")
            .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());
        return error;
      }

      cons = &use;
    } else {
      llvm_unreachable("Unexpected operand for a CLI");
    }
  }

  // If the CLI is source of a transformation, it must have a generator
  if (cons && !gen) {
    InFlightDiagnostic error = emitOpError("CLI has no generator");
    error.attachNote(cons->getOwner()->getLoc())
        .append("see consumer here: ")
        .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());
    return error;
  }

  return success();
}

void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            Value tripCount) {
  odsState.addOperands(tripCount);
  odsState.addOperands(Value());
  (void)odsState.addRegion();
}

void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            Value tripCount, ::mlir::Value cli) {
  odsState.addOperands(tripCount);
  odsState.addOperands(cli);
  (void)odsState.addRegion();
}

void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
  setNameFn(&getRegion().front(), "body_entry");
}

void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
                                               OpAsmSetValueNameFn setNameFn) {
  std::string ivName = generateLoopNestingName("iv", *this);
  setNameFn(region.getArgument(0), ivName);
}

void CanonicalLoopOp::print(OpAsmPrinter &p) {
  if (getCli())
    p << '(' << getCli() << ')';
  p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
    << " in range(" << getTripCount() << ") ";

  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                /*printBlockTerminators=*/true);

  p.printOptionalAttrDict((*this)->getAttrs());
}

mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
                                         ::mlir::OperationState &result) {
  CanonicalLoopInfoType cliType =
      CanonicalLoopInfoType::get(parser.getContext());

  // Parse (optional) omp.cli identifier
  OpAsmParser::UnresolvedOperand cli;
  SmallVector<mlir::Value, 1> cliOperand;
  if (!parser.parseOptionalLParen()) {
    if (parser.parseOperand(cli) ||
        parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())
      return failure();
  }

  // We derive the type of tripCount from inductionVariable. MLIR requires the
  // type of tripCount to be known when calling resolveOperand so we have parse
  // the type before processing the inductionVariable.
  OpAsmParser::Argument inductionVariable;
  OpAsmParser::UnresolvedOperand tripcount;
  if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||
      parser.parseKeyword("in") || parser.parseKeyword("range") ||
      parser.parseLParen() || parser.parseOperand(tripcount) ||
      parser.parseRParen() ||
      parser.resolveOperand(tripcount, inductionVariable.type, result.operands))
    return failure();

  // Parse the loop body.
  Region *region = result.addRegion();
  if (parser.parseRegion(*region, {inductionVariable}))
    return failure();

  // We parsed the cli operand forst, but because it is optional, it must be
  // last in the operand list.
  result.operands.append(cliOperand);

  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();

  return mlir::success();
}

LogicalResult CanonicalLoopOp::verify() {
  // The region's entry must accept the induction variable
  // It can also be empty if just created
  if (!getRegion().empty()) {
    Region &region = getRegion();
    if (region.getNumArguments() != 1)
      return emitOpError(
          "Canonical loop region must have exactly one argument");

    if (getInductionVar().getType() != getTripCount().getType())
      return emitOpError(
          "Region argument must be the same type as the trip count");
  }

  return success();
}

Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }

std::pair<unsigned, unsigned>
CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
  // No applyees
  return {0, 0};
}

std::pair<unsigned, unsigned>
CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_cli);
}

//===----------------------------------------------------------------------===//
// UnrollHeuristicOp
//===----------------------------------------------------------------------===//

void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
                              ::mlir::OperationState &odsState,
                              ::mlir::Value cli) {
  odsState.addOperands(cli);
}

void UnrollHeuristicOp::print(OpAsmPrinter &p) {
  p << '(' << getApplyee() << ')';

  p.printOptionalAttrDict((*this)->getAttrs());
}

mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
                                           ::mlir::OperationState &result) {
  auto cliType = CanonicalLoopInfoType::get(parser.getContext());

  if (parser.parseLParen())
    return failure();

  OpAsmParser::UnresolvedOperand applyee;
  if (parser.parseOperand(applyee) ||
      parser.resolveOperand(applyee, cliType, result.operands))
    return failure();

  if (parser.parseRParen())
    return failure();

  // Optional output loop (full unrolling has none)
  if (!parser.parseOptionalArrow()) {
    if (parser.parseLParen() || parser.parseRParen())
      return failure();
  }

  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();

  return mlir::success();
}

std::pair<unsigned, unsigned>
UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyee);
}

std::pair<unsigned, unsigned>
UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
  return {0, 0};
}

//===----------------------------------------------------------------------===//
// UnrollPartialOp
//===----------------------------------------------------------------------===//

void UnrollPartialOp::build(::mlir::OpBuilder &odsBuilder,
                            ::mlir::OperationState &odsState, ::mlir::Value cli,
                            uint64_t unrollFactor) {
  odsState.addOperands(cli);
  Properties &props = odsState.getOrAddProperties<Properties>();
  props.unroll_factor = odsBuilder.getI64IntegerAttr(unrollFactor);
}

void UnrollPartialOp::print(OpAsmPrinter &p) {
  p << '(' << getApplyee() << ')';

  p.printOptionalAttrDict((*this)->getAttrs());
}

mlir::ParseResult UnrollPartialOp::parse(::mlir::OpAsmParser &parser,
                                         ::mlir::OperationState &result) {
  auto cliType = CanonicalLoopInfoType::get(parser.getContext());

  if (parser.parseLParen())
    return failure();

  OpAsmParser::UnresolvedOperand applyee;
  if (parser.parseOperand(applyee) ||
      parser.resolveOperand(applyee, cliType, result.operands))
    return failure();

  if (parser.parseRParen())
    return failure();

  // The unroll factor is carried by the `unroll_factor` attribute.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();

  return mlir::success();
}

std::pair<unsigned, unsigned>
UnrollPartialOp::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyee);
}

std::pair<unsigned, unsigned>
UnrollPartialOp::getGenerateesODSOperandIndexAndLength() {
  return {0, 0};
}

//===----------------------------------------------------------------------===//
// TileOp
//===----------------------------------------------------------------------===//

static void printLoopTransformClis(OpAsmPrinter &p, TileOp op,
                                   OperandRange generatees,
                                   OperandRange applyees) {
  if (!generatees.empty())
    p << '(' << llvm::interleaved(generatees) << ')';

  if (!applyees.empty())
    p << " <- (" << llvm::interleaved(applyees) << ')';
}

static ParseResult parseLoopTransformClis(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &generateesOperands,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &applyeesOperands) {
  if (parser.parseOptionalLess()) {
    // Syntax 1: generatees present

    if (parser.parseOperandList(generateesOperands,
                                mlir::OpAsmParser::Delimiter::Paren))
      return failure();

    if (parser.parseLess())
      return failure();
  } else {
    // Syntax 2: generatees omitted
  }

  // Parse `<-` (`<` has already been parsed)
  if (parser.parseMinus())
    return failure();

  if (parser.parseOperandList(applyeesOperands,
                              mlir::OpAsmParser::Delimiter::Paren))
    return failure();

  return success();
}

/// Check properties of the loop nest consisting of the transformation's
/// applyees:
/// 1. They are nested inside each other
/// 2. They are perfectly nested
///    (no code with side-effects in-between the loops)
/// 3. They are rectangular
///    (loop bounds are invariant in respect to the outer loops)
///
/// TODO: Generalize for LoopTransformationInterface.
static LogicalResult checkApplyeesNesting(TileOp op) {
  // Collect the loops from the nest
  bool isOnlyCanonLoops = true;
  SmallVector<CanonicalLoopOp> canonLoops;
  for (Value applyee : op.getApplyees()) {
    auto [create, gen, cons] = decodeCli(applyee);

    if (!gen)
      return op.emitOpError() << "applyee CLI has no generator";

    auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
    canonLoops.push_back(loop);
    if (!loop)
      isOnlyCanonLoops = false;
  }

  // FIXME: We currently can only verify non-rectangularity and perfect nest of
  // omp.canonical_loop.
  if (!isOnlyCanonLoops)
    return success();

  DenseSet<Value> parentIVs;
  for (auto i : llvm::seq<int>(1, canonLoops.size())) {
    auto parentLoop = canonLoops[i - 1];
    auto loop = canonLoops[i];

    if (parentLoop.getOperation() != loop.getOperation()->getParentOp())
      return op.emitOpError()
             << "tiled loop nest must be nested within each other";

    parentIVs.insert(parentLoop.getInductionVar());

    // Canonical loop must be perfectly nested, i.e. the body of the parent must
    // only contain the omp.canonical_loop of the nested loops, and
    // omp.terminator
    bool isPerfectlyNested = [&]() {
      auto &parentBody = parentLoop.getRegion();
      if (!parentBody.hasOneBlock())
        return false;
      auto &parentBlock = parentBody.getBlocks().front();

      auto nestedLoopIt = parentBlock.begin();
      if (nestedLoopIt == parentBlock.end() ||
          (&*nestedLoopIt != loop.getOperation()))
        return false;

      auto termIt = std::next(nestedLoopIt);
      if (termIt == parentBlock.end() || !isa<TerminatorOp>(termIt))
        return false;

      if (std::next(termIt) != parentBlock.end())
        return false;

      return true;
    }();
    if (!isPerfectlyNested)
      return op.emitOpError() << "tiled loop nest must be perfectly nested";

    if (parentIVs.contains(loop.getTripCount()))
      return op.emitOpError() << "tiled loop nest must be rectangular";
  }

  // TODO: The tile sizes must be computed before the loop, but checking this
  // requires dominance analysis. For instance:
  //
  //      %canonloop = omp.new_cli
  //      omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
  //        // write to %x
  //        omp.terminator
  //      }
  //      %ts = llvm.load %x
  //      omp.tile <- (%canonloop) sizes(%ts : i32)

  return success();
}

LogicalResult TileOp::verify() {
  if (getApplyees().empty())
    return emitOpError() << "must apply to at least one loop";

  if (getSizes().size() != getApplyees().size())
    return emitOpError() << "there must be one tile size for each applyee";

  if (!getGeneratees().empty() &&
      2 * getSizes().size() != getGeneratees().size())
    return emitOpError()
           << "expecting two times the number of generatees than applyees";

  return checkApplyeesNesting(*this);
}

std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyees);
}

std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_generatees);
}

//===----------------------------------------------------------------------===//
// FuseOp
//===----------------------------------------------------------------------===//

static void printLoopTransformClis(OpAsmPrinter &p, FuseOp op,
                                   OperandRange generatees,
                                   OperandRange applyees) {
  if (!generatees.empty())
    p << '(' << llvm::interleaved(generatees) << ')';

  if (!applyees.empty())
    p << " <- (" << llvm::interleaved(applyees) << ')';
}

LogicalResult FuseOp::verify() {
  if (getApplyees().size() < 2)
    return emitOpError() << "must apply to at least two loops";

  if (getFirst().has_value() && getCount().has_value()) {
    int64_t first = getFirst().value();
    int64_t count = getCount().value();
    if ((unsigned)(first + count - 1) > getApplyees().size())
      return emitOpError() << "the numbers of applyees must be at least first "
                              "minus one plus count attributes";
    if (!getGeneratees().empty() &&
        getGeneratees().size() != getApplyees().size() + 1 - count)
      return emitOpError() << "the number of generatees must be the number of "
                              "aplyees plus one minus count";

  } else {
    if (!getGeneratees().empty() && getGeneratees().size() != 1)
      return emitOpError()
             << "in a complete fuse the number of generatees must be exactly 1";
  }
  for (auto &&applyee : getApplyees()) {
    auto [create, gen, cons] = decodeCli(applyee);

    if (!gen)
      return emitOpError() << "applyee CLI has no generator";
    auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
    if (!loop)
      return emitOpError()
             << "currently only supports omp.canonical_loop as applyee";
  }
  return success();
}
std::pair<unsigned, unsigned> FuseOp::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyees);
}

std::pair<unsigned, unsigned> FuseOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_generatees);
}

//===----------------------------------------------------------------------===//
// Critical construct (2.17.1)
//===----------------------------------------------------------------------===//

void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,
                              const CriticalDeclareOperands &clauses) {
  CriticalDeclareOp::build(builder, state, clauses.symName, clauses.hint);
}

LogicalResult CriticalDeclareOp::verify() {
  return verifySynchronizationHint(*this, getHint());
}

LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  if (getNameAttr()) {
    SymbolRefAttr symbolRef = getNameAttr();
    auto decl = symbolTable.lookupNearestSymbolFrom<CriticalDeclareOp>(
        *this, symbolRef);
    if (!decl) {
      return emitOpError() << "expected symbol reference " << symbolRef
                           << " to point to a critical declaration";
    }
  }

  return success();
}

//===----------------------------------------------------------------------===//
// Spec 5.1: Error directive (2.5.4)
//===----------------------------------------------------------------------===//

LogicalResult ErrorOp::verify() {
  if (getMessage() && getMessageExpr())
    return emitOpError() << "the message must be provided either as a constant "
                            "`message` attribute or as a `message_expr` "
                            "operand, but not both";
  return success();
}

//===----------------------------------------------------------------------===//
// Ordered construct
//===----------------------------------------------------------------------===//

static LogicalResult verifyOrderedParent(Operation &op) {
  bool hasRegion = op.getNumRegions() > 0;
  auto loopOp = op.getParentOfType<LoopNestOp>();
  if (!loopOp) {
    if (hasRegion)
      return success();

    // TODO: Consider if this needs to be the case only for the standalone
    // variant of the ordered construct.
    return op.emitOpError() << "must be nested inside of a loop";
  }

  Operation *wrapper = loopOp->getParentOp();
  if (auto wsloopOp = dyn_cast<WsloopOp>(wrapper)) {
    IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();
    if (!orderedAttr)
      return op.emitOpError() << "the enclosing worksharing-loop region must "
                                 "have an ordered clause";

    if (hasRegion && orderedAttr.getInt() != 0)
      return op.emitOpError() << "the enclosing loop's ordered clause must not "
                                 "have a parameter present";

    if (!hasRegion && orderedAttr.getInt() == 0)
      return op.emitOpError() << "the enclosing loop's ordered clause must "
                                 "have a parameter present";
  } else if (!isa<SimdOp>(wrapper)) {
    return op.emitOpError() << "must be nested inside of a worksharing, simd "
                               "or worksharing simd loop";
  }
  return success();
}

void OrderedOp::build(OpBuilder &builder, OperationState &state,
                      const OrderedOperands &clauses) {
  OrderedOp::build(builder, state, clauses.doacrossDependType,
                   clauses.doacrossNumLoops, clauses.doacrossDependVars);
}

LogicalResult OrderedOp::verify() {
  if (failed(verifyOrderedParent(**this)))
    return failure();

  auto wrapper = (*this)->getParentOfType<WsloopOp>();
  if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())
    return emitOpError() << "number of variables in depend clause does not "
                         << "match number of iteration variables in the "
                         << "doacross loop";

  return success();
}

void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,
                            const OrderedRegionOperands &clauses) {
  OrderedRegionOp::build(builder, state, clauses.parLevelSimd);
}

LogicalResult OrderedRegionOp::verify() { return verifyOrderedParent(**this); }

//===----------------------------------------------------------------------===//
// TaskwaitOp
//===----------------------------------------------------------------------===//

void TaskwaitOp::build(OpBuilder &builder, OperationState &state,
                       const TaskwaitOperands &clauses) {
  // TODO Store clauses in op: depend_iterated_kinds, depend_iterated, nowait.
  MLIRContext *ctx = builder.getContext();
  TaskwaitOp::build(
      builder, state,
      /*depend_kinds=*/makeArrayAttr(ctx, clauses.dependKinds),
      /*depend_vars=*/clauses.dependVars,
      /*depend_iterated_kinds=*/makeArrayAttr(ctx, clauses.dependIteratedKinds),
      /*depend_iterated=*/ValueRange(clauses.dependIterated),
      /*nowait=*/nullptr);
}

//===----------------------------------------------------------------------===//
// Verifier for AtomicReadOp
//===----------------------------------------------------------------------===//

LogicalResult AtomicReadOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();

  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();

  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Release) {
      return emitError("memory-order must not be release for atomic reads");
    }
    if (*mo == ClauseMemoryOrderKind::Acq_rel) {
      // acq_rel is prohibited on read only in OpenMP 5.0; allowed in 5.1+.
      if (version < 51)
        return emitError("memory-order must not be acq_rel for atomic reads");
    }
  }
  return verifySynchronizationHint(*this, getHint());
}

//===----------------------------------------------------------------------===//
// Verifier for AtomicWriteOp
//===----------------------------------------------------------------------===//

LogicalResult AtomicWriteOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();

  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();

  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Acquire) {
      return emitError("memory-order must not be acquire for atomic writes");
    }
    if (*mo == ClauseMemoryOrderKind::Acq_rel) {
      // acq_rel is prohibited on write only in OpenMP 5.0; allowed in 5.1+.
      if (version < 51)
        return emitError("memory-order must not be acq_rel for atomic writes");
    }
  }
  return verifySynchronizationHint(*this, getHint());
}

//===----------------------------------------------------------------------===//
// Verifier for AtomicUpdateOp
//===----------------------------------------------------------------------===//

LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
                                           PatternRewriter &rewriter) {
  if (op.isNoOp()) {
    rewriter.eraseOp(op);
    return success();
  }
  if (Value writeVal = op.getWriteOpVal()) {
    rewriter.replaceOpWithNewOp<AtomicWriteOp>(
        op, op.getX(), writeVal, op.getHintAttr(), op.getMemoryOrderAttr());
    return success();
  }
  return failure();
}

LogicalResult AtomicUpdateOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();

  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();

  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Acq_rel ||
        *mo == ClauseMemoryOrderKind::Acquire) {
      // This restriction applies only to OpenMP 5.0; removed in 5.1.
      if (version < 51)
        return emitError(
            "memory-order must not be acq_rel or acquire for atomic updates");
    }
  }

  return verifySynchronizationHint(*this, getHint());
}

LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }

//===----------------------------------------------------------------------===//
// Verifier for AtomicCaptureOp
//===----------------------------------------------------------------------===//

AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
  if (auto op = dyn_cast<AtomicReadOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicReadOp>(getSecondOp());
}

AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
  if (auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicWriteOp>(getSecondOp());
}

AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
  if (auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicUpdateOp>(getSecondOp());
}

LogicalResult AtomicCaptureOp::verify() {
  return verifySynchronizationHint(*this, getHint());
}

LogicalResult AtomicCaptureOp::verifyRegions() {
  if (verifyRegionsCommon().failed())
    return mlir::failure();

  if (getFirstOp()->getAttr("hint") || getSecondOp()->getAttr("hint"))
    return emitOpError(
        "operations inside capture region must not have hint clause");

  if (getFirstOp()->getAttr("memory_order") ||
      getSecondOp()->getAttr("memory_order"))
    return emitOpError(
        "operations inside capture region must not have memory_order clause");
  return success();
}

//===----------------------------------------------------------------------===//
// AtomicCompareOp
//===----------------------------------------------------------------------===//

LogicalResult AtomicCompareOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();
  return verifySynchronizationHint(*this, getHint());
}

LogicalResult AtomicCompareOp::verifyRegions() {
  if (verifyRegionsCommon().failed())
    return mlir::failure();

  if (verifyOperator().failed())
    return mlir::failure();

  Block &block = getRegion().front();

  Operation *terminator = block.getTerminator();
  if (!terminator || !isa<YieldOp>(terminator))
    return emitOpError("region must be terminated with omp.yield");

  return success();
}

//===----------------------------------------------------------------------===//
// CancelOp
//===----------------------------------------------------------------------===//

void CancelOp::build(OpBuilder &builder, OperationState &state,
                     const CancelOperands &clauses) {
  CancelOp::build(builder, state, clauses.cancelDirective, clauses.ifExpr);
}

static Operation *getParentInSameDialect(Operation *thisOp) {
  Operation *parent = thisOp->getParentOp();
  while (parent) {
    if (parent->getDialect() == thisOp->getDialect())
      return parent;
    parent = parent->getParentOp();
  }
  return nullptr;
}

LogicalResult CancelOp::verify() {
  ClauseCancellationConstructType cct = getCancelDirective();
  // The next OpenMP operation in the chain of parents
  Operation *structuralParent = getParentInSameDialect((*this).getOperation());
  if (!structuralParent)
    return emitOpError() << "Orphaned cancel construct";

  if ((cct == ClauseCancellationConstructType::Parallel) &&
      !mlir::isa<ParallelOp>(structuralParent)) {
    return emitOpError() << "cancel parallel must appear "
                         << "inside a parallel region";
  }
  if (cct == ClauseCancellationConstructType::Loop) {
    // structural parent will be omp.loop_nest, directly nested inside
    // omp.wsloop
    auto wsloopOp = mlir::dyn_cast<WsloopOp>(structuralParent->getParentOp());

    if (!wsloopOp) {
      return emitOpError()
             << "cancel loop must appear inside a worksharing-loop region";
    }
    if (wsloopOp.getNowaitAttr()) {
      return emitError() << "A worksharing construct that is canceled "
                         << "must not have a nowait clause";
    }
    if (wsloopOp.getOrderedAttr()) {
      return emitError() << "A worksharing construct that is canceled "
                         << "must not have an ordered clause";
    }

  } else if (cct == ClauseCancellationConstructType::Sections) {
    // structural parent will be an omp.section, directly nested inside
    // omp.sections
    auto sectionsOp =
        mlir::dyn_cast<SectionsOp>(structuralParent->getParentOp());
    if (!sectionsOp) {
      return emitOpError() << "cancel sections must appear "
                           << "inside a sections region";
    }
    if (sectionsOp.getNowait()) {
      return emitError() << "A sections construct that is canceled "
                         << "must not have a nowait clause";
    }
  }
  if ((cct == ClauseCancellationConstructType::Taskgroup) &&
      (!mlir::isa<omp::TaskOp>(structuralParent) &&
       !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
    return emitOpError() << "cancel taskgroup must appear "
                         << "inside a task region";
  }
  return success();
}

//===----------------------------------------------------------------------===//
// CancellationPointOp
//===----------------------------------------------------------------------===//

void CancellationPointOp::build(OpBuilder &builder, OperationState &state,
                                const CancellationPointOperands &clauses) {
  CancellationPointOp::build(builder, state, clauses.cancelDirective);
}

LogicalResult CancellationPointOp::verify() {
  ClauseCancellationConstructType cct = getCancelDirective();
  // The next OpenMP operation in the chain of parents
  Operation *structuralParent = getParentInSameDialect((*this).getOperation());
  if (!structuralParent)
    return emitOpError() << "Orphaned cancellation point";

  if ((cct == ClauseCancellationConstructType::Parallel) &&
      !mlir::isa<ParallelOp>(structuralParent)) {
    return emitOpError() << "cancellation point parallel must appear "
                         << "inside a parallel region";
  }
  // Strucutal parent here will be an omp.loop_nest. Get the parent of that to
  // find the wsloop
  if ((cct == ClauseCancellationConstructType::Loop) &&
      !mlir::isa<WsloopOp>(structuralParent->getParentOp())) {
    return emitOpError() << "cancellation point loop must appear "
                         << "inside a worksharing-loop region";
  }
  if ((cct == ClauseCancellationConstructType::Sections) &&
      !mlir::isa<omp::SectionOp>(structuralParent)) {
    return emitOpError() << "cancellation point sections must appear "
                         << "inside a sections region";
  }
  if ((cct == ClauseCancellationConstructType::Taskgroup) &&
      (!mlir::isa<omp::TaskOp>(structuralParent) &&
       !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
    return emitOpError() << "cancellation point taskgroup must appear "
                         << "inside a task region";
  }
  return success();
}

//===----------------------------------------------------------------------===//
// MapBoundsOp
//===----------------------------------------------------------------------===//

LogicalResult MapBoundsOp::verify() {
  auto extent = getExtent();
  auto upperbound = getUpperBound();
  if (!extent && !upperbound)
    return emitError("expected extent or upperbound.");
  return success();
}

void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            TypeRange /*result_types*/, StringAttr symName,
                            TypeAttr type) {
  PrivateClauseOp::build(
      odsBuilder, odsState, symName, type,
      DataSharingClauseTypeAttr::get(odsBuilder.getContext(),
                                     DataSharingClauseType::Private));
}

LogicalResult PrivateClauseOp::verifyRegions() {
  Type argType = getArgType();
  auto verifyTerminator = [&](Operation *terminator,
                              bool yieldsValue) -> LogicalResult {
    if (!terminator->getBlock()->getSuccessors().empty())
      return success();

    if (!llvm::isa<YieldOp>(terminator))
      return mlir::emitError(terminator->getLoc())
             << "expected exit block terminator to be an `omp.yield` op.";

    YieldOp yieldOp = llvm::cast<YieldOp>(terminator);
    TypeRange yieldedTypes = yieldOp.getResults().getTypes();

    if (!yieldsValue) {
      if (yieldedTypes.empty())
        return success();

      return mlir::emitError(terminator->getLoc())
             << "Did not expect any values to be yielded.";
    }

    if (yieldedTypes.size() == 1 && yieldedTypes.front() == argType)
      return success();

    auto error = mlir::emitError(yieldOp.getLoc())
                 << "Invalid yielded value. Expected type: " << argType
                 << ", got: ";

    if (yieldedTypes.empty())
      error << "None";
    else
      error << yieldedTypes;

    return error;
  };

  auto verifyRegion = [&](Region &region, unsigned expectedNumArgs,
                          StringRef regionName,
                          bool yieldsValue) -> LogicalResult {
    assert(!region.empty());

    if (region.getNumArguments() != expectedNumArgs)
      return mlir::emitError(region.getLoc())
             << "`" << regionName << "`: " << "expected " << expectedNumArgs
             << " region arguments, got: " << region.getNumArguments();

    for (Block &block : region) {
      // MLIR will verify the absence of the terminator for us.
      if (!block.mightHaveTerminator())
        continue;

      if (failed(verifyTerminator(block.getTerminator(), yieldsValue)))
        return failure();
    }

    return success();
  };

  // Ensure all of the region arguments have the same type
  for (Region *region : getRegions())
    for (Type ty : region->getArgumentTypes())
      if (ty != argType)
        return emitError() << "Region argument type mismatch: got " << ty
                           << " expected " << argType << ".";

  mlir::Region &initRegion = getInitRegion();
  if (!initRegion.empty() &&
      failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
                          /*yieldsValue=*/true)))
    return failure();

  DataSharingClauseType dsType = getDataSharingType();

  if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
    return emitError("`private` clauses do not require a `copy` region.");

  if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
    return emitError(
        "`firstprivate` clauses require at least a `copy` region.");

  if (dsType == DataSharingClauseType::FirstPrivate &&
      failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
                          /*yieldsValue=*/true)))
    return failure();

  if (!getDeallocRegion().empty() &&
      failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
                          /*yieldsValue=*/false)))
    return failure();

  return success();
}

//===----------------------------------------------------------------------===//
// Spec 5.2: Masked construct (10.5)
//===----------------------------------------------------------------------===//

void MaskedOp::build(OpBuilder &builder, OperationState &state,
                     const MaskedOperands &clauses) {
  MaskedOp::build(builder, state, clauses.filteredThreadId);
}

//===----------------------------------------------------------------------===//
// Spec 5.2: Scan construct (5.6)
//===----------------------------------------------------------------------===//

void ScanOp::build(OpBuilder &builder, OperationState &state,
                   const ScanOperands &clauses) {
  ScanOp::build(builder, state, clauses.inclusiveVars, clauses.exclusiveVars);
}

LogicalResult ScanOp::verify() {
  if (hasExclusiveVars() == hasInclusiveVars())
    return emitError(
        "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");
  if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {
    if (parentWsLoopOp.getReductionModAttr() &&
        parentWsLoopOp.getReductionModAttr().getValue() ==
            ReductionModifier::inscan)
      return success();
  }
  if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {
    if (parentSimdOp.getReductionModAttr() &&
        parentSimdOp.getReductionModAttr().getValue() ==
            ReductionModifier::inscan)
      return success();
  }
  return emitError("SCAN directive needs to be enclosed within a parent "
                   "worksharing loop construct or SIMD construct with INSCAN "
                   "reduction modifier");
}

/// Verifies align clause in allocate directive
LogicalResult verifyAlignment(Operation &op,
                              std::optional<uint64_t> alignment) {
  if (alignment.has_value()) {
    if ((alignment.value() != 0) && !llvm::has_single_bit(alignment.value()))
      return op.emitError()
             << "ALIGN value : " << alignment.value() << " must be power of 2";
  }
  return success();
}

LogicalResult AllocateDirOp::verify() {
  return verifyAlignment(*getOperation(), getAlign());
}

//===----------------------------------------------------------------------===//
// AllocSharedMemOp
//===----------------------------------------------------------------------===//

LogicalResult AllocSharedMemOp::verify() {
  return verifyAlignment(*getOperation(), getMemAlignment());
}

//===----------------------------------------------------------------------===//
// FreeSharedMemOp
//===----------------------------------------------------------------------===//

LogicalResult FreeSharedMemOp::verify() {
  return verifyAlignment(*getOperation(), getMemAlignment());
}

//===----------------------------------------------------------------------===//
// WorkdistributeOp
//===----------------------------------------------------------------------===//

LogicalResult WorkdistributeOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";

  // Check that region exists and is not empty
  Region &region = getRegion();
  if (region.empty())
    return emitOpError("region cannot be empty");
  // Verify single entry point.
  Block &entryBlock = region.front();
  if (entryBlock.empty())
    return emitOpError("region must contain a structured block");
  // Verify single exit point.
  bool hasTerminator = false;
  for (Block &block : region) {
    if (isa<TerminatorOp>(block.back())) {
      if (hasTerminator) {
        return emitOpError("region must have exactly one terminator");
      }
      hasTerminator = true;
    }
  }
  if (!hasTerminator) {
    return emitOpError("region must be terminated with omp.terminator");
  }
  auto walkResult = region.walk([&](Operation *op) -> WalkResult {
    // No implicit barrier at end
    if (isa<BarrierOp>(op)) {
      return emitOpError(
          "explicit barriers are not allowed in workdistribute region");
    }
    // Check for invalid nested constructs
    if (isa<ParallelOp>(op)) {
      return emitOpError(
          "nested parallel constructs not allowed in workdistribute");
    }
    if (isa<TeamsOp>(op)) {
      return emitOpError(
          "nested teams constructs not allowed in workdistribute");
    }
    return WalkResult::advance();
  });
  if (walkResult.wasInterrupted())
    return failure();

  Operation *parentOp = (*this)->getParentOp();
  if (!llvm::dyn_cast<TeamsOp>(parentOp))
    return emitOpError("workdistribute must be nested under teams");
  return success();
}

//===----------------------------------------------------------------------===//
// Declare simd [7.7]
//===----------------------------------------------------------------------===//

LogicalResult DeclareSimdOp::verify() {
  // Must be nested inside a function-like op
  auto func =
      dyn_cast_if_present<mlir::FunctionOpInterface>((*this)->getParentOp());
  if (!func)
    return emitOpError() << "must be nested inside a function";

  if (getInbranch() && getNotinbranch())
    return emitOpError("cannot have both 'inbranch' and 'notinbranch'");

  if (failed(verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars(),
                                   /*isDeclareSimd=*/true)))
    return failure();

  return verifyAlignedClause(*this, getAlignments(), getAlignedVars());
}

void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                          const DeclareSimdOperands &clauses) {
  MLIRContext *ctx = odsBuilder.getContext();
  DeclareSimdOp::build(odsBuilder, odsState, clauses.alignedVars,
                       makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
                       clauses.linearVars, clauses.linearStepVars,
                       clauses.linearVarTypes, clauses.linearModifiers,
                       clauses.notinbranch, clauses.simdlen,
                       clauses.uniformVars);
}

//===----------------------------------------------------------------------===//
// Parser and printer for Uniform Clause
//===----------------------------------------------------------------------===//

/// uniform ::= `uniform` `(` uniform-list `)`
/// uniform-list := uniform-val (`,` uniform-val)*
/// uniform-val := ssa-id `:` type
static ParseResult
parseUniformClause(OpAsmParser &parser,
                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &uniformVars,
                   SmallVectorImpl<Type> &uniformTypes) {
  return parser.parseCommaSeparatedList([&]() -> mlir::ParseResult {
    if (parser.parseOperand(uniformVars.emplace_back()) ||
        parser.parseColonType(uniformTypes.emplace_back()))
      return mlir::failure();
    return mlir::success();
  });
}

/// Print Uniform Clauses
static void printUniformClause(OpAsmPrinter &p, Operation *op,
                               ValueRange uniformVars, TypeRange uniformTypes) {
  for (unsigned i = 0; i < uniformVars.size(); ++i) {
    if (i != 0)
      p << ", ";
    p << uniformVars[i] << " : " << uniformTypes[i];
  }
}

//===----------------------------------------------------------------------===//
// Parser and printer for Affinity Clause
//===----------------------------------------------------------------------===//

static ParseResult parseAffinityClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iterated,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &affinityVars,
    SmallVectorImpl<Type> &iteratedTypes,
    SmallVectorImpl<Type> &affinityVarTypes) {
  if (failed(parseSplitIteratedList(
          parser, iterated, iteratedTypes, affinityVars, affinityVarTypes,
          /*parsePrefix=*/[&]() -> ParseResult { return success(); })))
    return failure();
  return success();
}

static void printAffinityClause(OpAsmPrinter &p, Operation *op,
                                ValueRange iterated, ValueRange affinityVars,
                                TypeRange iteratedTypes,
                                TypeRange affinityVarTypes) {
  auto nop = [&](Value, Type) {};
  printSplitIteratedList(p, iterated, iteratedTypes, affinityVars,
                         affinityVarTypes,
                         /*plain prefix*/ nop,
                         /*iterated prefix*/ nop);
}

//===----------------------------------------------------------------------===//
// Parser, printer, and verifier for Iterator modifier
//===----------------------------------------------------------------------===//

static ParseResult
parseIteratorHeader(OpAsmParser &parser, Region &region,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &lbs,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &ubs,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &steps,
                    SmallVectorImpl<Type> &lbTypes,
                    SmallVectorImpl<Type> &ubTypes,
                    SmallVectorImpl<Type> &stepTypes) {

  llvm::SMLoc ivLoc = parser.getCurrentLocation();
  SmallVector<OpAsmParser::Argument> ivArgs;

  // Parse induction variables: %i : i32, %j : i32
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        OpAsmParser::Argument &arg = ivArgs.emplace_back();
        if (parser.parseArgument(arg))
          return failure();

        // Optional type, default to Index if not provided
        if (succeeded(parser.parseOptionalColon())) {
          if (parser.parseType(arg.type))
            return failure();
        } else {
          arg.type = parser.getBuilder().getIndexType();
        }
        return success();
      }))
    return failure();

  // ) = (
  if (parser.parseRParen() || parser.parseEqual() || parser.parseLParen())
    return failure();

  // Parse Ranges: (%lb to %ub step %st, ...)
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        OpAsmParser::UnresolvedOperand lb, ub, st;
        if (parser.parseOperand(lb) || parser.parseKeyword("to") ||
            parser.parseOperand(ub) || parser.parseKeyword("step") ||
            parser.parseOperand(st))
          return failure();

        lbs.push_back(lb);
        ubs.push_back(ub);
        steps.push_back(st);
        return success();
      }))
    return failure();

  if (parser.parseRParen())
    return failure();

  if (ivArgs.size() != lbs.size())
    return parser.emitError(ivLoc)
           << "mismatch: " << ivArgs.size() << " variables but " << lbs.size()
           << " ranges";

  for (auto &arg : ivArgs) {
    lbTypes.push_back(arg.type);
    ubTypes.push_back(arg.type);
    stepTypes.push_back(arg.type);
  }

  return parser.parseRegion(region, ivArgs);
}

static void printIteratorHeader(OpAsmPrinter &p, Operation *op, Region &region,
                                ValueRange lbs, ValueRange ubs,
                                ValueRange steps, TypeRange, TypeRange,
                                TypeRange) {
  Block &entry = region.front();

  for (unsigned i = 0, e = entry.getNumArguments(); i < e; ++i) {
    if (i != 0)
      p << ", ";
    p.printRegionArgument(entry.getArgument(i));
  }
  p << ") = (";

  // (%lb0 to %ub0 step %step0, %lb1 to %ub1 step %step1, ...)
  for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
    if (i)
      p << ", ";
    p << lbs[i] << " to " << ubs[i] << " step " << steps[i];
  }
  p << ") ";

  p.printRegion(region, /*printEntryBlockArgs=*/false,
                /*printBlockTerminators=*/true);
}

LogicalResult IteratorOp::verify() {
  auto iteratedTy = llvm::dyn_cast<omp::IteratedType>(getIterated().getType());
  if (!iteratedTy)
    return emitOpError() << "result must be omp.iterated<entry_ty>";

  for (auto [lb, ub, step] : llvm::zip_equal(
           getLoopLowerBounds(), getLoopUpperBounds(), getLoopSteps())) {
    if (matchPattern(step, m_Zero()))
      return emitOpError() << "loop step must not be zero";

    IntegerAttr lbAttr;
    IntegerAttr ubAttr;
    IntegerAttr stepAttr;
    if (!matchPattern(lb, m_Constant(&lbAttr)) ||
        !matchPattern(ub, m_Constant(&ubAttr)) ||
        !matchPattern(step, m_Constant(&stepAttr)))
      continue;

    const APInt &lbVal = lbAttr.getValue();
    const APInt &ubVal = ubAttr.getValue();
    const APInt &stepVal = stepAttr.getValue();
    if (stepVal.isStrictlyPositive() && lbVal.sgt(ubVal))
      return emitOpError() << "positive loop step requires lower bound to be "
                              "less than or equal to upper bound";
    if (stepVal.isNegative() && lbVal.slt(ubVal))
      return emitOpError() << "negative loop step requires lower bound to be "
                              "greater than or equal to upper bound";
  }

  Block &b = getRegion().front();
  auto yield = llvm::dyn_cast<omp::YieldOp>(b.getTerminator());

  if (!yield)
    return emitOpError() << "region must be terminated by omp.yield";

  if (yield.getNumOperands() != 1)
    return emitOpError()
           << "omp.yield in omp.iterator region must yield exactly one value";

  mlir::Type yieldedTy = yield.getOperand(0).getType();
  mlir::Type elemTy = iteratedTy.getElementType();

  if (yieldedTy != elemTy)
    return emitOpError() << "omp.iterated element type (" << elemTy
                         << ") does not match omp.yield operand type ("
                         << yieldedTy << ")";

  return success();
}

//===----------------------------------------------------------------------===//
// GroupprivateOp
//===----------------------------------------------------------------------===//

LogicalResult
GroupprivateOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  auto *symbol = symbolTable.lookupNearestSymbolFrom(*this, getSymNameAttr());
  if (!symbol)
    return emitOpError() << "expected symbol reference '" << getSymName()
                         << "' to point to a global variable";

  if (isa<FunctionOpInterface>(symbol))
    return emitOpError() << "expected symbol reference '" << getSymName()
                         << "' to point to a global variable, not a function";

  return success();
}

#define GET_ATTRDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"

#define GET_OP_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"

#define GET_TYPEDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"