blob: d7f9fcf9844b499004de95ed73bc0031c96f69a6 [file] [edit]
// RUN: mlir-opt %s -acc-to-llvm -split-input-file | FileCheck %s
// A parallel construct with a copy clause and a scalar that has no clause. The
// data region of the construct is its implicit data region, so its entry and
// exit calls carry that region flag. The entry call returns the device address,
// which the kernel receives as a device pointer. The scalar is passed by value.
// A kernel that the module does not define is declared, and the runtime
// identifies it by the address of that declaration.
// copy = to|from|return_param = 0x43 = 67.
// devptr|literal = 0x500 = 1280.
// literal = 0x100 = 256.
// CHECK-LABEL: llvm.func @loop_0()
// CHECK: llvm.func @parallel_copy
// CHECK: llvm.mlir.addressof @acc.ident
// CHECK: %[[IMPLICIT:.*]] = llvm.mlir.constant(1 : i64) : i64
// CHECK: llvm.mlir.constant(67 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_begin(%{{[^,]+}}, %[[IMPLICIT]],
// CHECK: %[[A_SLOT:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.ptr
// CHECK: %[[A_DEVICE:.*]] = llvm.load %[[A_SLOT]] : !llvm.ptr -> !llvm.ptr
// CHECK: %[[KERNEL:.*]] = llvm.mlir.addressof @loop_0 : !llvm.ptr
// CHECK: llvm.mlir.constant(1280 : i64) : i64
// CHECK: llvm.mlir.constant(256 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel(%{{[^,]+}}, %[[KERNEL]],
// CHECK: llvm.call @abort
// CHECK: %[[IMPLICIT_END:.*]] = llvm.mlir.constant(1 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_end(%{{[^,]+}}, %[[IMPLICIT_END]],
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @loop_0(%a: !llvm.ptr, %n: i64) kernel {
gpu.return
}
}
func.func @parallel_copy(%a: !llvm.ptr, %n: i64) {
%c1 = arith.constant 1 : index
%c128 = arith.constant 128 : index
%copy = acc.copyin varPtr(%a : !llvm.ptr) varType(f32)
dataClause(acc_copy) name("a") -> !llvm.ptr
acc.kernel_environment dataOperands(%copy : !llvm.ptr) {
gpu.launch_func @device_module::@loop_0
blocks in (%c1, %c1, %c1) threads in (%c128, %c1, %c1)
args(%copy : !llvm.ptr, %n : i64)
}
acc.copyout accPtr(%copy : !llvm.ptr) to varPtr(%a : !llvm.ptr)
varType(f32) dataClause(acc_copy) name("a")
return
}
}
// -----
// A deviceptr clause names an address that is already on the device. It is
// mapped as devptr = 0x400 = 1024, and no device address is read back for it.
// CHECK-LABEL: llvm.func @parallel_deviceptr
// CHECK-NOT: llvm.mlir.constant(64 : i64)
// CHECK: %[[DEVPTR:.*]] = llvm.mlir.constant(1024 : i64) : i64
// CHECK: llvm.store %[[DEVPTR]], %{{.*}} : i64, !llvm.ptr
// CHECK: llvm.call @__tgt_acc_data_begin
// CHECK-NOT: llvm.mlir.constant(64 : i64)
// CHECK: llvm.call @__tgt_acc_kernel
// CHECK: llvm.call @__tgt_acc_data_end
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%p: !llvm.ptr) kernel {
gpu.return
}
}
func.func @parallel_deviceptr(%p: !llvm.ptr) {
%c1 = arith.constant 1 : index
%dev = acc.deviceptr varPtr(%p : !llvm.ptr) varType(f32) name("p")
-> !llvm.ptr
acc.kernel_environment dataOperands(%dev : !llvm.ptr) {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%dev : !llvm.ptr)
}
return
}
}
// -----
// A present clause maps its operand with present = 0x100000 = 1048576, and its
// device address is read back through return_param = 0x40 = 64. The kernel
// receives that address.
// CHECK-LABEL: llvm.func @parallel_present
// CHECK: %[[PRESENT:.*]] = llvm.mlir.constant(1048640 : i64) : i64
// CHECK: llvm.store %[[PRESENT]], %{{.*}} : i64, !llvm.ptr
// CHECK: llvm.call @__tgt_acc_data_begin
// CHECK: llvm.load %{{.*}} : !llvm.ptr -> !llvm.ptr
// CHECK: llvm.mlir.constant(1280 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%a: !llvm.ptr) kernel {
gpu.return
}
}
func.func @parallel_present(%a: !llvm.ptr) {
%c1 = arith.constant 1 : index
%present = acc.present varPtr(%a : !llvm.ptr) varType(f32) name("a")
-> !llvm.ptr
acc.kernel_environment dataOperands(%present : !llvm.ptr) {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%present : !llvm.ptr)
}
return
}
}
// -----
// A construct with async and wait clauses and no data clauses. The wait
// precedes the launch, and the async queue of the construct is the queue of the
// kernel call. The launch geometry fills the argument struct. The vector length
// is the first block dimension, and the number of workers is the product of the
// other two. Any status other than success reaches the host fallback.
// literal = 0x100 = 256.
// CHECK-LABEL: llvm.func @parallel_async_wait
// CHECK-SAME: (%[[X:.*]]: i32, %[[SMEM:.*]]: i32, %[[ASYNC:.*]]: i32, %[[WAIT:.*]]: i32)
// CHECK-NOT: llvm.call @__tgt_acc_data_begin
// CHECK: %[[GRID_X:.*]] = llvm.mlir.constant(2 : i64) : i64
// CHECK: %[[GRID_Y:.*]] = llvm.mlir.constant(3 : i64) : i64
// CHECK: %[[GRID_Z:.*]] = llvm.mlir.constant(5 : i64) : i64
// CHECK: %[[BLOCK_X:.*]] = llvm.mlir.constant(8 : i64) : i64
// CHECK: %[[BLOCK_Y:.*]] = llvm.mlir.constant(6 : i64) : i64
// CHECK: %[[BLOCK_Z:.*]] = llvm.mlir.constant(7 : i64) : i64
// CHECK: %[[QUEUE:.*]] = llvm.sext %[[ASYNC]] : i32 to i64
// CHECK: llvm.call @__tgt_acc_wait
// CHECK: %[[KERNEL:.*]] = llvm.mlir.addressof @region_0 : !llvm.ptr
// CHECK: llvm.store %[[X]], %{{.*}} : i32, !llvm.ptr
// CHECK: llvm.mlir.constant(256 : i64) : i64
// CHECK: %[[WORKERS:.*]] = llvm.mul %[[BLOCK_Y]], %[[BLOCK_Z]] : i64
// CHECK: llvm.insertvalue %[[WORKERS]], %{{.*}}[11]
// CHECK: llvm.insertvalue %[[BLOCK_X]], %{{.*}}[12]
// CHECK: %[[SMEM64:.*]] = llvm.sext %[[SMEM]] : i32 to i64
// CHECK: llvm.insertvalue %[[SMEM64]], %{{.*}}[13]
// CHECK: llvm.insertvalue %[[GRID_X]], %{{.*}}[10, 0]
// CHECK: llvm.insertvalue %[[GRID_Y]], %{{.*}}[10, 1]
// CHECK: llvm.insertvalue %[[GRID_Z]], %{{.*}}[10, 2]
// CHECK: llvm.mlir.addressof @acc.kernel_name.region_0
// CHECK: llvm.mlir.addressof @acc.ident
// CHECK: %[[KERNEL_FLAGS:.*]] = llvm.mlir.constant(0 : i64) : i64
// CHECK: %[[STATUS:.*]] = llvm.call @__tgt_acc_kernel(%{{[^,]+}}, %[[KERNEL]], %[[KERNEL_FLAGS]], %{{[^,]+}}, %{{[^,]+}}, %[[QUEUE]], %{{[^,]+}}, %{{[^,]+}})
// CHECK: %[[SUCCESS:.*]] = llvm.mlir.constant(0 : i32) : i32
// CHECK: %[[FAILED:.*]] = llvm.icmp "ne" %[[STATUS]], %[[SUCCESS]] : i32
// CHECK: llvm.cond_br %[[FAILED]]
// CHECK-NOT: llvm.call @__tgt_acc_data_end
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%x: i32) kernel {
gpu.return
}
}
func.func @parallel_async_wait(%x: i32, %smem: i32, %async: i32, %wait: i32) {
%c2 = arith.constant 2 : index
%c3 = arith.constant 3 : index
%c5 = arith.constant 5 : index
%c8 = arith.constant 8 : index
%c6 = arith.constant 6 : index
%c7 = arith.constant 7 : index
acc.kernel_environment async(%async : i32) wait(%wait : i32) {
gpu.launch_func @device_module::@region_0
blocks in (%c2, %c3, %c5) threads in (%c8, %c6, %c7)
dynamic_shared_memory_size %smem args(%x : i32)
}
return
}
}
// -----
// The runtime identifies a kernel by the address of a host function that has
// the kernel's name. The call takes the address of that function when the
// module defines it, so no declaration is added.
// CHECK-LABEL: llvm.func internal @region_0()
// CHECK-NOT: llvm.func @region_0
// CHECK-LABEL: llvm.func @parallel_with_host_stub
// CHECK: %[[KERNEL:.*]] = llvm.mlir.addressof @region_0 : !llvm.ptr
// CHECK: llvm.call @__tgt_acc_kernel(%{{[^,]+}}, %[[KERNEL]],
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%a: !llvm.ptr) kernel {
gpu.return
}
}
llvm.func internal @region_0() {
llvm.return
}
func.func @parallel_with_host_stub(%a: !llvm.ptr) {
%c1 = arith.constant 1 : index
acc.kernel_environment {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%a : !llvm.ptr)
}
return
}
}
// -----
// A memref operand is mapped as its descriptor fields, and only the first
// field, the one holding the allocated pointer, asks the runtime to return the
// device address. copy = to|from|return_param = 0x43 = 67 and to|from =
// 0x3 = 3.
// CHECK-LABEL: llvm.func @parallel_copy_memref
// CHECK: llvm.mlir.constant(67 : i64) : i64
// CHECK: llvm.mlir.constant(3 : i64) : i64
// CHECK: llvm.mlir.constant(3 : i64) : i64
// CHECK: llvm.mlir.constant(3 : i64) : i64
// CHECK: llvm.mlir.constant(3 : i64) : i64
// CHECK-NOT: llvm.mlir.constant(67 : i64)
// CHECK: llvm.call @__tgt_acc_data_begin
// CHECK: llvm.call @__tgt_acc_kernel
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%a: memref<100xf32>) kernel {
gpu.return
}
}
func.func @parallel_copy_memref(%a: memref<100xf32>) {
%c1 = arith.constant 1 : index
%copy = acc.copyin varPtr(%a : memref<100xf32>) varType(f32)
dataClause(acc_copy) name("a") -> memref<100xf32>
acc.kernel_environment dataOperands(%copy : memref<100xf32>) {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%copy : memref<100xf32>)
}
acc.copyout accPtr(%copy : memref<100xf32>) to varPtr(%a : memref<100xf32>)
varType(f32) dataClause(acc_copy) name("a")
return
}
}
// -----
// A memref that no clause maps is described to the runtime like a mapped one.
// The function parameter is converted to a descriptor, so the element size
// comes from the memref type.
// CHECK-LABEL: llvm.func @parallel_unmapped_memref
// CHECK: llvm.mlir.constant(1 : i8) : i8
// CHECK: llvm.alloca %{{.*}} x !llvm.struct<(i32, i8, i64, ptr)>
// CHECK: llvm.call @__tgt_acc_kernel
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%m: memref<4xf32>) kernel {
gpu.return
}
}
func.func @parallel_unmapped_memref(%m: memref<4xf32>) {
%c1 = arith.constant 1 : index
acc.kernel_environment {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%m : memref<4xf32>)
}
return
}
}
// -----
// An object of a declare device_resident clause is mapped by the declare
// directive outside of the environment, which has no data operand for it. The
// kernel looks up that mapping with present = 0x100000 = 1048576, and not with
// the device_resident = 0x40000 = 262144 of the declaration.
// CHECK-LABEL: llvm.func @parallel_declare_device_resident
// CHECK: llvm.mlir.constant(262144 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_begin
// CHECK-NOT: llvm.call @__tgt_acc_data_begin
// CHECK: llvm.mlir.constant(1048576 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel
// CHECK: llvm.call @__tgt_acc_data_end
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%b: !llvm.ptr) kernel {
gpu.return
}
}
func.func @parallel_declare_device_resident(%b: !llvm.ptr) {
%c1 = arith.constant 1 : index
%resident = acc.declare_device_resident varPtr(%b : !llvm.ptr)
varType(f32) name("b") -> !llvm.ptr
%token = acc.declare_enter dataOperands(%resident : !llvm.ptr)
acc.kernel_environment {
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%resident : !llvm.ptr)
}
acc.declare_exit token(%token) dataOperands(%resident : !llvm.ptr)
acc.delete accPtr(%resident : !llvm.ptr)
dataClause(acc_declare_device_resident) name("b")
return
}
}
// -----
// A gang private variable is a data entry in the environment region, since the
// storage is created for the kernel. The environment has no data operand for
// it, so the kernel call maps it, with private|gang_private =
// 0x4080 = 16512. The copied array is still mapped by the environment.
// CHECK-LABEL: llvm.func @parallel_gang_private
// CHECK: llvm.mlir.constant(67 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_begin
// CHECK-NOT: llvm.call @__tgt_acc_data_begin
// CHECK: llvm.mlir.constant(1280 : i64) : i64
// CHECK: llvm.mlir.constant(16512 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel
// CHECK: llvm.call @__tgt_acc_data_end
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @region_0(%a: !llvm.ptr,
%p: !acc.private_type<memref<99xi32>>) kernel {
gpu.return
}
}
func.func @parallel_gang_private(%a: !llvm.ptr) {
%c1 = arith.constant 1 : index
%size = arith.constant 396 : i64
%copy = acc.copyin varPtr(%a : !llvm.ptr) varType(f32)
dataClause(acc_copy) name("a") -> !llvm.ptr
acc.kernel_environment dataOperands(%copy : !llvm.ptr) {
%priv = acc.privatize par_dims(#acc<par_dims[block_x]>)
: () -> !acc.private_type<memref<99xi32>>
%map = acc.map_info varPtr(%priv : !acc.private_type<memref<99xi32>>)
size(%size : i64) elementSize(4) descKind(none)
mapFlags(private, gang_private) -> !acc.private_type<memref<99xi32>>
gpu.launch_func @device_module::@region_0
blocks in (%c1, %c1, %c1) threads in (%c1, %c1, %c1)
args(%copy : !llvm.ptr, %map : !acc.private_type<memref<99xi32>>)
}
acc.copyout accPtr(%copy : !llvm.ptr) to varPtr(%a : !llvm.ptr)
varType(f32) dataClause(acc_copy) name("a")
return
}
}
// -----
// Several launches in one kernel environment share its data region, as the
// kernels of a split kernels construct would. The array a has an explicit copy
// clause and the array b is copied implicitly. The entry call returns both
// device addresses, and every kernel receives them as device pointers.
// copy = to|from|return_param = 0x43 = 67.
// implicit copy = to|from|implicit|return_param = 0x243 = 579.
// devptr|literal = 0x500 = 1280, and devptr|literal|implicit = 0x700 = 1792.
// literal = 0x100 = 256.
// CHECK-LABEL: llvm.func @kernels_with_copy
// CHECK: llvm.mlir.addressof @acc.ident
// CHECK: %[[IMPLICIT:.*]] = llvm.mlir.constant(1 : i64) : i64
// CHECK: llvm.mlir.constant(67 : i64) : i64
// CHECK: llvm.mlir.constant(579 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_begin(%{{[^,]+}}, %[[IMPLICIT]],
// CHECK: %[[A_SLOT:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.ptr
// CHECK: %[[A_DEVICE:.*]] = llvm.load %[[A_SLOT]] : !llvm.ptr -> !llvm.ptr
// CHECK: %[[B_SLOT:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i32) -> !llvm.ptr, !llvm.ptr
// CHECK: %[[B_DEVICE:.*]] = llvm.load %[[B_SLOT]] : !llvm.ptr -> !llvm.ptr
// CHECK: llvm.mlir.constant(1280 : i64) : i64
// CHECK: llvm.mlir.constant(1792 : i64) : i64
// CHECK: llvm.mlir.constant(256 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel
// CHECK: llvm.call @abort
// CHECK-NOT: llvm.call @__tgt_acc_data_begin
// CHECK: llvm.mlir.constant(1280 : i64) : i64
// CHECK: llvm.mlir.constant(1792 : i64) : i64
// CHECK: llvm.mlir.constant(256 : i64) : i64
// CHECK: llvm.call @__tgt_acc_kernel
// CHECK: llvm.call @abort
// CHECK: %[[IMPLICIT_END:.*]] = llvm.mlir.constant(1 : i64) : i64
// CHECK: llvm.call @__tgt_acc_data_end(%{{[^,]+}}, %[[IMPLICIT_END]],
module attributes {gpu.container_module} {
gpu.module @device_module {
gpu.func @loop_0(%a: !llvm.ptr, %b: !llvm.ptr, %n: i64) kernel {
gpu.return
}
gpu.func @loop_1(%a: !llvm.ptr, %b: !llvm.ptr, %n: i64) kernel {
gpu.return
}
}
func.func @kernels_with_copy(%a: !llvm.ptr, %b: !llvm.ptr, %n: i64) {
%c1 = arith.constant 1 : index
%c128 = arith.constant 128 : index
%copy_a = acc.copyin varPtr(%a : !llvm.ptr) varType(f32)
dataClause(acc_copy) name("a") -> !llvm.ptr
%copy_b = acc.copyin varPtr(%b : !llvm.ptr) varType(f32)
dataClause(acc_copy) implicit(true) name("b") -> !llvm.ptr
acc.kernel_environment dataOperands(%copy_a, %copy_b : !llvm.ptr, !llvm.ptr) {
gpu.launch_func @device_module::@loop_0
blocks in (%c1, %c1, %c1) threads in (%c128, %c1, %c1)
args(%copy_a : !llvm.ptr, %copy_b : !llvm.ptr, %n : i64)
gpu.launch_func @device_module::@loop_1
blocks in (%c1, %c1, %c1) threads in (%c128, %c1, %c1)
args(%copy_a : !llvm.ptr, %copy_b : !llvm.ptr, %n : i64)
}
acc.copyout accPtr(%copy_a : !llvm.ptr) to varPtr(%a : !llvm.ptr)
varType(f32) dataClause(acc_copy) name("a")
acc.copyout accPtr(%copy_b : !llvm.ptr) to varPtr(%b : !llvm.ptr)
varType(f32) dataClause(acc_copy) implicit(true) name("b")
return
}
}