| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 6 |
| ; The state machine rewrite has to be on for this test to say anything; it is |
| ; the default upstream but not everywhere, so ask for it explicitly. |
| ; RUN: opt --mtriple=amdgpu-amd-amdhsa --data-layout=A5 -S -passes=openmp-opt \ |
| ; RUN: -openmp-opt-disable-state-machine-rewrite=false < %s | FileCheck %s |
| |
| ; The loop runtime functions take the loop body as a callback, so a parallel |
| ; region inside the body is reached by the kernel through the runtime call. |
| ; When the body is a definition we can see, the state machine should dispatch |
| ; to that region directly; when it is opaque, the kernel has to keep the |
| ; fallback that runs whatever work function it is handed. |
| ; |
| ; Both kernels stay in generic mode. Seeing the body does not make a kernel |
| ; that distributes over teams SPMD-izable: __kmpc_distribute_static_loop_* is |
| ; called by one thread per block by construction, so knowing what the body |
| ; reaches says nothing about running the call itself on every thread. |
| |
| %struct.ident_t = type { i32, i32, i32, i32, ptr } |
| %struct.KernelEnvironmentTy = type { %struct.ConfigurationEnvironmentTy, ptr, ptr } |
| %struct.ConfigurationEnvironmentTy = type { i8, i8, i8, i32, i32, i32, i32, i32, i32 } |
| |
| @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 |
| @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @0 }, align 8 |
| ; The two environments start out identical. What witnesses this change is |
| ; ConfigurationEnvironmentTy's second field, MayUseNestedParallelism: the checks |
| ; have it set for @visible_body, whose body the analysis can look into and find |
| ; the parallel region in, and left clear for @opaque_body. |
| @visible_body_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null } |
| @opaque_body_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 1, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @1, ptr null } |
| |
| ;. |
| ; CHECK: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c" |
| ; CHECK: @[[GLOB1:[0-9]+]] = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8 |
| ; CHECK: @visible_body_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 1, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null } |
| ; CHECK: @opaque_body_kernel_environment = local_unnamed_addr constant %struct.KernelEnvironmentTy { %struct.ConfigurationEnvironmentTy { i8 0, i8 0, i8 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0 }, ptr @[[GLOB1]], ptr null } |
| ;. |
| define weak amdgpu_kernel void @visible_body(ptr %dyn, ptr %payload) #0 { |
| ; CHECK-LABEL: define weak amdgpu_kernel void @visible_body( |
| ; CHECK-SAME: ptr [[DYN:%.*]], ptr [[PAYLOAD:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5) |
| ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @visible_body_kernel_environment, ptr [[DYN]]) |
| ; CHECK-NEXT: [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1 |
| ; CHECK-NEXT: br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]] |
| ; CHECK: [[IS_WORKER_CHECK]]: |
| ; CHECK-NEXT: [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads(i32 0) |
| ; CHECK-NEXT: [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]] |
| ; CHECK-NEXT: br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_BEGIN]]: |
| ; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]]) |
| ; CHECK-NEXT: [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr |
| ; CHECK-NEXT: [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]]) |
| ; CHECK-NEXT: [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8 |
| ; CHECK-NEXT: [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null |
| ; CHECK-NEXT: br i1 [[WORKER_IS_DONE]], label %[[WORKER_STATE_MACHINE_FINISHED]], label %[[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_FINISHED]]: |
| ; CHECK-NEXT: ret void |
| ; CHECK: [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK]]: |
| ; CHECK-NEXT: br i1 [[WORKER_IS_ACTIVE]], label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK:.*]], label %[[WORKER_STATE_MACHINE_DONE_BARRIER:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK]]: |
| ; CHECK-NEXT: br i1 true, label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE:.*]], label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_EXECUTE]]: |
| ; CHECK-NEXT: call void @outlined_wrapper(i16 0, i32 [[TMP0]]) |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_END:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_CHECK1]]: |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_END]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]: |
| ; CHECK-NEXT: call void @__kmpc_kernel_end_parallel() |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_DONE_BARRIER]] |
| ; CHECK: [[WORKER_STATE_MACHINE_DONE_BARRIER]]: |
| ; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]]) |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_BEGIN]] |
| ; CHECK: [[THREAD_USER_CODE_CHECK]]: |
| ; CHECK-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1 |
| ; CHECK-NEXT: br i1 [[EXEC_USER_CODE]], label %[[USER_CODE_ENTRY:.*]], label %[[WORKER_EXIT:.*]] |
| ; CHECK: [[USER_CODE_ENTRY]]: |
| ; CHECK-NEXT: call void @__kmpc_distribute_static_loop_4u(ptr @[[GLOB1]], ptr @loop_body, ptr [[PAYLOAD]], i32 100, i32 0, i8 0) |
| ; CHECK-NEXT: call void @__kmpc_target_deinit() |
| ; CHECK-NEXT: ret void |
| ; CHECK: [[WORKER_EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %0 = call i32 @__kmpc_target_init(ptr @visible_body_kernel_environment, ptr %dyn) |
| %exec_user_code = icmp eq i32 %0, -1 |
| br i1 %exec_user_code, label %user_code.entry, label %worker.exit |
| |
| user_code.entry: |
| call void @__kmpc_distribute_static_loop_4u(ptr @1, ptr @loop_body, ptr %payload, i32 100, i32 0, i8 0) |
| call void @__kmpc_target_deinit() |
| ret void |
| |
| worker.exit: |
| ret void |
| } |
| |
| define weak amdgpu_kernel void @opaque_body(ptr %dyn, ptr %payload) #0 { |
| ; CHECK-LABEL: define weak amdgpu_kernel void @opaque_body( |
| ; CHECK-SAME: ptr [[DYN:%.*]], ptr [[PAYLOAD:%.*]]) #[[ATTR0]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[WORKER_WORK_FN_ADDR:%.*]] = alloca ptr, align 8, addrspace(5) |
| ; CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_target_init(ptr @opaque_body_kernel_environment, ptr [[DYN]]) |
| ; CHECK-NEXT: [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1 |
| ; CHECK-NEXT: br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]] |
| ; CHECK: [[IS_WORKER_CHECK]]: |
| ; CHECK-NEXT: [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads(i32 0) |
| ; CHECK-NEXT: [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]] |
| ; CHECK-NEXT: br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_BEGIN]]: |
| ; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]]) |
| ; CHECK-NEXT: [[WORKER_WORK_FN_ADDR_GENERIC:%.*]] = addrspacecast ptr addrspace(5) [[WORKER_WORK_FN_ADDR]] to ptr |
| ; CHECK-NEXT: [[WORKER_IS_ACTIVE:%.*]] = call i1 @__kmpc_kernel_parallel(ptr [[WORKER_WORK_FN_ADDR_GENERIC]]) |
| ; CHECK-NEXT: [[WORKER_WORK_FN:%.*]] = load ptr, ptr [[WORKER_WORK_FN_ADDR_GENERIC]], align 8 |
| ; CHECK-NEXT: [[WORKER_IS_DONE:%.*]] = icmp eq ptr [[WORKER_WORK_FN]], null |
| ; CHECK-NEXT: br i1 [[WORKER_IS_DONE]], label %[[WORKER_STATE_MACHINE_FINISHED]], label %[[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_FINISHED]]: |
| ; CHECK-NEXT: ret void |
| ; CHECK: [[WORKER_STATE_MACHINE_IS_ACTIVE_CHECK]]: |
| ; CHECK-NEXT: br i1 [[WORKER_IS_ACTIVE]], label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE:.*]], label %[[WORKER_STATE_MACHINE_DONE_BARRIER:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_FALLBACK_EXECUTE]]: |
| ; CHECK-NEXT: call void [[WORKER_WORK_FN]](i16 0, i32 [[TMP0]]) |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_PARALLEL_REGION_END:.*]] |
| ; CHECK: [[WORKER_STATE_MACHINE_PARALLEL_REGION_END]]: |
| ; CHECK-NEXT: call void @__kmpc_kernel_end_parallel() |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_DONE_BARRIER]] |
| ; CHECK: [[WORKER_STATE_MACHINE_DONE_BARRIER]]: |
| ; CHECK-NEXT: call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]]) |
| ; CHECK-NEXT: br label %[[WORKER_STATE_MACHINE_BEGIN]] |
| ; CHECK: [[THREAD_USER_CODE_CHECK]]: |
| ; CHECK-NEXT: [[EXEC_USER_CODE:%.*]] = icmp eq i32 [[TMP0]], -1 |
| ; CHECK-NEXT: br i1 [[EXEC_USER_CODE]], label %[[USER_CODE_ENTRY:.*]], label %[[WORKER_EXIT:.*]] |
| ; CHECK: [[USER_CODE_ENTRY]]: |
| ; CHECK-NEXT: call void @__kmpc_distribute_static_loop_4u(ptr @[[GLOB1]], ptr @opaque_loop_body, ptr [[PAYLOAD]], i32 100, i32 0, i8 0) |
| ; CHECK-NEXT: call void @__kmpc_target_deinit() |
| ; CHECK-NEXT: ret void |
| ; CHECK: [[WORKER_EXIT]]: |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %0 = call i32 @__kmpc_target_init(ptr @opaque_body_kernel_environment, ptr %dyn) |
| %exec_user_code = icmp eq i32 %0, -1 |
| br i1 %exec_user_code, label %user_code.entry, label %worker.exit |
| |
| user_code.entry: |
| call void @__kmpc_distribute_static_loop_4u(ptr @1, ptr @opaque_loop_body, ptr %payload, i32 100, i32 0, i8 0) |
| call void @__kmpc_target_deinit() |
| ret void |
| |
| worker.exit: |
| ret void |
| } |
| |
| define internal void @loop_body(i32 %iv, ptr %payload) #1 { |
| ; CHECK-LABEL: define internal void @loop_body( |
| ; CHECK-SAME: i32 [[IV:%.*]], ptr [[PAYLOAD:%.*]]) #[[ATTR1:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[TID:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]]) #[[ATTR1]] |
| ; CHECK-NEXT: call void @__kmpc_parallel_60(ptr @[[GLOB1]], i32 [[TID]], i32 1, i32 -1, i32 -1, ptr @outlined, ptr @outlined_wrapper, ptr [[PAYLOAD]], i64 1, i32 0) |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %tid = call i32 @__kmpc_global_thread_num(ptr @1) |
| call void @__kmpc_parallel_60(ptr @1, i32 %tid, i32 1, i32 -1, i32 -1, ptr @outlined, ptr @outlined_wrapper, ptr %payload, i64 1, i32 0) |
| ret void |
| } |
| |
| define internal void @outlined(ptr noalias %tid.addr, ptr noalias %zero.addr) #1 { |
| ; CHECK-LABEL: define internal void @outlined( |
| ; CHECK-SAME: ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]]) #[[ATTR2:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[TID:%.*]] = load i32, ptr [[TID_ADDR]], align 4 |
| ; CHECK-NEXT: [[ZERO:%.*]] = load i32, ptr [[ZERO_ADDR]], align 4 |
| ; CHECK-NEXT: call void @p0(i32 [[TID]], i32 [[ZERO]]) #[[ATTR1]] |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %tid = load i32, ptr %tid.addr, align 4 |
| %zero = load i32, ptr %zero.addr, align 4 |
| call void @p0(i32 %tid, i32 %zero) |
| ret void |
| } |
| |
| define internal void @outlined_wrapper(i16 zeroext %0, i32 %1) #1 { |
| ; CHECK-LABEL: define internal void @outlined_wrapper( |
| ; CHECK-SAME: i16 zeroext [[TMP0:%.*]], i32 [[TMP1:%.*]]) #[[ATTR2]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[TID_ADDR:%.*]] = alloca i32, align 4, addrspace(5) |
| ; CHECK-NEXT: [[TID_ADDR_CAST:%.*]] = addrspacecast ptr addrspace(5) [[TID_ADDR]] to ptr |
| ; CHECK-NEXT: [[ZERO_ADDR:%.*]] = alloca i32, align 4, addrspace(5) |
| ; CHECK-NEXT: [[ZERO_ADDR_CAST:%.*]] = addrspacecast ptr addrspace(5) [[ZERO_ADDR]] to ptr |
| ; CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(5) [[TID_ADDR]], align 4 |
| ; CHECK-NEXT: store i32 0, ptr addrspace(5) [[ZERO_ADDR]], align 4 |
| ; CHECK-NEXT: call void @outlined(ptr [[TID_ADDR_CAST]], ptr [[ZERO_ADDR_CAST]]) #[[ATTR1]] |
| ; CHECK-NEXT: ret void |
| ; |
| entry: |
| %tid.addr = alloca i32, align 4, addrspace(5) |
| %tid.addr.cast = addrspacecast ptr addrspace(5) %tid.addr to ptr |
| %zero.addr = alloca i32, align 4, addrspace(5) |
| %zero.addr.cast = addrspacecast ptr addrspace(5) %zero.addr to ptr |
| store i32 %1, ptr %tid.addr.cast, align 4 |
| store i32 0, ptr %zero.addr.cast, align 4 |
| call void @outlined(ptr %tid.addr.cast, ptr %zero.addr.cast) |
| ret void |
| } |
| |
| define weak i32 @__kmpc_target_init(ptr %0, ptr %1) { |
| ; CHECK-LABEL: define weak i32 @__kmpc_target_init( |
| ; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]]) { |
| ; CHECK-NEXT: ret i32 0 |
| ; |
| ret i32 0 |
| } |
| |
| declare void @opaque_loop_body(i32, ptr) |
| declare void @p0(i32, i32) #1 |
| declare void @__kmpc_target_deinit() |
| declare i32 @__kmpc_global_thread_num(ptr) |
| declare void @__kmpc_parallel_60(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64, i32) |
| declare void @__kmpc_distribute_static_loop_4u(ptr, ptr, ptr, i32, i32, i8) |
| |
| attributes #0 = { "kernel" } |
| attributes #1 = { convergent nounwind } |
| |
| !llvm.module.flags = !{!0, !1} |
| |
| !0 = !{i32 7, !"openmp", i32 50} |
| !1 = !{i32 7, !"openmp-device", i32 50} |
| ;. |
| ; CHECK: attributes #[[ATTR0]] = { "kernel" } |
| ; CHECK: attributes #[[ATTR1]] = { nounwind } |
| ; CHECK: attributes #[[ATTR2]] = { convergent nounwind } |
| ; CHECK: attributes #[[ATTR3:[0-9]+]] = { alwaysinline } |
| ;. |
| ; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50} |
| ; CHECK: [[META1:![0-9]+]] = !{i32 7, !"openmp-device", i32 50} |
| ; CHECK: [[META2:![0-9]+]] = !{[[META3:![0-9]+]]} |
| ; CHECK: [[META3]] = !{i64 1, i64 -1, i64 -1, i1 false} |
| ;. |
