| // NOTE: Assertions have been autogenerated by utils/generate-test-checks.py |
| |
| // RUN: mlir-opt -test-gpu-rewrite \ |
| // RUN: -test-gpu-subgroup-reduce-lowering=expand-to-shuffles %s | FileCheck %s |
| |
| gpu.module @kernels { |
| // CHECK-LABEL: gpu.func @minimum( |
| // CHECK-SAME: %[[ARG0:.*]]: f32, |
| // CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup( |
| // CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel { |
| // CHECK-DAG: %[[CONSTANT_0:.*]] = arith.constant 0 : index |
| // CHECK-DAG: %[[CONSTANT_1:.*]] = arith.constant 16 : i32 |
| // CHECK-DAG: %[[CONSTANT_2:.*]] = arith.constant 8 : i32 |
| // CHECK-DAG: %[[CONSTANT_3:.*]] = arith.constant 4 : i32 |
| // CHECK-DAG: %[[CONSTANT_4:.*]] = arith.constant 2 : i32 |
| // CHECK-DAG: %[[CONSTANT_5:.*]] = arith.constant 1 : i32 |
| // CHECK-DAG: %[[CONSTANT_6:.*]] = arith.constant 32 : i32 |
| // CHECK-DAG: %[[CONSTANT_7:.*]] = arith.constant 0 : i32 |
| // CHECK-DAG: %[[CONSTANT_8:.*]] = arith.constant 31 : i32 |
| // CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x |
| // CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32 |
| // CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y |
| // CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32 |
| // CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z |
| // CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32 |
| // CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x |
| // CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32 |
| // CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y |
| // CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32 |
| // CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z |
| // CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32 |
| // CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32 |
| // CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32 |
| // CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32 |
| // CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32 |
| // CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32 |
| // CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32 |
| // CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32 |
| // CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32 |
| // CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32 |
| // CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32 |
| // CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17 |
| // CHECK: ^bb1: |
| // CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3 |
| // CHECK: ^bb2: |
| // CHECK: %[[MINIMUMF_0:.*]] = arith.minimumf %[[ARG0]], %[[VAL_0]] : f32 |
| // CHECK: cf.br ^bb4(%[[MINIMUMF_0]] : f32) |
| // CHECK: ^bb3: |
| // CHECK: cf.br ^bb4(%[[ARG0]] : f32) |
| // CHECK: ^bb4(%[[VAL_1:.*]]: f32): |
| // CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6 |
| // CHECK: ^bb5: |
| // CHECK: %[[MINIMUMF_1:.*]] = arith.minimumf %[[VAL_1]], %[[VAL_2]] : f32 |
| // CHECK: cf.br ^bb7(%[[MINIMUMF_1]] : f32) |
| // CHECK: ^bb6: |
| // CHECK: cf.br ^bb7(%[[VAL_1]] : f32) |
| // CHECK: ^bb7(%[[VAL_3:.*]]: f32): |
| // CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9 |
| // CHECK: ^bb8: |
| // CHECK: %[[MINIMUMF_2:.*]] = arith.minimumf %[[VAL_3]], %[[VAL_4]] : f32 |
| // CHECK: cf.br ^bb10(%[[MINIMUMF_2]] : f32) |
| // CHECK: ^bb9: |
| // CHECK: cf.br ^bb10(%[[VAL_3]] : f32) |
| // CHECK: ^bb10(%[[VAL_5:.*]]: f32): |
| // CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12 |
| // CHECK: ^bb11: |
| // CHECK: %[[MINIMUMF_3:.*]] = arith.minimumf %[[VAL_5]], %[[VAL_6]] : f32 |
| // CHECK: cf.br ^bb13(%[[MINIMUMF_3]] : f32) |
| // CHECK: ^bb12: |
| // CHECK: cf.br ^bb13(%[[VAL_5]] : f32) |
| // CHECK: ^bb13(%[[VAL_7:.*]]: f32): |
| // CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15 |
| // CHECK: ^bb14: |
| // CHECK: %[[MINIMUMF_4:.*]] = arith.minimumf %[[VAL_7]], %[[VAL_8]] : f32 |
| // CHECK: cf.br ^bb16(%[[MINIMUMF_4]] : f32) |
| // CHECK: ^bb15: |
| // CHECK: cf.br ^bb16(%[[VAL_7]] : f32) |
| // CHECK: ^bb16(%[[VAL_9:.*]]: f32): |
| // CHECK: cf.br ^bb18(%[[VAL_9]] : f32) |
| // CHECK: ^bb17: |
| // CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_5:.*]] = arith.minimumf %[[ARG0]], %[[VAL_10]] : f32 |
| // CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MINIMUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_6:.*]] = arith.minimumf %[[MINIMUMF_5]], %[[VAL_11]] : f32 |
| // CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MINIMUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_7:.*]] = arith.minimumf %[[MINIMUMF_6]], %[[VAL_12]] : f32 |
| // CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MINIMUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_8:.*]] = arith.minimumf %[[MINIMUMF_7]], %[[VAL_13]] : f32 |
| // CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MINIMUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_9:.*]] = arith.minimumf %[[MINIMUMF_8]], %[[VAL_14]] : f32 |
| // CHECK: cf.br ^bb18(%[[MINIMUMF_9]] : f32) |
| // CHECK: ^bb18(%[[VAL_15:.*]]: f32): |
| // CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20 |
| // CHECK: ^bb19: |
| // CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index |
| // CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: cf.br ^bb21 |
| // CHECK: ^bb20: |
| // CHECK: cf.br ^bb21 |
| // CHECK: ^bb21: |
| // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>] |
| // CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32 |
| // CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32 |
| // CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41 |
| // CHECK: ^bb22: |
| // CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index |
| // CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39 |
| // CHECK: ^bb23: |
| // CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25 |
| // CHECK: ^bb24: |
| // CHECK: %[[MINIMUMF_10:.*]] = arith.minimumf %[[LOAD_0]], %[[VAL_16]] : f32 |
| // CHECK: cf.br ^bb26(%[[MINIMUMF_10]] : f32) |
| // CHECK: ^bb25: |
| // CHECK: cf.br ^bb26(%[[LOAD_0]] : f32) |
| // CHECK: ^bb26(%[[VAL_17:.*]]: f32): |
| // CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28 |
| // CHECK: ^bb27: |
| // CHECK: %[[MINIMUMF_11:.*]] = arith.minimumf %[[VAL_17]], %[[VAL_18]] : f32 |
| // CHECK: cf.br ^bb29(%[[MINIMUMF_11]] : f32) |
| // CHECK: ^bb28: |
| // CHECK: cf.br ^bb29(%[[VAL_17]] : f32) |
| // CHECK: ^bb29(%[[VAL_19:.*]]: f32): |
| // CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31 |
| // CHECK: ^bb30: |
| // CHECK: %[[MINIMUMF_12:.*]] = arith.minimumf %[[VAL_19]], %[[VAL_20]] : f32 |
| // CHECK: cf.br ^bb32(%[[MINIMUMF_12]] : f32) |
| // CHECK: ^bb31: |
| // CHECK: cf.br ^bb32(%[[VAL_19]] : f32) |
| // CHECK: ^bb32(%[[VAL_21:.*]]: f32): |
| // CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34 |
| // CHECK: ^bb33: |
| // CHECK: %[[MINIMUMF_13:.*]] = arith.minimumf %[[VAL_21]], %[[VAL_22]] : f32 |
| // CHECK: cf.br ^bb35(%[[MINIMUMF_13]] : f32) |
| // CHECK: ^bb34: |
| // CHECK: cf.br ^bb35(%[[VAL_21]] : f32) |
| // CHECK: ^bb35(%[[VAL_23:.*]]: f32): |
| // CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37 |
| // CHECK: ^bb36: |
| // CHECK: %[[MINIMUMF_14:.*]] = arith.minimumf %[[VAL_23]], %[[VAL_24]] : f32 |
| // CHECK: cf.br ^bb38(%[[MINIMUMF_14]] : f32) |
| // CHECK: ^bb37: |
| // CHECK: cf.br ^bb38(%[[VAL_23]] : f32) |
| // CHECK: ^bb38(%[[VAL_25:.*]]: f32): |
| // CHECK: cf.br ^bb40(%[[VAL_25]] : f32) |
| // CHECK: ^bb39: |
| // CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_15:.*]] = arith.minimumf %[[LOAD_0]], %[[VAL_26]] : f32 |
| // CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MINIMUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_16:.*]] = arith.minimumf %[[MINIMUMF_15]], %[[VAL_27]] : f32 |
| // CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MINIMUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_17:.*]] = arith.minimumf %[[MINIMUMF_16]], %[[VAL_28]] : f32 |
| // CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MINIMUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_18:.*]] = arith.minimumf %[[MINIMUMF_17]], %[[VAL_29]] : f32 |
| // CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MINIMUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MINIMUMF_19:.*]] = arith.minimumf %[[MINIMUMF_18]], %[[VAL_30]] : f32 |
| // CHECK: cf.br ^bb40(%[[MINIMUMF_19]] : f32) |
| // CHECK: ^bb40(%[[VAL_31:.*]]: f32): |
| // CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: cf.br ^bb42 |
| // CHECK: ^bb41: |
| // CHECK: cf.br ^bb42 |
| // CHECK: ^bb42: |
| // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>] |
| // CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32> |
| // CHECK: gpu.return |
| // CHECK: } |
| gpu.func @minimum(%arg0 : f32, %out : memref<f32>) kernel { |
| %result = gpu.all_reduce minimumf %arg0 uniform {} : (f32) -> f32 |
| memref.store %result, %out[] : memref<f32> |
| gpu.return |
| } |
| |
| // CHECK-LABEL: gpu.func @maximum( |
| // CHECK-SAME: %[[ARG0:.*]]: f32, |
| // CHECK-SAME: %[[ARG1:.*]]: memref<f32>) workgroup( |
| // CHECK-SAME: %[[ARG2:.*]] : memref<32xf32, #gpu.address_space<workgroup>>) kernel { |
| // CHECK-DAG: %[[CONSTANT_0:.*]] = arith.constant 0 : index |
| // CHECK-DAG: %[[CONSTANT_1:.*]] = arith.constant 16 : i32 |
| // CHECK-DAG: %[[CONSTANT_2:.*]] = arith.constant 8 : i32 |
| // CHECK-DAG: %[[CONSTANT_3:.*]] = arith.constant 4 : i32 |
| // CHECK-DAG: %[[CONSTANT_4:.*]] = arith.constant 2 : i32 |
| // CHECK-DAG: %[[CONSTANT_5:.*]] = arith.constant 1 : i32 |
| // CHECK-DAG: %[[CONSTANT_6:.*]] = arith.constant 32 : i32 |
| // CHECK-DAG: %[[CONSTANT_7:.*]] = arith.constant 0 : i32 |
| // CHECK-DAG: %[[CONSTANT_8:.*]] = arith.constant 31 : i32 |
| // CHECK: %[[BLOCK_DIM_0:.*]] = gpu.block_dim x |
| // CHECK: %[[INDEX_CAST_0:.*]] = arith.index_cast %[[BLOCK_DIM_0]] : index to i32 |
| // CHECK: %[[BLOCK_DIM_1:.*]] = gpu.block_dim y |
| // CHECK: %[[INDEX_CAST_1:.*]] = arith.index_cast %[[BLOCK_DIM_1]] : index to i32 |
| // CHECK: %[[BLOCK_DIM_2:.*]] = gpu.block_dim z |
| // CHECK: %[[INDEX_CAST_2:.*]] = arith.index_cast %[[BLOCK_DIM_2]] : index to i32 |
| // CHECK: %[[THREAD_ID_0:.*]] = gpu.thread_id x |
| // CHECK: %[[INDEX_CAST_3:.*]] = arith.index_cast %[[THREAD_ID_0]] : index to i32 |
| // CHECK: %[[THREAD_ID_1:.*]] = gpu.thread_id y |
| // CHECK: %[[INDEX_CAST_4:.*]] = arith.index_cast %[[THREAD_ID_1]] : index to i32 |
| // CHECK: %[[THREAD_ID_2:.*]] = gpu.thread_id z |
| // CHECK: %[[INDEX_CAST_5:.*]] = arith.index_cast %[[THREAD_ID_2]] : index to i32 |
| // CHECK: %[[MULI_0:.*]] = arith.muli %[[INDEX_CAST_5]], %[[INDEX_CAST_1]] : i32 |
| // CHECK: %[[ADDI_0:.*]] = arith.addi %[[MULI_0]], %[[INDEX_CAST_4]] : i32 |
| // CHECK: %[[MULI_1:.*]] = arith.muli %[[ADDI_0]], %[[INDEX_CAST_0]] : i32 |
| // CHECK: %[[MULI_2:.*]] = arith.muli %[[INDEX_CAST_0]], %[[INDEX_CAST_1]] : i32 |
| // CHECK: %[[ADDI_1:.*]] = arith.addi %[[MULI_1]], %[[INDEX_CAST_3]] : i32 |
| // CHECK: %[[MULI_3:.*]] = arith.muli %[[MULI_2]], %[[INDEX_CAST_2]] : i32 |
| // CHECK: %[[ANDI_0:.*]] = arith.andi %[[ADDI_1]], %[[CONSTANT_8]] : i32 |
| // CHECK: %[[CMPI_0:.*]] = arith.cmpi eq, %[[ANDI_0]], %[[CONSTANT_7]] : i32 |
| // CHECK: %[[SUBI_0:.*]] = arith.subi %[[ADDI_1]], %[[ANDI_0]] : i32 |
| // CHECK: %[[SUBI_1:.*]] = arith.subi %[[MULI_3]], %[[SUBI_0]] : i32 |
| // CHECK: %[[CMPI_1:.*]] = arith.cmpi slt, %[[SUBI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_1]], ^bb1, ^bb17 |
| // CHECK: ^bb1: |
| // CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_0]], ^bb2, ^bb3 |
| // CHECK: ^bb2: |
| // CHECK: %[[MAXIMUMF_0:.*]] = arith.maximumf %[[ARG0]], %[[VAL_0]] : f32 |
| // CHECK: cf.br ^bb4(%[[MAXIMUMF_0]] : f32) |
| // CHECK: ^bb3: |
| // CHECK: cf.br ^bb4(%[[ARG0]] : f32) |
| // CHECK: ^bb4(%[[VAL_1:.*]]: f32): |
| // CHECK: %[[VAL_2:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[VAL_1]], %[[CONSTANT_4]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_1]], ^bb5, ^bb6 |
| // CHECK: ^bb5: |
| // CHECK: %[[MAXIMUMF_1:.*]] = arith.maximumf %[[VAL_1]], %[[VAL_2]] : f32 |
| // CHECK: cf.br ^bb7(%[[MAXIMUMF_1]] : f32) |
| // CHECK: ^bb6: |
| // CHECK: cf.br ^bb7(%[[VAL_1]] : f32) |
| // CHECK: ^bb7(%[[VAL_3:.*]]: f32): |
| // CHECK: %[[VAL_4:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[VAL_3]], %[[CONSTANT_3]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_2]], ^bb8, ^bb9 |
| // CHECK: ^bb8: |
| // CHECK: %[[MAXIMUMF_2:.*]] = arith.maximumf %[[VAL_3]], %[[VAL_4]] : f32 |
| // CHECK: cf.br ^bb10(%[[MAXIMUMF_2]] : f32) |
| // CHECK: ^bb9: |
| // CHECK: cf.br ^bb10(%[[VAL_3]] : f32) |
| // CHECK: ^bb10(%[[VAL_5:.*]]: f32): |
| // CHECK: %[[VAL_6:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[VAL_5]], %[[CONSTANT_2]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_3]], ^bb11, ^bb12 |
| // CHECK: ^bb11: |
| // CHECK: %[[MAXIMUMF_3:.*]] = arith.maximumf %[[VAL_5]], %[[VAL_6]] : f32 |
| // CHECK: cf.br ^bb13(%[[MAXIMUMF_3]] : f32) |
| // CHECK: ^bb12: |
| // CHECK: cf.br ^bb13(%[[VAL_5]] : f32) |
| // CHECK: ^bb13(%[[VAL_7:.*]]: f32): |
| // CHECK: %[[VAL_8:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[VAL_7]], %[[CONSTANT_1]], %[[SUBI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_4]], ^bb14, ^bb15 |
| // CHECK: ^bb14: |
| // CHECK: %[[MAXIMUMF_4:.*]] = arith.maximumf %[[VAL_7]], %[[VAL_8]] : f32 |
| // CHECK: cf.br ^bb16(%[[MAXIMUMF_4]] : f32) |
| // CHECK: ^bb15: |
| // CHECK: cf.br ^bb16(%[[VAL_7]] : f32) |
| // CHECK: ^bb16(%[[VAL_9:.*]]: f32): |
| // CHECK: cf.br ^bb18(%[[VAL_9]] : f32) |
| // CHECK: ^bb17: |
| // CHECK: %[[VAL_10:.*]], %[[SHUFFLE_5:.*]] = gpu.shuffle xor %[[ARG0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_5:.*]] = arith.maximumf %[[ARG0]], %[[VAL_10]] : f32 |
| // CHECK: %[[VAL_11:.*]], %[[SHUFFLE_6:.*]] = gpu.shuffle xor %[[MAXIMUMF_5]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_6:.*]] = arith.maximumf %[[MAXIMUMF_5]], %[[VAL_11]] : f32 |
| // CHECK: %[[VAL_12:.*]], %[[SHUFFLE_7:.*]] = gpu.shuffle xor %[[MAXIMUMF_6]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_7:.*]] = arith.maximumf %[[MAXIMUMF_6]], %[[VAL_12]] : f32 |
| // CHECK: %[[VAL_13:.*]], %[[SHUFFLE_8:.*]] = gpu.shuffle xor %[[MAXIMUMF_7]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_8:.*]] = arith.maximumf %[[MAXIMUMF_7]], %[[VAL_13]] : f32 |
| // CHECK: %[[VAL_14:.*]], %[[SHUFFLE_9:.*]] = gpu.shuffle xor %[[MAXIMUMF_8]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_9:.*]] = arith.maximumf %[[MAXIMUMF_8]], %[[VAL_14]] : f32 |
| // CHECK: cf.br ^bb18(%[[MAXIMUMF_9]] : f32) |
| // CHECK: ^bb18(%[[VAL_15:.*]]: f32): |
| // CHECK: cf.cond_br %[[CMPI_0]], ^bb19, ^bb20 |
| // CHECK: ^bb19: |
| // CHECK: %[[DIVSI_0:.*]] = arith.divsi %[[ADDI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: %[[INDEX_CAST_6:.*]] = arith.index_cast %[[DIVSI_0]] : i32 to index |
| // CHECK: memref.store %[[VAL_15]], %[[ARG2]]{{\[}}%[[INDEX_CAST_6]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: cf.br ^bb21 |
| // CHECK: ^bb20: |
| // CHECK: cf.br ^bb21 |
| // CHECK: ^bb21: |
| // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>] |
| // CHECK: %[[ADDI_2:.*]] = arith.addi %[[MULI_3]], %[[CONSTANT_8]] : i32 |
| // CHECK: %[[DIVSI_1:.*]] = arith.divsi %[[ADDI_2]], %[[CONSTANT_6]] : i32 |
| // CHECK: %[[CMPI_2:.*]] = arith.cmpi slt, %[[ADDI_1]], %[[DIVSI_1]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_2]], ^bb22, ^bb41 |
| // CHECK: ^bb22: |
| // CHECK: %[[INDEX_CAST_7:.*]] = arith.index_cast %[[ADDI_1]] : i32 to index |
| // CHECK: %[[LOAD_0:.*]] = memref.load %[[ARG2]]{{\[}}%[[INDEX_CAST_7]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: %[[CMPI_3:.*]] = arith.cmpi slt, %[[DIVSI_1]], %[[CONSTANT_6]] : i32 |
| // CHECK: cf.cond_br %[[CMPI_3]], ^bb23, ^bb39 |
| // CHECK: ^bb23: |
| // CHECK: %[[VAL_16:.*]], %[[SHUFFLE_10:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_10]], ^bb24, ^bb25 |
| // CHECK: ^bb24: |
| // CHECK: %[[MAXIMUMF_10:.*]] = arith.maximumf %[[LOAD_0]], %[[VAL_16]] : f32 |
| // CHECK: cf.br ^bb26(%[[MAXIMUMF_10]] : f32) |
| // CHECK: ^bb25: |
| // CHECK: cf.br ^bb26(%[[LOAD_0]] : f32) |
| // CHECK: ^bb26(%[[VAL_17:.*]]: f32): |
| // CHECK: %[[VAL_18:.*]], %[[SHUFFLE_11:.*]] = gpu.shuffle xor %[[VAL_17]], %[[CONSTANT_4]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_11]], ^bb27, ^bb28 |
| // CHECK: ^bb27: |
| // CHECK: %[[MAXIMUMF_11:.*]] = arith.maximumf %[[VAL_17]], %[[VAL_18]] : f32 |
| // CHECK: cf.br ^bb29(%[[MAXIMUMF_11]] : f32) |
| // CHECK: ^bb28: |
| // CHECK: cf.br ^bb29(%[[VAL_17]] : f32) |
| // CHECK: ^bb29(%[[VAL_19:.*]]: f32): |
| // CHECK: %[[VAL_20:.*]], %[[SHUFFLE_12:.*]] = gpu.shuffle xor %[[VAL_19]], %[[CONSTANT_3]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_12]], ^bb30, ^bb31 |
| // CHECK: ^bb30: |
| // CHECK: %[[MAXIMUMF_12:.*]] = arith.maximumf %[[VAL_19]], %[[VAL_20]] : f32 |
| // CHECK: cf.br ^bb32(%[[MAXIMUMF_12]] : f32) |
| // CHECK: ^bb31: |
| // CHECK: cf.br ^bb32(%[[VAL_19]] : f32) |
| // CHECK: ^bb32(%[[VAL_21:.*]]: f32): |
| // CHECK: %[[VAL_22:.*]], %[[SHUFFLE_13:.*]] = gpu.shuffle xor %[[VAL_21]], %[[CONSTANT_2]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_13]], ^bb33, ^bb34 |
| // CHECK: ^bb33: |
| // CHECK: %[[MAXIMUMF_13:.*]] = arith.maximumf %[[VAL_21]], %[[VAL_22]] : f32 |
| // CHECK: cf.br ^bb35(%[[MAXIMUMF_13]] : f32) |
| // CHECK: ^bb34: |
| // CHECK: cf.br ^bb35(%[[VAL_21]] : f32) |
| // CHECK: ^bb35(%[[VAL_23:.*]]: f32): |
| // CHECK: %[[VAL_24:.*]], %[[SHUFFLE_14:.*]] = gpu.shuffle xor %[[VAL_23]], %[[CONSTANT_1]], %[[DIVSI_1]] : f32 |
| // CHECK: cf.cond_br %[[SHUFFLE_14]], ^bb36, ^bb37 |
| // CHECK: ^bb36: |
| // CHECK: %[[MAXIMUMF_14:.*]] = arith.maximumf %[[VAL_23]], %[[VAL_24]] : f32 |
| // CHECK: cf.br ^bb38(%[[MAXIMUMF_14]] : f32) |
| // CHECK: ^bb37: |
| // CHECK: cf.br ^bb38(%[[VAL_23]] : f32) |
| // CHECK: ^bb38(%[[VAL_25:.*]]: f32): |
| // CHECK: cf.br ^bb40(%[[VAL_25]] : f32) |
| // CHECK: ^bb39: |
| // CHECK: %[[VAL_26:.*]], %[[SHUFFLE_15:.*]] = gpu.shuffle xor %[[LOAD_0]], %[[CONSTANT_5]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_15:.*]] = arith.maximumf %[[LOAD_0]], %[[VAL_26]] : f32 |
| // CHECK: %[[VAL_27:.*]], %[[SHUFFLE_16:.*]] = gpu.shuffle xor %[[MAXIMUMF_15]], %[[CONSTANT_4]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_16:.*]] = arith.maximumf %[[MAXIMUMF_15]], %[[VAL_27]] : f32 |
| // CHECK: %[[VAL_28:.*]], %[[SHUFFLE_17:.*]] = gpu.shuffle xor %[[MAXIMUMF_16]], %[[CONSTANT_3]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_17:.*]] = arith.maximumf %[[MAXIMUMF_16]], %[[VAL_28]] : f32 |
| // CHECK: %[[VAL_29:.*]], %[[SHUFFLE_18:.*]] = gpu.shuffle xor %[[MAXIMUMF_17]], %[[CONSTANT_2]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_18:.*]] = arith.maximumf %[[MAXIMUMF_17]], %[[VAL_29]] : f32 |
| // CHECK: %[[VAL_30:.*]], %[[SHUFFLE_19:.*]] = gpu.shuffle xor %[[MAXIMUMF_18]], %[[CONSTANT_1]], %[[CONSTANT_6]] : f32 |
| // CHECK: %[[MAXIMUMF_19:.*]] = arith.maximumf %[[MAXIMUMF_18]], %[[VAL_30]] : f32 |
| // CHECK: cf.br ^bb40(%[[MAXIMUMF_19]] : f32) |
| // CHECK: ^bb40(%[[VAL_31:.*]]: f32): |
| // CHECK: memref.store %[[VAL_31]], %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: cf.br ^bb42 |
| // CHECK: ^bb41: |
| // CHECK: cf.br ^bb42 |
| // CHECK: ^bb42: |
| // CHECK: gpu.barrier memfence [#gpu.address_space<workgroup>] |
| // CHECK: %[[LOAD_1:.*]] = memref.load %[[ARG2]]{{\[}}%[[CONSTANT_0]]] : memref<32xf32, #gpu.address_space<workgroup>> |
| // CHECK: memref.store %[[LOAD_1]], %[[ARG1]][] : memref<f32> |
| // CHECK: gpu.return |
| // CHECK: } |
| gpu.func @maximum(%arg0 : f32, %out : memref<f32>) kernel { |
| %result = gpu.all_reduce maximumf %arg0 uniform {} : (f32) -> f32 |
| memref.store %result, %out[] : memref<f32> |
| gpu.return |
| } |
| |
| // CHECK-LABEL: gpu.func @minimum_vector( |
| // CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>, |
| // CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel { |
| // CHECK-DAG: %[[CONSTANT_0:.*]] = arith.constant 16 : i32 |
| // CHECK-DAG: %[[CONSTANT_1:.*]] = arith.constant 8 : i32 |
| // CHECK-DAG: %[[CONSTANT_2:.*]] = arith.constant 4 : i32 |
| // CHECK-DAG: %[[CONSTANT_3:.*]] = arith.constant 2 : i32 |
| // CHECK-DAG: %[[CONSTANT_4:.*]] = arith.constant 32 : i32 |
| // CHECK-DAG: %[[CONSTANT_5:.*]] = arith.constant 1 : i32 |
| // CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MINIMUMF_0:.*]] = arith.minimumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16> |
| // CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MINIMUMF_0]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MINIMUMF_1:.*]] = arith.minimumf %[[MINIMUMF_0]], %[[BITCAST_3]] : vector<2xf16> |
| // CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MINIMUMF_1]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MINIMUMF_2:.*]] = arith.minimumf %[[MINIMUMF_1]], %[[BITCAST_5]] : vector<2xf16> |
| // CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MINIMUMF_2]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MINIMUMF_3:.*]] = arith.minimumf %[[MINIMUMF_2]], %[[BITCAST_7]] : vector<2xf16> |
| // CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MINIMUMF_3]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MINIMUMF_4:.*]] = arith.minimumf %[[MINIMUMF_3]], %[[BITCAST_9]] : vector<2xf16> |
| // CHECK: memref.store %[[MINIMUMF_4]], %[[ARG1]][] : memref<vector<2xf16>> |
| // CHECK: gpu.return |
| // CHECK: } |
| gpu.func @minimum_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel { |
| %result = gpu.subgroup_reduce minimumf %arg0 : (vector<2xf16>) -> vector<2xf16> |
| memref.store %result, %out[] : memref<vector<2xf16>> |
| gpu.return |
| } |
| |
| // CHECK-LABEL: gpu.func @maximum_vector( |
| // CHECK-SAME: %[[ARG0:.*]]: vector<2xf16>, |
| // CHECK-SAME: %[[ARG1:.*]]: memref<vector<2xf16>>) kernel { |
| // CHECK-DAG: %[[CONSTANT_0:.*]] = arith.constant 16 : i32 |
| // CHECK-DAG: %[[CONSTANT_1:.*]] = arith.constant 8 : i32 |
| // CHECK-DAG: %[[CONSTANT_2:.*]] = arith.constant 4 : i32 |
| // CHECK-DAG: %[[CONSTANT_3:.*]] = arith.constant 2 : i32 |
| // CHECK-DAG: %[[CONSTANT_4:.*]] = arith.constant 32 : i32 |
| // CHECK-DAG: %[[CONSTANT_5:.*]] = arith.constant 1 : i32 |
| // CHECK: %[[BITCAST_0:.*]] = vector.bitcast %[[ARG0]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_0:.*]] = vector.extract %[[BITCAST_0]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_0:.*]], %[[SHUFFLE_0:.*]] = gpu.shuffle xor %[[EXTRACT_0]], %[[CONSTANT_5]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_0:.*]] = vector.broadcast %[[VAL_0]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_1:.*]] = vector.bitcast %[[BROADCAST_0]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MAXIMUMF_0:.*]] = arith.maximumf %[[ARG0]], %[[BITCAST_1]] : vector<2xf16> |
| // CHECK: %[[BITCAST_2:.*]] = vector.bitcast %[[MAXIMUMF_0]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_1:.*]] = vector.extract %[[BITCAST_2]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_1:.*]], %[[SHUFFLE_1:.*]] = gpu.shuffle xor %[[EXTRACT_1]], %[[CONSTANT_3]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_1:.*]] = vector.broadcast %[[VAL_1]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_3:.*]] = vector.bitcast %[[BROADCAST_1]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MAXIMUMF_1:.*]] = arith.maximumf %[[MAXIMUMF_0]], %[[BITCAST_3]] : vector<2xf16> |
| // CHECK: %[[BITCAST_4:.*]] = vector.bitcast %[[MAXIMUMF_1]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_2:.*]] = vector.extract %[[BITCAST_4]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_2:.*]], %[[SHUFFLE_2:.*]] = gpu.shuffle xor %[[EXTRACT_2]], %[[CONSTANT_2]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_2:.*]] = vector.broadcast %[[VAL_2]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_5:.*]] = vector.bitcast %[[BROADCAST_2]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MAXIMUMF_2:.*]] = arith.maximumf %[[MAXIMUMF_1]], %[[BITCAST_5]] : vector<2xf16> |
| // CHECK: %[[BITCAST_6:.*]] = vector.bitcast %[[MAXIMUMF_2]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_3:.*]] = vector.extract %[[BITCAST_6]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_3:.*]], %[[SHUFFLE_3:.*]] = gpu.shuffle xor %[[EXTRACT_3]], %[[CONSTANT_1]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_3:.*]] = vector.broadcast %[[VAL_3]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_7:.*]] = vector.bitcast %[[BROADCAST_3]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MAXIMUMF_3:.*]] = arith.maximumf %[[MAXIMUMF_2]], %[[BITCAST_7]] : vector<2xf16> |
| // CHECK: %[[BITCAST_8:.*]] = vector.bitcast %[[MAXIMUMF_3]] : vector<2xf16> to vector<1xi32> |
| // CHECK: %[[EXTRACT_4:.*]] = vector.extract %[[BITCAST_8]][0] : i32 from vector<1xi32> |
| // CHECK: %[[VAL_4:.*]], %[[SHUFFLE_4:.*]] = gpu.shuffle xor %[[EXTRACT_4]], %[[CONSTANT_0]], %[[CONSTANT_4]] : i32 |
| // CHECK: %[[BROADCAST_4:.*]] = vector.broadcast %[[VAL_4]] : i32 to vector<1xi32> |
| // CHECK: %[[BITCAST_9:.*]] = vector.bitcast %[[BROADCAST_4]] : vector<1xi32> to vector<2xf16> |
| // CHECK: %[[MAXIMUMF_4:.*]] = arith.maximumf %[[MAXIMUMF_3]], %[[BITCAST_9]] : vector<2xf16> |
| // CHECK: memref.store %[[MAXIMUMF_4]], %[[ARG1]][] : memref<vector<2xf16>> |
| // CHECK: gpu.return |
| // CHECK: } |
| gpu.func @maximum_vector(%arg0 : vector<2xf16>, %out : memref<vector<2xf16>>) kernel { |
| %result = gpu.subgroup_reduce maximumf %arg0 : (vector<2xf16>) -> vector<2xf16> |
| memref.store %result, %out[] : memref<vector<2xf16>> |
| gpu.return |
| } |
| } |
