Stanislav Mekhanoshin | 9ee272f | 2020-06-15 14:10:39 -0700 | [diff] [blame] | 1 | ; RUN: llc -march=amdgcn -mattr=+fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s |
| 2 | ; RUN: llc -march=amdgcn -mattr=-fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 3 | |
Stanislav Mekhanoshin | 9ee272f | 2020-06-15 14:10:39 -0700 | [diff] [blame] | 4 | ; RUN: llc -march=amdgcn -mattr=+fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=ieee -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FASTFMA %s |
| 5 | ; RUN: llc -march=amdgcn -mattr=-fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=ieee -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SLOWFMA %s |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 6 | |
| 7 | ; FIXME: This should also fold when fma is actually fast if an FMA |
| 8 | ; exists in the original program. |
| 9 | |
| 10 | ; (fadd (fma x, y, (fmul u, v), z) -> (fma x, y (fma u, v, z)) |
| 11 | |
| 12 | ; GCN-LABEL: {{^}}fast_add_fmuladd_fmul: |
| 13 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 14 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 15 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 16 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 17 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 18 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 19 | ; GCN-FLUSH: v_mac_f32_e32 [[Z]], [[U]], [[V]] |
| 20 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[Z]], [[X]], [[Y]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 21 | ; GCN-FLUSH-NEXT: buffer_store_dword [[Z]] |
| 22 | |
| 23 | ; GCN-FASTFMA: v_fma_f32 [[FMA0:v[0-9]+]], [[U]], [[V]], [[Z]] |
| 24 | ; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[FMA0]] |
| 25 | ; GCN-FASTFMA: buffer_store_dword [[FMA1]] |
| 26 | |
| 27 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 28 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 29 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 30 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 31 | define amdgpu_kernel void @fast_add_fmuladd_fmul() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 32 | %x = load volatile float, float addrspace(1)* undef |
| 33 | %y = load volatile float, float addrspace(1)* undef |
| 34 | %z = load volatile float, float addrspace(1)* undef |
| 35 | %u = load volatile float, float addrspace(1)* undef |
| 36 | %v = load volatile float, float addrspace(1)* undef |
| 37 | %mul.u.v = fmul fast float %u, %v |
| 38 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 39 | %add = fadd fast float %fma, %z |
| 40 | store volatile float %add, float addrspace(1)* undef |
| 41 | ret void |
| 42 | } |
| 43 | |
| 44 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul: |
| 45 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 46 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 47 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 48 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 49 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 50 | |
| 51 | ; GCN-FLUSH: v_mad_f32 [[TMP:v[0-9]]], [[U]], [[V]], -[[Z]] |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 52 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[TMP]], [[X]], [[Y]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 53 | ; GCN-FLUSH-NEXT: buffer_store_dword [[Z]] |
| 54 | |
| 55 | ; GCN-FASTFMA: v_fma_f32 [[FMA0:v[0-9]+]], [[U]], [[V]], -[[Z]] |
| 56 | ; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[FMA0]] |
| 57 | ; GCN-FASTFMA: buffer_store_dword [[FMA1]] |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 58 | define amdgpu_kernel void @fast_sub_fmuladd_fmul() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 59 | %x = load volatile float, float addrspace(1)* undef |
| 60 | %y = load volatile float, float addrspace(1)* undef |
| 61 | %z = load volatile float, float addrspace(1)* undef |
| 62 | %u = load volatile float, float addrspace(1)* undef |
| 63 | %v = load volatile float, float addrspace(1)* undef |
| 64 | %mul.u.v = fmul fast float %u, %v |
| 65 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 66 | %add = fsub fast float %fma, %z |
| 67 | store volatile float %add, float addrspace(1)* undef |
| 68 | ret void |
| 69 | } |
| 70 | |
| 71 | ; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_mul: |
| 72 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 73 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 74 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 75 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 76 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 77 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 78 | ; GCN-FLUSH-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
| 79 | ; GCN-FLUSH-DAG: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 80 | ; GCN-FLUSH: v_add_f32_e32 v{{[0-9]+}}, [[U]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 81 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 82 | ; GCN-FASTFMA: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 83 | ; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[MUL]] |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 84 | ; GCN-FASTFMA: v_add_f32_e32 v{{[0-9]+}}, [[FMA1]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 85 | |
| 86 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 87 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 88 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 89 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 90 | define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_mul() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 91 | %x = load volatile float, float addrspace(1)* undef |
| 92 | %y = load volatile float, float addrspace(1)* undef |
| 93 | %z = load volatile float, float addrspace(1)* undef |
| 94 | %u = load volatile float, float addrspace(1)* undef |
| 95 | %v = load volatile float, float addrspace(1)* undef |
| 96 | %mul.u.v = fmul fast float %u, %v |
| 97 | store volatile float %mul.u.v, float addrspace(1)* undef |
| 98 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 99 | %add = fadd fast float %fma, %z |
| 100 | store volatile float %add, float addrspace(1)* undef |
| 101 | ret void |
| 102 | } |
| 103 | |
| 104 | ; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_mul_commute: |
| 105 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 106 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 107 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 108 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 109 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 110 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 111 | ; GCN-FLUSH-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
| 112 | ; GCN-FLUSH-DAG: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 113 | ; GCN-FLUSH: v_add_f32_e32 v{{[0-9]+}}, [[Z]], [[U]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 114 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 115 | ; GCN-FASTFMA: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 116 | ; GCN-FASTFMA: v_fma_f32 [[FMA1:v[0-9]+]], [[X]], [[Y]], [[MUL]] |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 117 | ; GCN-FASTFMA: v_add_f32_e32 v{{[0-9]+}}, [[Z]], [[FMA1]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 118 | |
| 119 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 120 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 121 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 122 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 123 | define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_mul_commute() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 124 | %x = load volatile float, float addrspace(1)* undef |
| 125 | %y = load volatile float, float addrspace(1)* undef |
| 126 | %z = load volatile float, float addrspace(1)* undef |
| 127 | %u = load volatile float, float addrspace(1)* undef |
| 128 | %v = load volatile float, float addrspace(1)* undef |
| 129 | %mul.u.v = fmul fast float %u, %v |
| 130 | store volatile float %mul.u.v, float addrspace(1)* undef |
| 131 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 132 | %add = fadd fast float %z, %fma |
| 133 | store volatile float %add, float addrspace(1)* undef |
| 134 | ret void |
| 135 | } |
| 136 | |
| 137 | ; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_fmuladd: |
| 138 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 139 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 140 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 141 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 142 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 143 | |
| 144 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 145 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 146 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 147 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 148 | define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_fmuladd() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 149 | %x = load volatile float, float addrspace(1)* undef |
| 150 | %y = load volatile float, float addrspace(1)* undef |
| 151 | %z = load volatile float, float addrspace(1)* undef |
| 152 | %u = load volatile float, float addrspace(1)* undef |
| 153 | %v = load volatile float, float addrspace(1)* undef |
| 154 | %mul.u.v = fmul fast float %u, %v |
| 155 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 156 | store volatile float %fma, float addrspace(1)* undef |
| 157 | %add = fadd fast float %fma, %z |
| 158 | store volatile float %add, float addrspace(1)* undef |
| 159 | ret void |
| 160 | } |
| 161 | |
| 162 | ; GCN-LABEL: {{^}}fast_add_fmuladd_fmul_multi_use_fmuladd_commute: |
| 163 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 164 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 165 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 166 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 167 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 168 | |
| 169 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 170 | ; GCN-SLOWFMA: v_mul_f32_e32 |
| 171 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 172 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 173 | define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_fmuladd_commute() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 174 | %x = load volatile float, float addrspace(1)* undef |
| 175 | %y = load volatile float, float addrspace(1)* undef |
| 176 | %z = load volatile float, float addrspace(1)* undef |
| 177 | %u = load volatile float, float addrspace(1)* undef |
| 178 | %v = load volatile float, float addrspace(1)* undef |
| 179 | %mul.u.v = fmul fast float %u, %v |
| 180 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 181 | store volatile float %fma, float addrspace(1)* undef |
| 182 | %add = fadd fast float %z, %fma |
| 183 | store volatile float %add, float addrspace(1)* undef |
| 184 | ret void |
| 185 | } |
| 186 | |
| 187 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_mul: |
| 188 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 189 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 190 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 191 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 192 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 193 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 194 | ; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 195 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 196 | ; GCN-FLUSH: v_mad_f32 [[MAD:v[0-9]+]], [[X]], [[Y]], [[MUL]] |
| 197 | ; GCN-FLUSH: v_sub_f32_e32 [[SUB:v[0-9]+]], [[MAD]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 198 | |
| 199 | ; GCN-FASTFMA: v_fma_f32 [[MAD:v[0-9]+]], [[X]], [[Y]], [[MUL]] |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 200 | ; GCN-FASTFMA: v_sub_f32_e32 [[SUB:v[0-9]+]], [[MAD]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 201 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 202 | ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 203 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 204 | ; GCN-SLOWFMA: v_sub_f32_e32 [[MAD:v[0-9]+]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 205 | |
| 206 | ; GCN: buffer_store_dword [[MUL]] |
| 207 | ; GCN: buffer_store_dword [[MAD]] |
Matt Arsenault | 3dbeefa | 2017-03-21 21:39:51 +0000 | [diff] [blame] | 208 | define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_mul() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 209 | %x = load volatile float, float addrspace(1)* undef |
| 210 | %y = load volatile float, float addrspace(1)* undef |
| 211 | %z = load volatile float, float addrspace(1)* undef |
| 212 | %u = load volatile float, float addrspace(1)* undef |
| 213 | %v = load volatile float, float addrspace(1)* undef |
| 214 | %mul.u.v = fmul fast float %u, %v |
| 215 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 216 | %add = fsub fast float %fma, %z |
| 217 | store volatile float %mul.u.v, float addrspace(1)* undef |
| 218 | store volatile float %add, float addrspace(1)* undef |
| 219 | ret void |
| 220 | } |
| 221 | |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 222 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs: |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 223 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 224 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 225 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 226 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 227 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 228 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 229 | ; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 230 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 231 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 232 | ; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[MUL]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 233 | ; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 234 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 235 | ; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 236 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 237 | |
| 238 | ; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[U]] |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 239 | ; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[FMA]], [[Z]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 240 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 241 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 242 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 243 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 244 | |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 245 | ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]] |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 246 | ; GCN-SLOWFMA: v_add_f32_e32 |
Matt Arsenault | 6c29c5a | 2017-07-10 19:53:57 +0000 | [diff] [blame] | 247 | ; GCN-SLOWFMA: v_sub_f32_e32 |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 248 | define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs() #0 { |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 249 | %x = load volatile float, float addrspace(1)* undef |
| 250 | %y = load volatile float, float addrspace(1)* undef |
| 251 | %z = load volatile float, float addrspace(1)* undef |
| 252 | %u = load volatile float, float addrspace(1)* undef |
| 253 | %v = load volatile float, float addrspace(1)* undef |
| 254 | %mul.u.v = fmul fast float %u, %v |
| 255 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 256 | %add = fsub fast float %fma, %z |
| 257 | store volatile float %fma, float addrspace(1)* undef |
| 258 | store volatile float %add, float addrspace(1)* undef |
| 259 | ret void |
| 260 | } |
| 261 | |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 262 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs: |
| 263 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 264 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 265 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 266 | ; GCN: buffer_load_dword [[U:v[0-9]+]] |
| 267 | ; GCN: buffer_load_dword [[V:v[0-9]+]] |
| 268 | |
| 269 | ; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[U]], [[V]] |
| 270 | |
| 271 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 272 | ; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[MUL]] |
| 273 | ; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 274 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 275 | ; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 276 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 277 | |
| 278 | ; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[U]] |
| 279 | ; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[FMA]] |
| 280 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 281 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 282 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 283 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 284 | |
| 285 | ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]] |
| 286 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 287 | ; GCN-SLOWFMA: v_sub_f32_e32 |
| 288 | define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs() #0 { |
| 289 | %x = load volatile float, float addrspace(1)* undef |
| 290 | %y = load volatile float, float addrspace(1)* undef |
| 291 | %z = load volatile float, float addrspace(1)* undef |
| 292 | %u = load volatile float, float addrspace(1)* undef |
| 293 | %v = load volatile float, float addrspace(1)* undef |
| 294 | %mul.u.v = fmul fast float %u, %v |
| 295 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 296 | %add = fsub fast float %z, %fma |
| 297 | store volatile float %fma, float addrspace(1)* undef |
| 298 | store volatile float %add, float addrspace(1)* undef |
| 299 | ret void |
| 300 | } |
| 301 | |
| 302 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs: |
| 303 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 304 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 305 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 306 | ; GCN: buffer_load_ushort [[U:v[0-9]+]] |
| 307 | ; GCN: buffer_load_ushort [[V:v[0-9]+]] |
| 308 | |
| 309 | ; GCN-DAG: v_cvt_f32_f16_e32 [[UFLOAT:v[0-9]+]], [[U]] |
| 310 | ; GCN-DAG: v_cvt_f32_f16_e32 [[VFLOAT:v[0-9]+]], [[V]] |
| 311 | ; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[UFLOAT]], [[VFLOAT]] |
| 312 | |
| 313 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 314 | ; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[MUL]], [[Z]] |
| 315 | ; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 316 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 317 | ; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 318 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 319 | |
| 320 | ; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[UFLOAT]] |
| 321 | ; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[FMA]], [[Z]] |
| 322 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 323 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 324 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 325 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 326 | |
| 327 | ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]] |
| 328 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 329 | ; GCN-SLOWFMA: v_sub_f32_e32 |
| 330 | define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs() #0 { |
| 331 | %x = load volatile float, float addrspace(1)* undef |
| 332 | %y = load volatile float, float addrspace(1)* undef |
| 333 | %z = load volatile float, float addrspace(1)* undef |
| 334 | %u = load volatile half, half addrspace(1)* undef |
| 335 | %v = load volatile half, half addrspace(1)* undef |
| 336 | %mul.u.v.half = fmul fast half %u, %v |
| 337 | %mul.u.v = fpext half %mul.u.v.half to float |
| 338 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 339 | %add = fsub fast float %fma, %z |
| 340 | store volatile float %fma, float addrspace(1)* undef |
| 341 | store volatile float %add, float addrspace(1)* undef |
| 342 | ret void |
| 343 | } |
| 344 | |
| 345 | ; GCN-LABEL: {{^}}fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs: |
| 346 | ; GCN: buffer_load_dword [[X:v[0-9]+]] |
| 347 | ; GCN: buffer_load_dword [[Y:v[0-9]+]] |
| 348 | ; GCN: buffer_load_dword [[Z:v[0-9]+]] |
| 349 | ; GCN: buffer_load_ushort [[U:v[0-9]+]] |
| 350 | ; GCN: buffer_load_ushort [[V:v[0-9]+]] |
| 351 | |
| 352 | ; GCN-DAG: v_cvt_f32_f16_e32 [[UFLOAT:v[0-9]+]], [[U]] |
| 353 | ; GCN-DAG: v_cvt_f32_f16_e32 [[VFLOAT:v[0-9]+]], [[V]] |
| 354 | ; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[UFLOAT]], [[VFLOAT]] |
| 355 | |
| 356 | ; GCN-FLUSH-NEXT: v_mac_f32_e32 [[MUL]], [[X]], [[Y]] |
| 357 | ; GCN-FLUSH-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[MUL]] |
| 358 | ; GCN-FLUSH-NEXT: buffer_store_dword [[MUL]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 359 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 360 | ; GCN-FLUSH-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 361 | ; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 362 | |
| 363 | ; GCN-FASTFMA-NEXT: v_fma_f32 [[FMA:v[0-9]+]], [[X]], [[Y]], [[UFLOAT]] |
| 364 | ; GCN-FASTFMA-NEXT: v_sub_f32_e32 [[SUB:v[0-9]+]], [[Z]], [[FMA]] |
| 365 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[FMA]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 366 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 367 | ; GCN-FASTFMA-NEXT: buffer_store_dword [[SUB]] |
Tony | 2f499b9 | 2020-12-19 02:05:11 +0000 | [diff] [blame] | 368 | ; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) |
Carl Ritson | 2791667 | 2019-12-20 12:51:50 +0900 | [diff] [blame] | 369 | |
| 370 | ; GCN-SLOWFMA-DAG: v_mul_f32_e32 v{{[0-9]+}}, [[X]], [[Y]] |
| 371 | ; GCN-SLOWFMA: v_add_f32_e32 |
| 372 | ; GCN-SLOWFMA: v_sub_f32_e32 |
| 373 | define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs() #0 { |
| 374 | %x = load volatile float, float addrspace(1)* undef |
| 375 | %y = load volatile float, float addrspace(1)* undef |
| 376 | %z = load volatile float, float addrspace(1)* undef |
| 377 | %u = load volatile half, half addrspace(1)* undef |
| 378 | %v = load volatile half, half addrspace(1)* undef |
| 379 | %mul.u.v.half = fmul fast half %u, %v |
| 380 | %mul.u.v = fpext half %mul.u.v.half to float |
| 381 | %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v) |
| 382 | %add = fsub fast float %z, %fma |
| 383 | store volatile float %fma, float addrspace(1)* undef |
| 384 | store volatile float %add, float addrspace(1)* undef |
| 385 | ret void |
| 386 | } |
| 387 | |
Matt Arsenault | e482403 | 2017-01-11 02:02:12 +0000 | [diff] [blame] | 388 | declare float @llvm.fma.f32(float, float, float) #1 |
| 389 | declare float @llvm.fmuladd.f32(float, float, float) #1 |
| 390 | |
| 391 | attributes #0 = { nounwind } |
| 392 | attributes #1 = { nounwind readnone } |