test/Transforms/SLPVectorizer/AMDGPU/latency-bound-data-movement.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgpu9.42-amd-amdhsa -passes=slp-vectorizer < %s | FileCheck %s

; Two chained complex multiplies, (x * w) * v: a chain of four dependent FP
; operations in a short block outside loops. For throughput, the moves that
; form the pairs outweigh the packed arithmetic.
define void @chained_complex_muls(ptr addrspace(1) %out, float %xr, float %xi, float %wr, float %wi, float %vr, float %vi) {
; CHECK-LABEL: define void @chained_complex_muls(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[XR:%.*]], float [[XI:%.*]], float [[WR:%.*]], float [[WI:%.*]], float [[VR:%.*]], float [[VI:%.*]]) {
; CHECK-NEXT:    [[XR_WR:%.*]] = fmul float [[XR]], [[WR]]
; CHECK-NEXT:    [[XI_WI:%.*]] = fmul float [[XI]], [[WI]]
; CHECK-NEXT:    [[AR:%.*]] = fsub float [[XR_WR]], [[XI_WI]]
; CHECK-NEXT:    [[XI_WR:%.*]] = fmul float [[XI]], [[WR]]
; CHECK-NEXT:    [[XR_WI:%.*]] = fmul float [[XR]], [[WI]]
; CHECK-NEXT:    [[AI:%.*]] = fadd float [[XI_WR]], [[XR_WI]]
; CHECK-NEXT:    [[AR_VR:%.*]] = fmul float [[AR]], [[VR]]
; CHECK-NEXT:    [[AI_VI:%.*]] = fmul float [[AI]], [[VI]]
; CHECK-NEXT:    [[RE:%.*]] = fsub float [[AR_VR]], [[AI_VI]]
; CHECK-NEXT:    [[AI_VR:%.*]] = fmul float [[AI]], [[VR]]
; CHECK-NEXT:    [[AR_VI:%.*]] = fmul float [[AR]], [[VI]]
; CHECK-NEXT:    [[IM:%.*]] = fadd float [[AI_VR]], [[AR_VI]]
; CHECK-NEXT:    store float [[RE]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT:    [[OUT_IM:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; CHECK-NEXT:    store float [[IM]], ptr addrspace(1) [[OUT_IM]], align 4
; CHECK-NEXT:    ret void
;
  %xr.wr = fmul float %xr, %wr
  %xi.wi = fmul float %xi, %wi
  %ar = fsub float %xr.wr, %xi.wi
  %xi.wr = fmul float %xi, %wr
  %xr.wi = fmul float %xr, %wi
  %ai = fadd float %xi.wr, %xr.wi
  %ar.vr = fmul float %ar, %vr
  %ai.vi = fmul float %ai, %vi
  %re = fsub float %ar.vr, %ai.vi
  %ai.vr = fmul float %ai, %vr
  %ar.vi = fmul float %ar, %vi
  %im = fadd float %ai.vr, %ar.vi
  store float %re, ptr addrspace(1) %out, align 8
  %out.im = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  store float %im, ptr addrspace(1) %out.im, align 4
  ret void
}

; The same chain inside a loop is priced for throughput.
define void @chained_complex_muls_in_loop(ptr addrspace(1) %out, float %xr, float %xi, float %wr, float %wi, float %vr, float %vi, i32 %n) {
; CHECK-LABEL: define void @chained_complex_muls_in_loop(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[XR:%.*]], float [[XI:%.*]], float [[WR:%.*]], float [[WI:%.*]], float [[VR:%.*]], float [[VI:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[XR_WR:%.*]] = fmul float [[XR]], [[WR]]
; CHECK-NEXT:    [[XI_WI:%.*]] = fmul float [[XI]], [[WI]]
; CHECK-NEXT:    [[AR:%.*]] = fsub float [[XR_WR]], [[XI_WI]]
; CHECK-NEXT:    [[XI_WR:%.*]] = fmul float [[XI]], [[WR]]
; CHECK-NEXT:    [[XR_WI:%.*]] = fmul float [[XR]], [[WI]]
; CHECK-NEXT:    [[AI:%.*]] = fadd float [[XI_WR]], [[XR_WI]]
; CHECK-NEXT:    [[AR_VR:%.*]] = fmul float [[AR]], [[VR]]
; CHECK-NEXT:    [[AI_VI:%.*]] = fmul float [[AI]], [[VI]]
; CHECK-NEXT:    [[RE:%.*]] = fsub float [[AR_VR]], [[AI_VI]]
; CHECK-NEXT:    [[AI_VR:%.*]] = fmul float [[AI]], [[VR]]
; CHECK-NEXT:    [[AR_VI:%.*]] = fmul float [[AR]], [[VI]]
; CHECK-NEXT:    [[IM:%.*]] = fadd float [[AI_VR]], [[AR_VI]]
; CHECK-NEXT:    [[IDX:%.*]] = zext i32 [[I]] to i64
; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds <2 x float>, ptr addrspace(1) [[OUT]], i64 [[IDX]]
; CHECK-NEXT:    store float [[RE]], ptr addrspace(1) [[P]], align 8
; CHECK-NEXT:    [[P_IM:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[P]], i64 1
; CHECK-NEXT:    store float [[IM]], ptr addrspace(1) [[P_IM]], align 4
; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %loop

loop:
  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
  %xr.wr = fmul float %xr, %wr
  %xi.wi = fmul float %xi, %wi
  %ar = fsub float %xr.wr, %xi.wi
  %xi.wr = fmul float %xi, %wr
  %xr.wi = fmul float %xr, %wi
  %ai = fadd float %xi.wr, %xr.wi
  %ar.vr = fmul float %ar, %vr
  %ai.vi = fmul float %ai, %vi
  %re = fsub float %ar.vr, %ai.vi
  %ai.vr = fmul float %ai, %vr
  %ar.vi = fmul float %ar, %vi
  %im = fadd float %ai.vr, %ar.vi
  %idx = zext i32 %i to i64
  %p = getelementptr inbounds <2 x float>, ptr addrspace(1) %out, i64 %idx
  store float %re, ptr addrspace(1) %p, align 8
  %p.im = getelementptr inbounds float, ptr addrspace(1) %p, i64 1
  store float %im, ptr addrspace(1) %p.im, align 4
  %i.next = add i32 %i, 1
  %done = icmp eq i32 %i.next, %n
  br i1 %done, label %exit, label %loop

exit:
  ret void
}

; A chain from a loop does not run on into the block after it: the complex
; multiply after the loop is a chain of two.
define void @complex_mul_after_loop(ptr addrspace(1) %out, float %xr, float %xi, float %wr, float %wi, float %vr, float %vi, i32 %n) {
; CHECK-LABEL: define void @complex_mul_after_loop(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[XR:%.*]], float [[XI:%.*]], float [[WR:%.*]], float [[WI:%.*]], float [[VR:%.*]], float [[VI:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[AR:%.*]] = phi float [ [[XR]], %[[ENTRY]] ], [ [[BR:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[AI:%.*]] = phi float [ [[XI]], %[[ENTRY]] ], [ [[BI:%.*]], %[[LOOP]] ]
; CHECK-NEXT:    [[AR_WR:%.*]] = fmul float [[AR]], [[WR]]
; CHECK-NEXT:    [[AI_WI:%.*]] = fmul float [[AI]], [[WI]]
; CHECK-NEXT:    [[BR]] = fsub float [[AR_WR]], [[AI_WI]]
; CHECK-NEXT:    [[AI_WR:%.*]] = fmul float [[AI]], [[WR]]
; CHECK-NEXT:    [[AR_WI:%.*]] = fmul float [[AR]], [[WI]]
; CHECK-NEXT:    [[BI]] = fadd float [[AI_WR]], [[AR_WI]]
; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    [[BR_VR:%.*]] = fmul float [[BR]], [[VR]]
; CHECK-NEXT:    [[BI_VI:%.*]] = fmul float [[BI]], [[VI]]
; CHECK-NEXT:    [[RE:%.*]] = fsub float [[BR_VR]], [[BI_VI]]
; CHECK-NEXT:    [[BI_VR:%.*]] = fmul float [[BI]], [[VR]]
; CHECK-NEXT:    [[BR_VI:%.*]] = fmul float [[BR]], [[VI]]
; CHECK-NEXT:    [[IM:%.*]] = fadd float [[BI_VR]], [[BR_VI]]
; CHECK-NEXT:    store float [[RE]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT:    [[OUT_IM:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; CHECK-NEXT:    store float [[IM]], ptr addrspace(1) [[OUT_IM]], align 4
; CHECK-NEXT:    ret void
;
entry:
  br label %loop

loop:
  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
  %ar = phi float [ %xr, %entry ], [ %br, %loop ]
  %ai = phi float [ %xi, %entry ], [ %bi, %loop ]
  %ar.wr = fmul float %ar, %wr
  %ai.wi = fmul float %ai, %wi
  %br = fsub float %ar.wr, %ai.wi
  %ai.wr = fmul float %ai, %wr
  %ar.wi = fmul float %ar, %wi
  %bi = fadd float %ai.wr, %ar.wi
  %i.next = add i32 %i, 1
  %done = icmp eq i32 %i.next, %n
  br i1 %done, label %exit, label %loop

exit:
  %br.vr = fmul float %br, %vr
  %bi.vi = fmul float %bi, %vi
  %re = fsub float %br.vr, %bi.vi
  %bi.vr = fmul float %bi, %vr
  %br.vi = fmul float %br, %vi
  %im = fadd float %bi.vr, %br.vi
  store float %re, ptr addrspace(1) %out, align 8
  %out.im = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  store float %im, ptr addrspace(1) %out.im, align 4
  ret void
}

; The second complex multiply is on one side of a branch. Phis in the block
; of the stores merge its result with the first; the chain of four ends there.
define void @chained_complex_muls_across_branch(ptr addrspace(1) %out, float %xr, float %xi, float %wr, float %wi, float %vr, float %vi, i1 %c) {
; CHECK-LABEL: define void @chained_complex_muls_across_branch(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[XR:%.*]], float [[XI:%.*]], float [[WR:%.*]], float [[WI:%.*]], float [[VR:%.*]], float [[VI:%.*]], i1 [[C:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    [[XR_WR:%.*]] = fmul float [[XR]], [[WR]]
; CHECK-NEXT:    [[XI_WI:%.*]] = fmul float [[XI]], [[WI]]
; CHECK-NEXT:    [[AR:%.*]] = fsub float [[XR_WR]], [[XI_WI]]
; CHECK-NEXT:    [[XI_WR:%.*]] = fmul float [[XI]], [[WR]]
; CHECK-NEXT:    [[XR_WI:%.*]] = fmul float [[XR]], [[WI]]
; CHECK-NEXT:    [[AI:%.*]] = fadd float [[XI_WR]], [[XR_WI]]
; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[JOIN:.*]]
; CHECK:       [[THEN]]:
; CHECK-NEXT:    [[AR_VR:%.*]] = fmul float [[AR]], [[VR]]
; CHECK-NEXT:    [[AI_VI:%.*]] = fmul float [[AI]], [[VI]]
; CHECK-NEXT:    [[BR:%.*]] = fsub float [[AR_VR]], [[AI_VI]]
; CHECK-NEXT:    [[AI_VR:%.*]] = fmul float [[AI]], [[VR]]
; CHECK-NEXT:    [[AR_VI:%.*]] = fmul float [[AR]], [[VI]]
; CHECK-NEXT:    [[BI:%.*]] = fadd float [[AI_VR]], [[AR_VI]]
; CHECK-NEXT:    br label %[[JOIN]]
; CHECK:       [[JOIN]]:
; CHECK-NEXT:    [[RE:%.*]] = phi float [ [[AR]], %[[ENTRY]] ], [ [[BR]], %[[THEN]] ]
; CHECK-NEXT:    [[IM:%.*]] = phi float [ [[AI]], %[[ENTRY]] ], [ [[BI]], %[[THEN]] ]
; CHECK-NEXT:    store float [[RE]], ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT:    [[OUT_IM:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[OUT]], i64 1
; CHECK-NEXT:    store float [[IM]], ptr addrspace(1) [[OUT_IM]], align 4
; CHECK-NEXT:    ret void
;
entry:
  %xr.wr = fmul float %xr, %wr
  %xi.wi = fmul float %xi, %wi
  %ar = fsub float %xr.wr, %xi.wi
  %xi.wr = fmul float %xi, %wr
  %xr.wi = fmul float %xr, %wi
  %ai = fadd float %xi.wr, %xr.wi
  br i1 %c, label %then, label %join

then:
  %ar.vr = fmul float %ar, %vr
  %ai.vi = fmul float %ai, %vi
  %br = fsub float %ar.vr, %ai.vi
  %ai.vr = fmul float %ai, %vr
  %ar.vi = fmul float %ar, %vi
  %bi = fadd float %ai.vr, %ar.vi
  br label %join

join:
  %re = phi float [ %ar, %entry ], [ %br, %then ]
  %im = phi float [ %ai, %entry ], [ %bi, %then ]
  store float %re, ptr addrspace(1) %out, align 8
  %out.im = getelementptr inbounds float, ptr addrspace(1) %out, i64 1
  store float %im, ptr addrspace(1) %out.im, align 4
  ret void
}