llvm/test/Transforms/LoopUnroll/X86/partial-reductions.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -p loop-unroll -mcpu=x86-64-v4 -S %s | FileCheck %s
target triple = "x86_64-unknown-linux-gnu"

;; Check that we unroll this loop completely with partial reduction intrinsics
;; in place of plain adds.
define i32 @unroll_partial_rdx(ptr nofree noundef readonly captures(none) %0, ptr nofree noundef readonly captures(none) %1) {
; CHECK-LABEL: define i32 @unroll_partial_rdx(
; CHECK-SAME: ptr nofree noundef readonly captures(none) [[TMP0:%.*]], ptr nofree noundef readonly captures(none) [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[LOOP:.*]]
; CHECK:       [[LOOP]]:
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 32
; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 96
; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
; CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x i32>, ptr [[TMP3]], align 4
; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, ptr [[TMP4]], align 4
; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 64
; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 96
; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x i32>, ptr [[TMP9]], align 4
; CHECK-NEXT:    [[TMP14:%.*]] = load <8 x i32>, ptr [[TMP10]], align 4
; CHECK-NEXT:    [[TMP15:%.*]] = load <8 x i32>, ptr [[TMP11]], align 4
; CHECK-NEXT:    [[TMP16:%.*]] = mul <8 x i32> [[TMP12]], [[TMP5]]
; CHECK-NEXT:    [[TMP17:%.*]] = mul <8 x i32> [[TMP13]], [[TMP6]]
; CHECK-NEXT:    [[TMP18:%.*]] = mul <8 x i32> [[TMP14]], [[TMP7]]
; CHECK-NEXT:    [[TMP19:%.*]] = mul <8 x i32> [[TMP15]], [[TMP8]]
; CHECK-NEXT:    [[TMP68:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> [[TMP16]])
; CHECK-NEXT:    [[TMP69:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> [[TMP17]])
; CHECK-NEXT:    [[TMP70:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> [[TMP18]])
; CHECK-NEXT:    [[TMP71:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> zeroinitializer, <8 x i32> [[TMP19]])
; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP0]], i64 32
; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP72]], i64 32
; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP72]], i64 64
; CHECK-NEXT:    [[TMP75:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP72]], i64 96
; CHECK-NEXT:    [[TMP76:%.*]] = load <8 x i32>, ptr [[TMP72]], align 4
; CHECK-NEXT:    [[TMP77:%.*]] = load <8 x i32>, ptr [[TMP73]], align 4
; CHECK-NEXT:    [[TMP78:%.*]] = load <8 x i32>, ptr [[TMP74]], align 4
; CHECK-NEXT:    [[TMP79:%.*]] = load <8 x i32>, ptr [[TMP75]], align 4
; CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 32
; CHECK-NEXT:    [[TMP81:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP80]], i64 32
; CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP80]], i64 64
; CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP80]], i64 96
; CHECK-NEXT:    [[TMP84:%.*]] = load <8 x i32>, ptr [[TMP80]], align 4
; CHECK-NEXT:    [[TMP85:%.*]] = load <8 x i32>, ptr [[TMP81]], align 4
; CHECK-NEXT:    [[TMP86:%.*]] = load <8 x i32>, ptr [[TMP82]], align 4
; CHECK-NEXT:    [[TMP87:%.*]] = load <8 x i32>, ptr [[TMP83]], align 4
; CHECK-NEXT:    [[TMP88:%.*]] = mul <8 x i32> [[TMP84]], [[TMP76]]
; CHECK-NEXT:    [[TMP89:%.*]] = mul <8 x i32> [[TMP85]], [[TMP77]]
; CHECK-NEXT:    [[TMP90:%.*]] = mul <8 x i32> [[TMP86]], [[TMP78]]
; CHECK-NEXT:    [[TMP91:%.*]] = mul <8 x i32> [[TMP87]], [[TMP79]]
; CHECK-NEXT:    [[TMP92:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP68]], <8 x i32> [[TMP88]])
; CHECK-NEXT:    [[TMP93:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP69]], <8 x i32> [[TMP89]])
; CHECK-NEXT:    [[TMP94:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP70]], <8 x i32> [[TMP90]])
; CHECK-NEXT:    [[TMP95:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP71]], <8 x i32> [[TMP91]])
; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP0]], i64 64
; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP48]], i64 32
; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP48]], i64 64
; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP48]], i64 96
; CHECK-NEXT:    [[TMP52:%.*]] = load <8 x i32>, ptr [[TMP48]], align 4
; CHECK-NEXT:    [[TMP53:%.*]] = load <8 x i32>, ptr [[TMP49]], align 4
; CHECK-NEXT:    [[TMP54:%.*]] = load <8 x i32>, ptr [[TMP50]], align 4
; CHECK-NEXT:    [[TMP55:%.*]] = load <8 x i32>, ptr [[TMP51]], align 4
; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 64
; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 32
; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 64
; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 96
; CHECK-NEXT:    [[TMP60:%.*]] = load <8 x i32>, ptr [[TMP56]], align 4
; CHECK-NEXT:    [[TMP61:%.*]] = load <8 x i32>, ptr [[TMP57]], align 4
; CHECK-NEXT:    [[TMP62:%.*]] = load <8 x i32>, ptr [[TMP58]], align 4
; CHECK-NEXT:    [[TMP63:%.*]] = load <8 x i32>, ptr [[TMP59]], align 4
; CHECK-NEXT:    [[TMP64:%.*]] = mul <8 x i32> [[TMP60]], [[TMP52]]
; CHECK-NEXT:    [[TMP65:%.*]] = mul <8 x i32> [[TMP61]], [[TMP53]]
; CHECK-NEXT:    [[TMP66:%.*]] = mul <8 x i32> [[TMP62]], [[TMP54]]
; CHECK-NEXT:    [[TMP67:%.*]] = mul <8 x i32> [[TMP63]], [[TMP55]]
; CHECK-NEXT:    [[TMP96:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP92]], <8 x i32> [[TMP64]])
; CHECK-NEXT:    [[TMP97:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP93]], <8 x i32> [[TMP65]])
; CHECK-NEXT:    [[TMP100:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP94]], <8 x i32> [[TMP66]])
; CHECK-NEXT:    [[TMP101:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP95]], <8 x i32> [[TMP67]])
; CHECK-NEXT:    [[TMP102:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP0]], i64 96
; CHECK-NEXT:    [[TMP103:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP102]], i64 32
; CHECK-NEXT:    [[TMP104:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP102]], i64 64
; CHECK-NEXT:    [[TMP105:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP102]], i64 96
; CHECK-NEXT:    [[TMP106:%.*]] = load <8 x i32>, ptr [[TMP102]], align 4
; CHECK-NEXT:    [[TMP107:%.*]] = load <8 x i32>, ptr [[TMP103]], align 4
; CHECK-NEXT:    [[TMP108:%.*]] = load <8 x i32>, ptr [[TMP104]], align 4
; CHECK-NEXT:    [[TMP109:%.*]] = load <8 x i32>, ptr [[TMP105]], align 4
; CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 96
; CHECK-NEXT:    [[TMP111:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP110]], i64 32
; CHECK-NEXT:    [[TMP112:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP110]], i64 64
; CHECK-NEXT:    [[TMP113:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP110]], i64 96
; CHECK-NEXT:    [[TMP114:%.*]] = load <8 x i32>, ptr [[TMP110]], align 4
; CHECK-NEXT:    [[TMP115:%.*]] = load <8 x i32>, ptr [[TMP111]], align 4
; CHECK-NEXT:    [[TMP116:%.*]] = load <8 x i32>, ptr [[TMP112]], align 4
; CHECK-NEXT:    [[TMP117:%.*]] = load <8 x i32>, ptr [[TMP113]], align 4
; CHECK-NEXT:    [[TMP118:%.*]] = mul <8 x i32> [[TMP114]], [[TMP106]]
; CHECK-NEXT:    [[TMP119:%.*]] = mul <8 x i32> [[TMP115]], [[TMP107]]
; CHECK-NEXT:    [[TMP120:%.*]] = mul <8 x i32> [[TMP116]], [[TMP108]]
; CHECK-NEXT:    [[TMP121:%.*]] = mul <8 x i32> [[TMP117]], [[TMP109]]
; CHECK-NEXT:    [[DOTLCSSA3:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP96]], <8 x i32> [[TMP118]])
; CHECK-NEXT:    [[DOTLCSSA2:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP97]], <8 x i32> [[TMP119]])
; CHECK-NEXT:    [[DOTLCSSA1:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP100]], <8 x i32> [[TMP120]])
; CHECK-NEXT:    [[DOTLCSSA:%.*]] = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> [[TMP101]], <8 x i32> [[TMP121]])
; CHECK-NEXT:    [[TMP33:%.*]] = add <8 x i32> [[DOTLCSSA2]], [[DOTLCSSA3]]
; CHECK-NEXT:    [[TMP34:%.*]] = add <8 x i32> [[DOTLCSSA1]], [[TMP33]]
; CHECK-NEXT:    [[TMP98:%.*]] = add <8 x i32> [[DOTLCSSA]], [[TMP34]]
; CHECK-NEXT:    [[TMP99:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP98]])
; CHECK-NEXT:    ret i32 [[TMP99]]
;
entry:
  br label %loop

loop:
  %4 = phi i64 [ 0, %entry ], [ %33, %loop ]
  %5 = phi <8 x i32> [ zeroinitializer, %entry ], [ %29, %loop ]
  %6 = phi <8 x i32> [ zeroinitializer, %entry ], [ %30, %loop ]
  %7 = phi <8 x i32> [ zeroinitializer, %entry ], [ %31, %loop ]
  %8 = phi <8 x i32> [ zeroinitializer, %entry ], [ %32, %loop ]
  %9 = getelementptr inbounds nuw [4 x i8], ptr %0, i64 %4
  %10 = getelementptr inbounds nuw i8, ptr %9, i64 32
  %11 = getelementptr inbounds nuw i8, ptr %9, i64 64
  %12 = getelementptr inbounds nuw i8, ptr %9, i64 96
  %13 = load <8 x i32>, ptr %9, align 4
  %14 = load <8 x i32>, ptr %10, align 4
  %15 = load <8 x i32>, ptr %11, align 4
  %16 = load <8 x i32>, ptr %12, align 4
  %17 = getelementptr inbounds nuw [4 x i8], ptr %1, i64 %4
  %18 = getelementptr inbounds nuw i8, ptr %17, i64 32
  %19 = getelementptr inbounds nuw i8, ptr %17, i64 64
  %20 = getelementptr inbounds nuw i8, ptr %17, i64 96
  %21 = load <8 x i32>, ptr %17, align 4
  %22 = load <8 x i32>, ptr %18, align 4
  %23 = load <8 x i32>, ptr %19, align 4
  %24 = load <8 x i32>, ptr %20, align 4
  %25 = mul <8 x i32> %21, %13
  %26 = mul <8 x i32> %22, %14
  %27 = mul <8 x i32> %23, %15
  %28 = mul <8 x i32> %24, %16
  %29 = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> %5, <8 x i32> %25)
  %30 = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> %6, <8 x i32> %26)
  %31 = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> %7, <8 x i32> %27)
  %32 = call <8 x i32> @llvm.vector.partial.reduce.add.v8i32.v8i32(<8 x i32> %8, <8 x i32> %28)
  %33 = add nuw i64 %4, 32
  %34 = icmp eq i64 %33, 128
  br i1 %34, label %exit, label %loop

exit:
  %36 = add <8 x i32> %30, %29
  %37 = add <8 x i32> %31, %36
  %38 = add <8 x i32> %32, %37
  %39 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %38)
  ret i32 %39
}