test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll

; RUN: opt -S -slp-vectorizer -dce -instcombine < %s | FileCheck %s --check-prefix=GENERIC
; RUN: opt -S -mcpu=kryo -slp-vectorizer -dce -instcombine < %s | FileCheck %s --check-prefix=KRYO

target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-gnu"

; These tests check that we vectorize the index calculations in the
; gather-reduce pattern shown below. We check cases having i32 and i64
; subtraction.
;
; int gather_reduce_8x16(short *a, short *b, short *g, int n) {
;   int sum = 0;
;   for (int i = 0; i < n ; ++i) {
;     sum += g[*a++ - b[0]]; sum += g[*a++ - b[1]];
;     sum += g[*a++ - b[2]]; sum += g[*a++ - b[3]];
;     sum += g[*a++ - b[4]]; sum += g[*a++ - b[5]];
;     sum += g[*a++ - b[6]]; sum += g[*a++ - b[7]];
;   }
;   return sum;
; }

; GENERIC-LABEL: @gather_reduce_8x16_i32
;
; GENERIC: [[L:%[a-zA-Z0-9.]+]] = load <8 x i16>
; GENERIC: zext <8 x i16> [[L]] to <8 x i32>
; GENERIC: [[S:%[a-zA-Z0-9.]+]] = sub nsw <8 x i32>
; GENERIC: [[X:%[a-zA-Z0-9.]+]] = extractelement <8 x i32> [[S]]
; GENERIC: sext i32 [[X]] to i64
;
define i32 @gather_reduce_8x16_i32(i16* nocapture readonly %a, i16* nocapture readonly %b, i16* nocapture readonly %g, i32 %n) {
entry:
  %cmp.99 = icmp sgt i32 %n, 0
  br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup

for.body.preheader:
  br label %for.body

for.cond.cleanup.loopexit:
  br label %for.cond.cleanup

for.cond.cleanup:
  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]
  ret i32 %sum.0.lcssa

for.body:
  %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
  %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]
  %a.addr.0101 = phi i16* [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]
  %incdec.ptr = getelementptr inbounds i16, i16* %a.addr.0101, i64 1
  %0 = load i16, i16* %a.addr.0101, align 2
  %conv = zext i16 %0 to i32
  %incdec.ptr1 = getelementptr inbounds i16, i16* %b, i64 1
  %1 = load i16, i16* %b, align 2
  %conv2 = zext i16 %1 to i32
  %sub = sub nsw i32 %conv, %conv2
  %arrayidx = getelementptr inbounds i16, i16* %g, i32 %sub
  %2 = load i16, i16* %arrayidx, align 2
  %conv3 = zext i16 %2 to i32
  %add = add nsw i32 %conv3, %sum.0102
  %incdec.ptr4 = getelementptr inbounds i16, i16* %a.addr.0101, i64 2
  %3 = load i16, i16* %incdec.ptr, align 2
  %conv5 = zext i16 %3 to i32
  %incdec.ptr6 = getelementptr inbounds i16, i16* %b, i64 2
  %4 = load i16, i16* %incdec.ptr1, align 2
  %conv7 = zext i16 %4 to i32
  %sub8 = sub nsw i32 %conv5, %conv7
  %arrayidx10 = getelementptr inbounds i16, i16* %g, i32 %sub8
  %5 = load i16, i16* %arrayidx10, align 2
  %conv11 = zext i16 %5 to i32
  %add12 = add nsw i32 %add, %conv11
  %incdec.ptr13 = getelementptr inbounds i16, i16* %a.addr.0101, i64 3
  %6 = load i16, i16* %incdec.ptr4, align 2
  %conv14 = zext i16 %6 to i32
  %incdec.ptr15 = getelementptr inbounds i16, i16* %b, i64 3
  %7 = load i16, i16* %incdec.ptr6, align 2
  %conv16 = zext i16 %7 to i32
  %sub17 = sub nsw i32 %conv14, %conv16
  %arrayidx19 = getelementptr inbounds i16, i16* %g, i32 %sub17
  %8 = load i16, i16* %arrayidx19, align 2
  %conv20 = zext i16 %8 to i32
  %add21 = add nsw i32 %add12, %conv20
  %incdec.ptr22 = getelementptr inbounds i16, i16* %a.addr.0101, i64 4
  %9 = load i16, i16* %incdec.ptr13, align 2
  %conv23 = zext i16 %9 to i32
  %incdec.ptr24 = getelementptr inbounds i16, i16* %b, i64 4
  %10 = load i16, i16* %incdec.ptr15, align 2
  %conv25 = zext i16 %10 to i32
  %sub26 = sub nsw i32 %conv23, %conv25
  %arrayidx28 = getelementptr inbounds i16, i16* %g, i32 %sub26
  %11 = load i16, i16* %arrayidx28, align 2
  %conv29 = zext i16 %11 to i32
  %add30 = add nsw i32 %add21, %conv29
  %incdec.ptr31 = getelementptr inbounds i16, i16* %a.addr.0101, i64 5
  %12 = load i16, i16* %incdec.ptr22, align 2
  %conv32 = zext i16 %12 to i32
  %incdec.ptr33 = getelementptr inbounds i16, i16* %b, i64 5
  %13 = load i16, i16* %incdec.ptr24, align 2
  %conv34 = zext i16 %13 to i32
  %sub35 = sub nsw i32 %conv32, %conv34
  %arrayidx37 = getelementptr inbounds i16, i16* %g, i32 %sub35
  %14 = load i16, i16* %arrayidx37, align 2
  %conv38 = zext i16 %14 to i32
  %add39 = add nsw i32 %add30, %conv38
  %incdec.ptr40 = getelementptr inbounds i16, i16* %a.addr.0101, i64 6
  %15 = load i16, i16* %incdec.ptr31, align 2
  %conv41 = zext i16 %15 to i32
  %incdec.ptr42 = getelementptr inbounds i16, i16* %b, i64 6
  %16 = load i16, i16* %incdec.ptr33, align 2
  %conv43 = zext i16 %16 to i32
  %sub44 = sub nsw i32 %conv41, %conv43
  %arrayidx46 = getelementptr inbounds i16, i16* %g, i32 %sub44
  %17 = load i16, i16* %arrayidx46, align 2
  %conv47 = zext i16 %17 to i32
  %add48 = add nsw i32 %add39, %conv47
  %incdec.ptr49 = getelementptr inbounds i16, i16* %a.addr.0101, i64 7
  %18 = load i16, i16* %incdec.ptr40, align 2
  %conv50 = zext i16 %18 to i32
  %incdec.ptr51 = getelementptr inbounds i16, i16* %b, i64 7
  %19 = load i16, i16* %incdec.ptr42, align 2
  %conv52 = zext i16 %19 to i32
  %sub53 = sub nsw i32 %conv50, %conv52
  %arrayidx55 = getelementptr inbounds i16, i16* %g, i32 %sub53
  %20 = load i16, i16* %arrayidx55, align 2
  %conv56 = zext i16 %20 to i32
  %add57 = add nsw i32 %add48, %conv56
  %incdec.ptr58 = getelementptr inbounds i16, i16* %a.addr.0101, i64 8
  %21 = load i16, i16* %incdec.ptr49, align 2
  %conv59 = zext i16 %21 to i32
  %22 = load i16, i16* %incdec.ptr51, align 2
  %conv61 = zext i16 %22 to i32
  %sub62 = sub nsw i32 %conv59, %conv61
  %arrayidx64 = getelementptr inbounds i16, i16* %g, i32 %sub62
  %23 = load i16, i16* %arrayidx64, align 2
  %conv65 = zext i16 %23 to i32
  %add66 = add nsw i32 %add57, %conv65
  %inc = add nuw nsw i32 %i.0103, 1
  %exitcond = icmp eq i32 %inc, %n
  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
}

; KRYO-LABEL: @gather_reduce_8x16_i64
;
; KRYO: [[L:%[a-zA-Z0-9.]+]] = load <8 x i16>
; KRYO: zext <8 x i16> [[L]] to <8 x i32>
; KRYO: [[S:%[a-zA-Z0-9.]+]] = sub nsw <8 x i32>
; KRYO: [[X:%[a-zA-Z0-9.]+]] = extractelement <8 x i32> [[S]]
; KRYO: sext i32 [[X]] to i64
;
define i32 @gather_reduce_8x16_i64(i16* nocapture readonly %a, i16* nocapture readonly %b, i16* nocapture readonly %g, i32 %n) {
entry:
  %cmp.99 = icmp sgt i32 %n, 0
  br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup

for.body.preheader:
  br label %for.body

for.cond.cleanup.loopexit:
  br label %for.cond.cleanup

for.cond.cleanup:
  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]
  ret i32 %sum.0.lcssa

for.body:
  %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]
  %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]
  %a.addr.0101 = phi i16* [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]
  %incdec.ptr = getelementptr inbounds i16, i16* %a.addr.0101, i64 1
  %0 = load i16, i16* %a.addr.0101, align 2
  %conv = zext i16 %0 to i64
  %incdec.ptr1 = getelementptr inbounds i16, i16* %b, i64 1
  %1 = load i16, i16* %b, align 2
  %conv2 = zext i16 %1 to i64
  %sub = sub nsw i64 %conv, %conv2
  %arrayidx = getelementptr inbounds i16, i16* %g, i64 %sub
  %2 = load i16, i16* %arrayidx, align 2
  %conv3 = zext i16 %2 to i32
  %add = add nsw i32 %conv3, %sum.0102
  %incdec.ptr4 = getelementptr inbounds i16, i16* %a.addr.0101, i64 2
  %3 = load i16, i16* %incdec.ptr, align 2
  %conv5 = zext i16 %3 to i64
  %incdec.ptr6 = getelementptr inbounds i16, i16* %b, i64 2
  %4 = load i16, i16* %incdec.ptr1, align 2
  %conv7 = zext i16 %4 to i64
  %sub8 = sub nsw i64 %conv5, %conv7
  %arrayidx10 = getelementptr inbounds i16, i16* %g, i64 %sub8
  %5 = load i16, i16* %arrayidx10, align 2
  %conv11 = zext i16 %5 to i32
  %add12 = add nsw i32 %add, %conv11
  %incdec.ptr13 = getelementptr inbounds i16, i16* %a.addr.0101, i64 3
  %6 = load i16, i16* %incdec.ptr4, align 2
  %conv14 = zext i16 %6 to i64
  %incdec.ptr15 = getelementptr inbounds i16, i16* %b, i64 3
  %7 = load i16, i16* %incdec.ptr6, align 2
  %conv16 = zext i16 %7 to i64
  %sub17 = sub nsw i64 %conv14, %conv16
  %arrayidx19 = getelementptr inbounds i16, i16* %g, i64 %sub17
  %8 = load i16, i16* %arrayidx19, align 2
  %conv20 = zext i16 %8 to i32
  %add21 = add nsw i32 %add12, %conv20
  %incdec.ptr22 = getelementptr inbounds i16, i16* %a.addr.0101, i64 4
  %9 = load i16, i16* %incdec.ptr13, align 2
  %conv23 = zext i16 %9 to i64
  %incdec.ptr24 = getelementptr inbounds i16, i16* %b, i64 4
  %10 = load i16, i16* %incdec.ptr15, align 2
  %conv25 = zext i16 %10 to i64
  %sub26 = sub nsw i64 %conv23, %conv25
  %arrayidx28 = getelementptr inbounds i16, i16* %g, i64 %sub26
  %11 = load i16, i16* %arrayidx28, align 2
  %conv29 = zext i16 %11 to i32
  %add30 = add nsw i32 %add21, %conv29
  %incdec.ptr31 = getelementptr inbounds i16, i16* %a.addr.0101, i64 5
  %12 = load i16, i16* %incdec.ptr22, align 2
  %conv32 = zext i16 %12 to i64
  %incdec.ptr33 = getelementptr inbounds i16, i16* %b, i64 5
  %13 = load i16, i16* %incdec.ptr24, align 2
  %conv34 = zext i16 %13 to i64
  %sub35 = sub nsw i64 %conv32, %conv34
  %arrayidx37 = getelementptr inbounds i16, i16* %g, i64 %sub35
  %14 = load i16, i16* %arrayidx37, align 2
  %conv38 = zext i16 %14 to i32
  %add39 = add nsw i32 %add30, %conv38
  %incdec.ptr40 = getelementptr inbounds i16, i16* %a.addr.0101, i64 6
  %15 = load i16, i16* %incdec.ptr31, align 2
  %conv41 = zext i16 %15 to i64
  %incdec.ptr42 = getelementptr inbounds i16, i16* %b, i64 6
  %16 = load i16, i16* %incdec.ptr33, align 2
  %conv43 = zext i16 %16 to i64
  %sub44 = sub nsw i64 %conv41, %conv43
  %arrayidx46 = getelementptr inbounds i16, i16* %g, i64 %sub44
  %17 = load i16, i16* %arrayidx46, align 2
  %conv47 = zext i16 %17 to i32
  %add48 = add nsw i32 %add39, %conv47
  %incdec.ptr49 = getelementptr inbounds i16, i16* %a.addr.0101, i64 7
  %18 = load i16, i16* %incdec.ptr40, align 2
  %conv50 = zext i16 %18 to i64
  %incdec.ptr51 = getelementptr inbounds i16, i16* %b, i64 7
  %19 = load i16, i16* %incdec.ptr42, align 2
  %conv52 = zext i16 %19 to i64
  %sub53 = sub nsw i64 %conv50, %conv52
  %arrayidx55 = getelementptr inbounds i16, i16* %g, i64 %sub53
  %20 = load i16, i16* %arrayidx55, align 2
  %conv56 = zext i16 %20 to i32
  %add57 = add nsw i32 %add48, %conv56
  %incdec.ptr58 = getelementptr inbounds i16, i16* %a.addr.0101, i64 8
  %21 = load i16, i16* %incdec.ptr49, align 2
  %conv59 = zext i16 %21 to i64
  %22 = load i16, i16* %incdec.ptr51, align 2
  %conv61 = zext i16 %22 to i64
  %sub62 = sub nsw i64 %conv59, %conv61
  %arrayidx64 = getelementptr inbounds i16, i16* %g, i64 %sub62
  %23 = load i16, i16* %arrayidx64, align 2
  %conv65 = zext i16 %23 to i32
  %add66 = add nsw i32 %add57, %conv65
  %inc = add nuw nsw i32 %i.0103, 1
  %exitcond = icmp eq i32 %inc, %n
  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
}