llvm/test/Transforms/SLPVectorizer/AArch64/bitpacked-aggregate.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-linux-gnu | FileCheck %s
; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=aarch64_be-linux-gnu | FileCheck %s --check-prefix=BE

; The packs of the aggregate are vectorized together as the single vector of the fields.
; Big-endian targets keep the per-pack vectorization.

define { i64, i64 } @pack_i16x8(ptr %x, ptr %y) {
; CHECK-LABEL: @pack_i16x8(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[X:%.*]], align 2
; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[Y:%.*]], align 2
; CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.smin.v8i16(<8 x i16> [[TMP0]], <8 x i16> [[TMP1]])
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <2 x i64>
; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
; CHECK-NEXT:    [[AGG_0:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP4]], 0
; CHECK-NEXT:    [[AGG_1:%.*]] = insertvalue { i64, i64 } [[AGG_0]], i64 [[TMP5]], 1
; CHECK-NEXT:    ret { i64, i64 } [[AGG_1]]
;
; BE-LABEL: @pack_i16x8(
; BE-NEXT:  entry:
; BE-NEXT:    [[X_4_P:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i64 4
; BE-NEXT:    [[Y_4_P:%.*]] = getelementptr inbounds i16, ptr [[Y:%.*]], i64 4
; BE-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr [[X]], align 2
; BE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[Y]], align 2
; BE-NEXT:    [[TMP2:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
; BE-NEXT:    [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
; BE-NEXT:    [[TMP4:%.*]] = shl nuw <4 x i64> [[TMP3]], <i64 0, i64 16, i64 32, i64 48>
; BE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP4]])
; BE-NEXT:    [[AGG_0:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP5]], 0
; BE-NEXT:    [[TMP6:%.*]] = load <4 x i16>, ptr [[X_4_P]], align 2
; BE-NEXT:    [[TMP7:%.*]] = load <4 x i16>, ptr [[Y_4_P]], align 2
; BE-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP6]], <4 x i16> [[TMP7]])
; BE-NEXT:    [[TMP9:%.*]] = zext <4 x i16> [[TMP8]] to <4 x i64>
; BE-NEXT:    [[TMP10:%.*]] = shl nuw <4 x i64> [[TMP9]], <i64 0, i64 16, i64 32, i64 48>
; BE-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP10]])
; BE-NEXT:    [[AGG_1:%.*]] = insertvalue { i64, i64 } [[AGG_0]], i64 [[TMP11]], 1
; BE-NEXT:    ret { i64, i64 } [[AGG_1]]
;
entry:
  %x.0 = load i16, ptr %x, align 2
  %y.0 = load i16, ptr %y, align 2
  %m.0 = call i16 @llvm.smin.i16(i16 %x.0, i16 %y.0)
  %x.1.p = getelementptr inbounds i16, ptr %x, i64 1
  %y.1.p = getelementptr inbounds i16, ptr %y, i64 1
  %x.1 = load i16, ptr %x.1.p, align 2
  %y.1 = load i16, ptr %y.1.p, align 2
  %m.1 = call i16 @llvm.smin.i16(i16 %x.1, i16 %y.1)
  %x.2.p = getelementptr inbounds i16, ptr %x, i64 2
  %y.2.p = getelementptr inbounds i16, ptr %y, i64 2
  %x.2 = load i16, ptr %x.2.p, align 2
  %y.2 = load i16, ptr %y.2.p, align 2
  %m.2 = call i16 @llvm.smin.i16(i16 %x.2, i16 %y.2)
  %x.3.p = getelementptr inbounds i16, ptr %x, i64 3
  %y.3.p = getelementptr inbounds i16, ptr %y, i64 3
  %x.3 = load i16, ptr %x.3.p, align 2
  %y.3 = load i16, ptr %y.3.p, align 2
  %m.3 = call i16 @llvm.smin.i16(i16 %x.3, i16 %y.3)
  %x.4.p = getelementptr inbounds i16, ptr %x, i64 4
  %y.4.p = getelementptr inbounds i16, ptr %y, i64 4
  %x.4 = load i16, ptr %x.4.p, align 2
  %y.4 = load i16, ptr %y.4.p, align 2
  %m.4 = call i16 @llvm.smin.i16(i16 %x.4, i16 %y.4)
  %x.5.p = getelementptr inbounds i16, ptr %x, i64 5
  %y.5.p = getelementptr inbounds i16, ptr %y, i64 5
  %x.5 = load i16, ptr %x.5.p, align 2
  %y.5 = load i16, ptr %y.5.p, align 2
  %m.5 = call i16 @llvm.smin.i16(i16 %x.5, i16 %y.5)
  %x.6.p = getelementptr inbounds i16, ptr %x, i64 6
  %y.6.p = getelementptr inbounds i16, ptr %y, i64 6
  %x.6 = load i16, ptr %x.6.p, align 2
  %y.6 = load i16, ptr %y.6.p, align 2
  %m.6 = call i16 @llvm.smin.i16(i16 %x.6, i16 %y.6)
  %x.7.p = getelementptr inbounds i16, ptr %x, i64 7
  %y.7.p = getelementptr inbounds i16, ptr %y, i64 7
  %x.7 = load i16, ptr %x.7.p, align 2
  %y.7 = load i16, ptr %y.7.p, align 2
  %m.7 = call i16 @llvm.smin.i16(i16 %x.7, i16 %y.7)
  %z.0 = zext i16 %m.0 to i64
  %z.1 = zext i16 %m.1 to i64
  %s.1 = shl nuw nsw i64 %z.1, 16
  %z.2 = zext i16 %m.2 to i64
  %s.2 = shl nuw nsw i64 %z.2, 32
  %z.3 = zext i16 %m.3 to i64
  %s.3 = shl nuw i64 %z.3, 48
  %o.2 = or disjoint i64 %s.3, %s.2
  %o.1 = or disjoint i64 %o.2, %s.1
  %w.0 = or disjoint i64 %o.1, %z.0
  %agg.0 = insertvalue { i64, i64 } poison, i64 %w.0, 0
  %z.4 = zext i16 %m.4 to i64
  %z.5 = zext i16 %m.5 to i64
  %s.5 = shl nuw nsw i64 %z.5, 16
  %z.6 = zext i16 %m.6 to i64
  %s.6 = shl nuw nsw i64 %z.6, 32
  %z.7 = zext i16 %m.7 to i64
  %s.7 = shl nuw i64 %z.7, 48
  %o.6 = or disjoint i64 %s.7, %s.6
  %o.5 = or disjoint i64 %o.6, %s.5
  %w.1 = or disjoint i64 %o.5, %z.4
  %agg.1 = insertvalue { i64, i64 } %agg.0, i64 %w.1, 1
  ret { i64, i64 } %agg.1
}

define { i64, i64 } @pack_i8x16(ptr %x, ptr %y) {
; CHECK-LABEL: @pack_i8x16(
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[X:%.*]], align 1
; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[Y:%.*]], align 1
; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> [[TMP0]], <16 x i8> [[TMP1]])
; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <16 x i8> [[TMP2]] to <2 x i64>
; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
; CHECK-NEXT:    [[AGG_0:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP4]], 0
; CHECK-NEXT:    [[AGG_1:%.*]] = insertvalue { i64, i64 } [[AGG_0]], i64 [[TMP5]], 1
; CHECK-NEXT:    ret { i64, i64 } [[AGG_1]]
;
; BE-LABEL: @pack_i8x16(
; BE-NEXT:  entry:
; BE-NEXT:    [[X_8_P:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 8
; BE-NEXT:    [[Y_8_P:%.*]] = getelementptr inbounds i8, ptr [[Y:%.*]], i64 8
; BE-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[X]], align 1
; BE-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[Y]], align 1
; BE-NEXT:    [[TMP2:%.*]] = call <8 x i8> @llvm.smin.v8i8(<8 x i8> [[TMP0]], <8 x i8> [[TMP1]])
; BE-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i64>
; BE-NEXT:    [[TMP4:%.*]] = shl nuw <8 x i64> [[TMP3]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
; BE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
; BE-NEXT:    [[AGG_0:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP5]], 0
; BE-NEXT:    [[TMP6:%.*]] = load <8 x i8>, ptr [[X_8_P]], align 1
; BE-NEXT:    [[TMP7:%.*]] = load <8 x i8>, ptr [[Y_8_P]], align 1
; BE-NEXT:    [[TMP8:%.*]] = call <8 x i8> @llvm.smin.v8i8(<8 x i8> [[TMP6]], <8 x i8> [[TMP7]])
; BE-NEXT:    [[TMP9:%.*]] = zext <8 x i8> [[TMP8]] to <8 x i64>
; BE-NEXT:    [[TMP10:%.*]] = shl nuw <8 x i64> [[TMP9]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
; BE-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP10]])
; BE-NEXT:    [[AGG_1:%.*]] = insertvalue { i64, i64 } [[AGG_0]], i64 [[TMP11]], 1
; BE-NEXT:    ret { i64, i64 } [[AGG_1]]
;
entry:
  %x.0 = load i8, ptr %x, align 1
  %y.0 = load i8, ptr %y, align 1
  %m.0 = call i8 @llvm.smin.i8(i8 %x.0, i8 %y.0)
  %x.1.p = getelementptr inbounds i8, ptr %x, i64 1
  %y.1.p = getelementptr inbounds i8, ptr %y, i64 1
  %x.1 = load i8, ptr %x.1.p, align 1
  %y.1 = load i8, ptr %y.1.p, align 1
  %m.1 = call i8 @llvm.smin.i8(i8 %x.1, i8 %y.1)
  %x.2.p = getelementptr inbounds i8, ptr %x, i64 2
  %y.2.p = getelementptr inbounds i8, ptr %y, i64 2
  %x.2 = load i8, ptr %x.2.p, align 1
  %y.2 = load i8, ptr %y.2.p, align 1
  %m.2 = call i8 @llvm.smin.i8(i8 %x.2, i8 %y.2)
  %x.3.p = getelementptr inbounds i8, ptr %x, i64 3
  %y.3.p = getelementptr inbounds i8, ptr %y, i64 3
  %x.3 = load i8, ptr %x.3.p, align 1
  %y.3 = load i8, ptr %y.3.p, align 1
  %m.3 = call i8 @llvm.smin.i8(i8 %x.3, i8 %y.3)
  %x.4.p = getelementptr inbounds i8, ptr %x, i64 4
  %y.4.p = getelementptr inbounds i8, ptr %y, i64 4
  %x.4 = load i8, ptr %x.4.p, align 1
  %y.4 = load i8, ptr %y.4.p, align 1
  %m.4 = call i8 @llvm.smin.i8(i8 %x.4, i8 %y.4)
  %x.5.p = getelementptr inbounds i8, ptr %x, i64 5
  %y.5.p = getelementptr inbounds i8, ptr %y, i64 5
  %x.5 = load i8, ptr %x.5.p, align 1
  %y.5 = load i8, ptr %y.5.p, align 1
  %m.5 = call i8 @llvm.smin.i8(i8 %x.5, i8 %y.5)
  %x.6.p = getelementptr inbounds i8, ptr %x, i64 6
  %y.6.p = getelementptr inbounds i8, ptr %y, i64 6
  %x.6 = load i8, ptr %x.6.p, align 1
  %y.6 = load i8, ptr %y.6.p, align 1
  %m.6 = call i8 @llvm.smin.i8(i8 %x.6, i8 %y.6)
  %x.7.p = getelementptr inbounds i8, ptr %x, i64 7
  %y.7.p = getelementptr inbounds i8, ptr %y, i64 7
  %x.7 = load i8, ptr %x.7.p, align 1
  %y.7 = load i8, ptr %y.7.p, align 1
  %m.7 = call i8 @llvm.smin.i8(i8 %x.7, i8 %y.7)
  %x.8.p = getelementptr inbounds i8, ptr %x, i64 8
  %y.8.p = getelementptr inbounds i8, ptr %y, i64 8
  %x.8 = load i8, ptr %x.8.p, align 1
  %y.8 = load i8, ptr %y.8.p, align 1
  %m.8 = call i8 @llvm.smin.i8(i8 %x.8, i8 %y.8)
  %x.9.p = getelementptr inbounds i8, ptr %x, i64 9
  %y.9.p = getelementptr inbounds i8, ptr %y, i64 9
  %x.9 = load i8, ptr %x.9.p, align 1
  %y.9 = load i8, ptr %y.9.p, align 1
  %m.9 = call i8 @llvm.smin.i8(i8 %x.9, i8 %y.9)
  %x.10.p = getelementptr inbounds i8, ptr %x, i64 10
  %y.10.p = getelementptr inbounds i8, ptr %y, i64 10
  %x.10 = load i8, ptr %x.10.p, align 1
  %y.10 = load i8, ptr %y.10.p, align 1
  %m.10 = call i8 @llvm.smin.i8(i8 %x.10, i8 %y.10)
  %x.11.p = getelementptr inbounds i8, ptr %x, i64 11
  %y.11.p = getelementptr inbounds i8, ptr %y, i64 11
  %x.11 = load i8, ptr %x.11.p, align 1
  %y.11 = load i8, ptr %y.11.p, align 1
  %m.11 = call i8 @llvm.smin.i8(i8 %x.11, i8 %y.11)
  %x.12.p = getelementptr inbounds i8, ptr %x, i64 12
  %y.12.p = getelementptr inbounds i8, ptr %y, i64 12
  %x.12 = load i8, ptr %x.12.p, align 1
  %y.12 = load i8, ptr %y.12.p, align 1
  %m.12 = call i8 @llvm.smin.i8(i8 %x.12, i8 %y.12)
  %x.13.p = getelementptr inbounds i8, ptr %x, i64 13
  %y.13.p = getelementptr inbounds i8, ptr %y, i64 13
  %x.13 = load i8, ptr %x.13.p, align 1
  %y.13 = load i8, ptr %y.13.p, align 1
  %m.13 = call i8 @llvm.smin.i8(i8 %x.13, i8 %y.13)
  %x.14.p = getelementptr inbounds i8, ptr %x, i64 14
  %y.14.p = getelementptr inbounds i8, ptr %y, i64 14
  %x.14 = load i8, ptr %x.14.p, align 1
  %y.14 = load i8, ptr %y.14.p, align 1
  %m.14 = call i8 @llvm.smin.i8(i8 %x.14, i8 %y.14)
  %x.15.p = getelementptr inbounds i8, ptr %x, i64 15
  %y.15.p = getelementptr inbounds i8, ptr %y, i64 15
  %x.15 = load i8, ptr %x.15.p, align 1
  %y.15 = load i8, ptr %y.15.p, align 1
  %m.15 = call i8 @llvm.smin.i8(i8 %x.15, i8 %y.15)
  %z.0 = zext i8 %m.0 to i64
  %z.1 = zext i8 %m.1 to i64
  %s.1 = shl nuw nsw i64 %z.1, 8
  %z.2 = zext i8 %m.2 to i64
  %s.2 = shl nuw nsw i64 %z.2, 16
  %z.3 = zext i8 %m.3 to i64
  %s.3 = shl nuw nsw i64 %z.3, 24
  %z.4 = zext i8 %m.4 to i64
  %s.4 = shl nuw nsw i64 %z.4, 32
  %z.5 = zext i8 %m.5 to i64
  %s.5 = shl nuw nsw i64 %z.5, 40
  %z.6 = zext i8 %m.6 to i64
  %s.6 = shl nuw nsw i64 %z.6, 48
  %z.7 = zext i8 %m.7 to i64
  %s.7 = shl nuw i64 %z.7, 56
  %o.6 = or disjoint i64 %s.7, %s.6
  %o.5 = or disjoint i64 %o.6, %s.5
  %o.4 = or disjoint i64 %o.5, %s.4
  %o.3 = or disjoint i64 %o.4, %s.3
  %o.2 = or disjoint i64 %o.3, %s.2
  %o.1 = or disjoint i64 %o.2, %s.1
  %w.0 = or disjoint i64 %o.1, %z.0
  %agg.0 = insertvalue { i64, i64 } poison, i64 %w.0, 0
  %z.8 = zext i8 %m.8 to i64
  %z.9 = zext i8 %m.9 to i64
  %s.9 = shl nuw nsw i64 %z.9, 8
  %z.10 = zext i8 %m.10 to i64
  %s.10 = shl nuw nsw i64 %z.10, 16
  %z.11 = zext i8 %m.11 to i64
  %s.11 = shl nuw nsw i64 %z.11, 24
  %z.12 = zext i8 %m.12 to i64
  %s.12 = shl nuw nsw i64 %z.12, 32
  %z.13 = zext i8 %m.13 to i64
  %s.13 = shl nuw nsw i64 %z.13, 40
  %z.14 = zext i8 %m.14 to i64
  %s.14 = shl nuw nsw i64 %z.14, 48
  %z.15 = zext i8 %m.15 to i64
  %s.15 = shl nuw i64 %z.15, 56
  %o.14 = or disjoint i64 %s.15, %s.14
  %o.13 = or disjoint i64 %o.14, %s.13
  %o.12 = or disjoint i64 %o.13, %s.12
  %o.11 = or disjoint i64 %o.12, %s.11
  %o.10 = or disjoint i64 %o.11, %s.10
  %o.9 = or disjoint i64 %o.10, %s.9
  %w.1 = or disjoint i64 %o.9, %z.8
  %agg.1 = insertvalue { i64, i64 } %agg.0, i64 %w.1, 1
  ret { i64, i64 } %agg.1
}

declare i16 @llvm.smin.i16(i16, i16)
declare i8 @llvm.smin.i8(i8, i8)