llvm/test/Transforms/LoopVectorize/WebAssembly/CostModel/int-mac-reduction-costs.ll

; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "Cost of [0-9]+(.[0-9]+)? for VF [0-9]+: (EXPRESSION|INTERLEAVE-GROUP|REPLICATE|WIDEN)" --filter "LV: Selecting VF:" --filter "Estimated cost per lane" --version 6
; REQUIRES: asserts
; RUN: opt -mattr=+simd128 -passes=loop-vectorize -debug-only=loop-vectorize -disable-output -S < %s 2>&1 | FileCheck %s
; RUN: opt -mattr=+simd128 -passes=loop-vectorize -vectorizer-maximize-bandwidth -debug-only=loop-vectorize -disable-output -S < %s 2>&1 | FileCheck %s --check-prefix=MAX

target triple = "wasm32"

define hidden i32 @i32_mac_s8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_s8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_s8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 18 (Estimated cost per lane: 2.25)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP8]]> = ir<%res.08> + partial.reduce.add (mul nsw (ir<%1> sext to i32), (ir<%0> sext to i32))
; MAX:  Cost for VF 16: 10 (Estimated cost per lane: 0.625)
; MAX:  LV: Selecting VF: 16.
;


entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = sext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i8, ptr %arrayidx1, align 1
  %conv2 = sext i8 %1 to i32
  %mul = mul nsw i32 %conv2, %conv
  %add = add nsw i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_s8_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_s8_u8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_s8_u8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 18 (Estimated cost per lane: 2.25)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 16: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 16: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 16: 28 (Estimated cost per lane: 1.75)
; MAX:  LV: Selecting VF: 16.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = sext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i8, ptr %arrayidx1, align 1
  %conv2 = zext i8 %1 to i32
  %mul = mul nsw i32 %conv2, %conv
  %add = add nsw i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_s8_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_s8_s16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5:%[0-9]+]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%1> = load ir<%arrayidx1>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 16 (Estimated cost per lane: 8)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%1> = load ir<%arrayidx1>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 22 (Estimated cost per lane: 5.5)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_s8_s16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5:%[0-9]+]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 16 (Estimated cost per lane: 8)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 22 (Estimated cost per lane: 5.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 2 for VF 8: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 40 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 4 for VF 16: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 16: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 16: 73 (Estimated cost per lane: 4.56)
; MAX:  LV: Selecting VF: 16.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = sext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 1
  %conv2 = sext i16 %1 to i32
  %mul = mul nsw i32 %conv2, %conv
  %add = add nsw i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_s16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_s16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/2)
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 8: EXPRESSION vp<[[VP8]]> = ir<%res.08> + partial.reduce.add (mul nsw (ir<%1> sext to i32), (ir<%0> sext to i32))
; MAX:  Cost for VF 8: 9 (Estimated cost per lane: 1.13)
; MAX:  LV: Selecting VF: 8.
;


entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
  %0 = load i16, ptr %arrayidx, align 2
  %conv = sext i16 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 2
  %conv2 = sext i16 %1 to i32
  %mul = mul nsw i32 %conv2, %conv
  %add = add nsw i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_s16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i64
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_s16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i64
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 18 (Estimated cost per lane: 4.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 7 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i64
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 7 for VF 8: WIDEN-CAST ir<%conv2> = sext ir<%1> to i64
; MAX:  Cost of 4 for VF 8: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 30 (Estimated cost per lane: 3.75)
; MAX:  LV: Selecting VF: 8.
;

entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
  %0 = load i16, ptr %arrayidx, align 2
  %conv = sext i16 %0 to i64
  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 2
  %conv2 = sext i16 %1 to i64
  %mul = mul nsw i64 %conv2, %conv
  %add = add nsw i64 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_s32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_s32'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; CHECK:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_s32'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 4: 13 (Estimated cost per lane: 3.25)
; MAX:  LV: Selecting VF: 4.
;

entry:
  %cmp6.not = icmp eq i32 %N, 0
  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i.08
  %0 = load i32, ptr %arrayidx, align 4
  %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i32 %i.08
  %1 = load i32, ptr %arrayidx1, align 4
  %mul = mul i32 %1, %0
  %conv = sext i32 %mul to i64
  %add = add i64 %res.07, %conv
  %inc = add nuw i32 %i.08, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_u8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_u8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 18 (Estimated cost per lane: 2.25)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 4 for VF 16: EXPRESSION vp<[[VP8]]> = ir<%res.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
; MAX:  Cost for VF 16: 12 (Estimated cost per lane: 0.75)
; MAX:  LV: Selecting VF: 16.
;


entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = zext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i8, ptr %arrayidx1, align 1
  %conv2 = zext i8 %1 to i32
  %mul = mul nuw nsw i32 %conv2, %conv
  %add = add i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_u8_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_u8_u16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5:%[0-9]+]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%1> = load ir<%arrayidx1>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 16 (Estimated cost per lane: 8)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%1> = load ir<%arrayidx1>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 22 (Estimated cost per lane: 5.5)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_u8_u16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5:%[0-9]+]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 16 (Estimated cost per lane: 8)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 22 (Estimated cost per lane: 5.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 2 for VF 8: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 40 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%arrayidx1> = getelementptr inbounds nuw ir<%b>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%1> = load ir<%arrayidx1>
; MAX:  Cost of 4 for VF 16: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 16: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 16: 73 (Estimated cost per lane: 4.56)
; MAX:  LV: Selecting VF: 16.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = zext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 1
  %conv2 = zext i16 %1 to i32
  %mul = mul nuw nsw i32 %conv2, %conv
  %add = add i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_u16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_u16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/2)
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 2 for VF 8: EXPRESSION vp<[[VP8]]> = ir<%res.08> + partial.reduce.add (mul nuw nsw (ir<%1> zext to i32), (ir<%0> zext to i32))
; MAX:  Cost for VF 8: 10 (Estimated cost per lane: 1.25)
; MAX:  LV: Selecting VF: 8.
;


entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
  %0 = load i16, ptr %arrayidx, align 2
  %conv = zext i16 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 2
  %conv2 = zext i16 %1 to i32
  %mul = mul nuw nsw i32 %conv2, %conv
  %add = add i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i32 @i32_mac_u16_s16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i32_mac_u16_s16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; CHECK:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; CHECK:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; CHECK:  LV: Selecting VF: 4.
;
; MAX-LABEL: 'i32_mac_u16_s16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 0 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 4: 10 (Estimated cost per lane: 2.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 2 for VF 8: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.08>
; MAX:  Cost for VF 8: 16 (Estimated cost per lane: 2)
; MAX:  LV: Selecting VF: 8.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ]
  ret i32 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i32 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.09
  %0 = load i16, ptr %arrayidx, align 2
  %conv = zext i16 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.09
  %1 = load i16, ptr %arrayidx1, align 2
  %conv2 = sext i16 %1 to i32
  %mul = mul nuw nsw i32 %conv2, %conv
  %add = add i32 %mul, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_u16(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_u16'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.09> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i64
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.09>
; CHECK:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_u16'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.09> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i64
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%mul>, ir<%res.09>
; MAX:  Cost for VF 2: 12 (Estimated cost per lane: 6)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.09> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%mul>, ir<%res.09>
; MAX:  Cost for VF 4: 18 (Estimated cost per lane: 4.5)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.09> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 7 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i64
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 7 for VF 8: WIDEN-CAST ir<%conv2> = zext ir<%1> to i64
; MAX:  Cost of 4 for VF 8: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 8: WIDEN ir<%add> = add ir<%mul>, ir<%res.09>
; MAX:  Cost for VF 8: 30 (Estimated cost per lane: 3.75)
; MAX:  LV: Selecting VF: 8.
;

entry:
  %cmp8.not = icmp eq i32 %N, 0
  br i1 %cmp8.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.010 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.09 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i16, ptr %a, i32 %i.010
  %0 = load i16, ptr %arrayidx, align 2
  %conv = zext i16 %0 to i64
  %arrayidx1 = getelementptr inbounds nuw i16, ptr %b, i32 %i.010
  %1 = load i16, ptr %arrayidx1, align 2
  %conv2 = zext i16 %1 to i64
  %mul = mul nuw nsw i64 %conv2, %conv
  %add = add i64 %mul, %res.09
  %inc = add nuw i32 %i.010, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_u32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_u32'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%mul> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; CHECK:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_u32'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%mul> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%mul> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 4: 13 (Estimated cost per lane: 3.25)
; MAX:  LV: Selecting VF: 4.
;

entry:
  %cmp6.not = icmp eq i32 %N, 0
  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i.08
  %0 = load i32, ptr %arrayidx, align 4
  %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i32 %i.08
  %1 = load i32, ptr %arrayidx1, align 4
  %mul = mul i32 %1, %0
  %conv = zext i32 %mul to i64
  %add = add i64 %res.07, %conv
  %inc = add nuw i32 %i.08, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_u32_s32(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_u32_s32'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; CHECK:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_u32_s32'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 2 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 2: 11 (Estimated cost per lane: 5.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.07> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul ir<%1>, ir<%0>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%mul> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%res.07>, ir<%conv>
; MAX:  Cost for VF 4: 13 (Estimated cost per lane: 3.25)
; MAX:  LV: Selecting VF: 4.
;
entry:
  %cmp6.not = icmp eq i32 %N, 0
  br i1 %cmp6.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.08 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.07 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i.08
  %0 = load i32, ptr %arrayidx, align 4
  %arrayidx1 = getelementptr inbounds nuw i32, ptr %b, i32 %i.08
  %1 = load i32, ptr %arrayidx1, align 4
  %mul = mul i32 %1, %0
  %conv = sext i32 %mul to i64
  %add = add i64 %res.07, %conv
  %inc = add nuw i32 %i.08, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_u8_u8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_u8_u8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv3> = zext nneg ir<%mul> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; CHECK:  Cost for VF 2: 13 (Estimated cost per lane: 6.5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_u8_u8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv3> = zext nneg ir<%mul> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 2: 13 (Estimated cost per lane: 6.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%conv3> = zext nneg ir<%mul> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 8: WIDEN-CAST ir<%conv3> = zext nneg ir<%mul> to i64
; MAX:  Cost of 4 for VF 8: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 8: 24 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv> = zext ir<%0> to i32
; MAX:  Cost of 2 for VF 16: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv2> = zext ir<%1> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%mul> = mul nuw nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 8 for VF 16: WIDEN-CAST ir<%conv3> = zext nneg ir<%mul> to i64
; MAX:  Cost of 8 for VF 16: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 16: 40 (Estimated cost per lane: 2.5)
; MAX:  LV: Selecting VF: 16.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = zext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i8, ptr %arrayidx1, align 1
  %conv2 = zext i8 %1 to i32
  %mul = mul nuw nsw i32 %conv2, %conv
  %conv3 = zext nneg i32 %mul to i64
  %add = add nsw i64 %conv3, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @i64_mac_s8_s8(ptr nocapture noundef readonly %a, ptr nocapture noundef readonly %b, i32 noundef %N) {
; CHECK-LABEL: 'i64_mac_s8_s8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv3> = sext ir<%mul> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; CHECK:  Cost for VF 2: 13 (Estimated cost per lane: 6.5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'i64_mac_s8_s8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%add>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%0> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 3 for VF 2: WIDEN ir<%1> = load vp<[[VP7:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%conv3> = sext ir<%mul> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 2: 13 (Estimated cost per lane: 6.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 4: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%conv3> = sext ir<%mul> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 4: 15 (Estimated cost per lane: 3.75)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 4 for VF 8: WIDEN-CAST ir<%conv3> = sext ir<%mul> to i64
; MAX:  Cost of 4 for VF 8: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 8: 24 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%res.08> = phi (add) vp<[[VP3]]>, ir<%add>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%0> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv> = sext ir<%0> to i32
; MAX:  Cost of 2 for VF 16: WIDEN ir<%1> = load vp<[[VP7]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%conv2> = sext ir<%1> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%mul> = mul nsw ir<%conv2>, ir<%conv>
; MAX:  Cost of 8 for VF 16: WIDEN-CAST ir<%conv3> = sext ir<%mul> to i64
; MAX:  Cost of 8 for VF 16: WIDEN ir<%add> = add ir<%conv3>, ir<%res.08>
; MAX:  Cost for VF 16: 40 (Estimated cost per lane: 2.5)
; MAX:  LV: Selecting VF: 16.
;
entry:
  %cmp7.not = icmp eq i32 %N, 0
  br i1 %cmp7.not, label %for.cond.cleanup, label %for.body

for.cond.cleanup:
  %res.0.lcssa = phi i64 [ 0, %entry ], [ %add, %for.body ]
  ret i64 %res.0.lcssa

for.body:
  %i.09 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
  %res.08 = phi i64 [ %add, %for.body ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds nuw i8, ptr %a, i32 %i.09
  %0 = load i8, ptr %arrayidx, align 1
  %conv = sext i8 %0 to i32
  %arrayidx1 = getelementptr inbounds nuw i8, ptr %b, i32 %i.09
  %1 = load i8, ptr %arrayidx1, align 1
  %conv2 = sext i8 %1 to i32
  %mul = mul nsw i32 %conv2, %conv
  %conv3 = sext i32 %mul to i64
  %add = add nsw i64 %conv3, %res.08
  %inc = add nuw i32 %i.09, 1
  %exitcond.not = icmp eq i32 %inc, %N
  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
}

define hidden i64 @u64_mac_u8(ptr noundef readonly captures(none) %arg, i32 noundef %arg1) {
; CHECK-LABEL: 'u64_mac_u8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%i6> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%i12>
; CHECK:  Cost of 3 for VF 2: WIDEN ir<%i8> = load vp<[[VP6:%[0-9]+]]>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%i9> = zext ir<%i8> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%i10> = mul nuw nsw ir<%i9>, ir<%i9>
; CHECK:  Cost of 1 for VF 2: WIDEN-CAST ir<%i11> = zext nneg ir<%i10> to i64
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%i12> = add ir<%i6>, ir<%i11>
; CHECK:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; CHECK:  LV: Selecting VF: 2.
;
; MAX-LABEL: 'u64_mac_u8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%i6> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%i12>
; MAX:  Cost of 3 for VF 2: WIDEN ir<%i8> = load vp<[[VP6:%[0-9]+]]>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%i9> = zext ir<%i8> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%i10> = mul nuw nsw ir<%i9>, ir<%i9>
; MAX:  Cost of 1 for VF 2: WIDEN-CAST ir<%i11> = zext nneg ir<%i10> to i64
; MAX:  Cost of 1 for VF 2: WIDEN ir<%i12> = add ir<%i6>, ir<%i11>
; MAX:  Cost for VF 2: 10 (Estimated cost per lane: 5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%i6> = phi (add) vp<[[VP3]]>, ir<%i12>
; MAX:  Cost of 2 for VF 4: WIDEN ir<%i8> = load vp<[[VP6]]>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%i9> = zext ir<%i8> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%i10> = mul nuw nsw ir<%i9>, ir<%i9>
; MAX:  Cost of 2 for VF 4: WIDEN-CAST ir<%i11> = zext nneg ir<%i10> to i64
; MAX:  Cost of 2 for VF 4: WIDEN ir<%i12> = add ir<%i6>, ir<%i11>
; MAX:  Cost for VF 4: 12 (Estimated cost per lane: 3)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%i6> = phi (add) vp<[[VP3]]>, ir<%i12>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%i8> = load vp<[[VP6]]>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%i9> = zext ir<%i8> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%i10> = mul nuw nsw ir<%i9>, ir<%i9>
; MAX:  Cost of 4 for VF 8: WIDEN-CAST ir<%i11> = zext nneg ir<%i10> to i64
; MAX:  Cost of 4 for VF 8: WIDEN ir<%i12> = add ir<%i6>, ir<%i11>
; MAX:  Cost for VF 8: 19 (Estimated cost per lane: 2.38)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%i6> = phi (add) vp<[[VP3]]>, ir<%i12>
; MAX:  Cost of 2 for VF 16: WIDEN ir<%i8> = load vp<[[VP6]]>
; MAX:  Cost of 6 for VF 16: WIDEN-CAST ir<%i9> = zext ir<%i8> to i32
; MAX:  Cost of 4 for VF 16: WIDEN ir<%i10> = mul nuw nsw ir<%i9>, ir<%i9>
; MAX:  Cost of 8 for VF 16: WIDEN-CAST ir<%i11> = zext nneg ir<%i10> to i64
; MAX:  Cost of 8 for VF 16: WIDEN ir<%i12> = add ir<%i6>, ir<%i11>
; MAX:  Cost for VF 16: 32 (Estimated cost per lane: 2)
; MAX:  LV: Selecting VF: 16.
;
bb:
  %i = icmp eq i32 %arg1, 0
  br i1 %i, label %bb2, label %bb4

bb2:
  %i3 = phi i64 [ 0, %bb ], [ %i12, %bb4 ]
  ret i64 %i3

bb4:
  %i5 = phi i32 [ %i13, %bb4 ], [ 0, %bb ]
  %i6 = phi i64 [ %i12, %bb4 ], [ 0, %bb ]
  %i7 = getelementptr inbounds nuw i8, ptr %arg, i32 %i5
  %i8 = load i8, ptr %i7, align 1
  %i9 = zext i8 %i8 to i32
  %i10 = mul nuw nsw i32 %i9, %i9
  %i11 = zext nneg i32 %i10 to i64
  %i12 = add i64 %i6, %i11
  %i13 = add nuw i32 %i5, 1
  %i14 = icmp eq i32 %i13, %arg1
  br i1 %i14, label %bb2, label %bb4
}

; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite, inaccessiblemem: readwrite)
define hidden void @mac_3d_i32_s8(ptr dead_on_unwind noalias nofree writable writeonly sret(%struct.ThreeInts) align 4 captures(none) %0, ptr nofree noundef readonly captures(none) %1, ptr nofree noundef readonly captures(none) %2, i32 noundef %3) {
; CHECK-LABEL: 'mac_3d_i32_s8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%18>
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5:%[0-9]+]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%15> = load ir<%14>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%18> = add ir<%17>, ir<%8>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%23> = load ir<%22>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%26> = add ir<%25>, ir<%9>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%28> = load ir<%27>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%31> = load ir<%30>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%34> = add ir<%33>, ir<%10>
; CHECK:  Cost for VF 2: 49 (Estimated cost per lane: 24.5)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%15> = load ir<%14>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%18> = add ir<%17>, ir<%8>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%20> = load ir<%19>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%23> = load ir<%22>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%26> = add ir<%25>, ir<%9>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%28> = load ir<%27>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%31> = load ir<%30>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%34> = add ir<%33>, ir<%10>
; CHECK:  Cost for VF 4: 91 (Estimated cost per lane: 22.8)
; CHECK:  LV: Selecting VF: 1.
;
; MAX-LABEL: 'mac_3d_i32_s8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%18>
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5:%[0-9]+]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 2: 49 (Estimated cost per lane: 24.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 4: 91 (Estimated cost per lane: 22.8)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 8: 181 (Estimated cost per lane: 22.6)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, vp<[[VP6:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP6]]> = ir<%8> + partial.reduce.add (mul nsw (ir<%15> sext to i32), (ir<%12> sext to i32))
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP7]]> = ir<%9> + partial.reduce.add (mul nsw (ir<%23> sext to i32), (ir<%20> sext to i32))
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP8]]> = ir<%10> + partial.reduce.add (mul nsw (ir<%31> sext to i32), (ir<%28> sext to i32))
; MAX:  Cost for VF 16: 301 (Estimated cost per lane: 18.8)
; MAX:  LV: Selecting VF: 16.
;
  %5 = icmp eq i32 %3, 0
  br i1 %5, label %37, label %6

6:                                                ; preds = %4, %6
  %7 = phi i32 [ %35, %6 ], [ 0, %4 ]
  %8 = phi i32 [ %18, %6 ], [ 0, %4 ]
  %9 = phi i32 [ %26, %6 ], [ 0, %4 ]
  %10 = phi i32 [ %34, %6 ], [ 0, %4 ]
  %11 = getelementptr inbounds nuw [3 x i8], ptr %1, i32 %7
  %12 = load i8, ptr %11, align 1
  %13 = sext i8 %12 to i32
  %14 = getelementptr inbounds nuw [3 x i8], ptr %2, i32 %7
  %15 = load i8, ptr %14, align 1
  %16 = sext i8 %15 to i32
  %17 = mul nsw i32 %16, %13
  %18 = add nsw i32 %17, %8
  %19 = getelementptr inbounds nuw i8, ptr %11, i32 1
  %20 = load i8, ptr %19, align 1
  %21 = sext i8 %20 to i32
  %22 = getelementptr inbounds nuw i8, ptr %14, i32 1
  %23 = load i8, ptr %22, align 1
  %24 = sext i8 %23 to i32
  %25 = mul nsw i32 %24, %21
  %26 = add nsw i32 %25, %9
  %27 = getelementptr inbounds nuw i8, ptr %11, i32 2
  %28 = load i8, ptr %27, align 1
  %29 = sext i8 %28 to i32
  %30 = getelementptr inbounds nuw i8, ptr %14, i32 2
  %31 = load i8, ptr %30, align 1
  %32 = sext i8 %31 to i32
  %33 = mul nsw i32 %32, %29
  %34 = add nsw i32 %33, %10
  %35 = add nuw i32 %7, 1
  %36 = icmp eq i32 %35, %3
  br i1 %36, label %37, label %6

37:                                               ; preds = %6, %4
  %38 = phi i32 [ 0, %4 ], [ %34, %6 ]
  %39 = phi i32 [ 0, %4 ], [ %26, %6 ]
  %40 = phi i32 [ 0, %4 ], [ %18, %6 ]
  %41 = getelementptr inbounds nuw i8, ptr %0, i32 8
  %42 = getelementptr inbounds nuw i8, ptr %0, i32 4
  store i32 %40, ptr %0, align 4
  store i32 %39, ptr %42, align 4
  store i32 %38, ptr %41, align 4
  ret void
}

%struct.ThreeInts = type { i32, i32, i32 }
; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite, inaccessiblemem: readwrite)
define hidden void @mac_3d_u32_s8(ptr dead_on_unwind noalias nofree writable writeonly sret(%struct.ThreeInts) align 4 captures(none) %0, ptr nofree noundef readonly captures(none) %1, ptr nofree noundef readonly captures(none) %2, i32 noundef %3) {
; CHECK-LABEL: 'mac_3d_u32_s8'
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%18>
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; CHECK:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5:%[0-9]+]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%15> = load ir<%14>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%18> = add ir<%17>, ir<%8>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%23> = load ir<%22>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%26> = add ir<%25>, ir<%9>
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%28> = load ir<%27>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; CHECK:  Cost of 0 for VF 2: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; CHECK:  Cost of 6 for VF 2: REPLICATE ir<%31> = load ir<%30>
; CHECK:  Cost of 0 for VF 2: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; CHECK:  Cost of 1 for VF 2: WIDEN ir<%34> = add ir<%33>, ir<%10>
; CHECK:  Cost for VF 2: 49 (Estimated cost per lane: 24.5)
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; CHECK:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%15> = load ir<%14>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%18> = add ir<%17>, ir<%8>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%20> = load ir<%19>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%23> = load ir<%22>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%26> = add ir<%25>, ir<%9>
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%28> = load ir<%27>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; CHECK:  Cost of 0 for VF 4: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; CHECK:  Cost of 12 for VF 4: REPLICATE ir<%31> = load ir<%30>
; CHECK:  Cost of 1 for VF 4: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; CHECK:  Cost of 1 for VF 4: WIDEN ir<%34> = add ir<%33>, ir<%10>
; CHECK:  Cost for VF 4: 91 (Estimated cost per lane: 22.8)
; CHECK:  LV: Selecting VF: 1.
;
; MAX-LABEL: 'mac_3d_u32_s8'
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3:%[0-9]+]]>, ir<%18>
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5:%[0-9]+]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 2: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 6 for VF 2: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 0 for VF 2: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 1 for VF 2: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 1 for VF 2: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 2: 49 (Estimated cost per lane: 24.5)
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 4: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 12 for VF 4: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 1 for VF 4: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 1 for VF 4: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 1 for VF 4: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 4: 91 (Estimated cost per lane: 22.8)
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, ir<%18>
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, ir<%26>
; MAX:  Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, ir<%34>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%13> = sext ir<%12> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%16> = sext ir<%15> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%17> = mul nsw ir<%16>, ir<%13>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%18> = add ir<%17>, ir<%8>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%21> = sext ir<%20> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%24> = sext ir<%23> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%25> = mul nsw ir<%24>, ir<%21>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%26> = add ir<%25>, ir<%9>
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%29> = sext ir<%28> to i32
; MAX:  Cost of 0 for VF 8: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 24 for VF 8: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 3 for VF 8: WIDEN-CAST ir<%32> = sext ir<%31> to i32
; MAX:  Cost of 2 for VF 8: WIDEN ir<%33> = mul nsw ir<%32>, ir<%29>
; MAX:  Cost of 2 for VF 8: WIDEN ir<%34> = add ir<%33>, ir<%10>
; MAX:  Cost for VF 8: 181 (Estimated cost per lane: 22.6)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%8> = phi (add) vp<[[VP3]]>, vp<[[VP6:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%9> = phi (add) vp<[[VP3]]>, vp<[[VP7:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%10> = phi (add) vp<[[VP3]]>, vp<[[VP8:%[0-9]+]]> (VF scaled by 1/4)
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%11> = getelementptr inbounds nuw ir<%1>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%12> = load ir<%11>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%14> = getelementptr inbounds nuw ir<%2>, vp<[[VP5]]>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%15> = load ir<%14>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP6]]> = ir<%8> + partial.reduce.add (mul nsw (ir<%15> sext to i32), (ir<%12> sext to i32))
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%19> = getelementptr inbounds nuw ir<%11>, ir<1>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%20> = load ir<%19>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%22> = getelementptr inbounds nuw ir<%14>, ir<1>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%23> = load ir<%22>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP7]]> = ir<%9> + partial.reduce.add (mul nsw (ir<%23> sext to i32), (ir<%20> sext to i32))
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%27> = getelementptr inbounds nuw ir<%11>, ir<2>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%28> = load ir<%27>
; MAX:  Cost of 0 for VF 16: REPLICATE ir<%30> = getelementptr inbounds nuw ir<%14>, ir<2>
; MAX:  Cost of 48 for VF 16: REPLICATE ir<%31> = load ir<%30>
; MAX:  Cost of 2 for VF 16: EXPRESSION vp<[[VP8]]> = ir<%10> + partial.reduce.add (mul nsw (ir<%31> sext to i32), (ir<%28> sext to i32))
; MAX:  Cost for VF 16: 301 (Estimated cost per lane: 18.8)
; MAX:  LV: Selecting VF: 16.
;
  %5 = icmp eq i32 %3, 0
  br i1 %5, label %37, label %6

6:                                                ; preds = %4, %6
  %7 = phi i32 [ %35, %6 ], [ 0, %4 ]
  %8 = phi i32 [ %18, %6 ], [ 0, %4 ]
  %9 = phi i32 [ %26, %6 ], [ 0, %4 ]
  %10 = phi i32 [ %34, %6 ], [ 0, %4 ]
  %11 = getelementptr inbounds nuw [3 x i8], ptr %1, i32 %7
  %12 = load i8, ptr %11, align 1
  %13 = sext i8 %12 to i32
  %14 = getelementptr inbounds nuw [3 x i8], ptr %2, i32 %7
  %15 = load i8, ptr %14, align 1
  %16 = sext i8 %15 to i32
  %17 = mul nsw i32 %16, %13
  %18 = add i32 %17, %8
  %19 = getelementptr inbounds nuw i8, ptr %11, i32 1
  %20 = load i8, ptr %19, align 1
  %21 = sext i8 %20 to i32
  %22 = getelementptr inbounds nuw i8, ptr %14, i32 1
  %23 = load i8, ptr %22, align 1
  %24 = sext i8 %23 to i32
  %25 = mul nsw i32 %24, %21
  %26 = add i32 %25, %9
  %27 = getelementptr inbounds nuw i8, ptr %11, i32 2
  %28 = load i8, ptr %27, align 1
  %29 = sext i8 %28 to i32
  %30 = getelementptr inbounds nuw i8, ptr %14, i32 2
  %31 = load i8, ptr %30, align 1
  %32 = sext i8 %31 to i32
  %33 = mul nsw i32 %32, %29
  %34 = add i32 %33, %10
  %35 = add nuw i32 %7, 1
  %36 = icmp eq i32 %35, %3
  br i1 %36, label %37, label %6

37:                                               ; preds = %6, %4
  %38 = phi i32 [ 0, %4 ], [ %34, %6 ]
  %39 = phi i32 [ 0, %4 ], [ %26, %6 ]
  %40 = phi i32 [ 0, %4 ], [ %18, %6 ]
  %41 = getelementptr inbounds nuw i8, ptr %0, i32 8
  %42 = getelementptr inbounds nuw i8, ptr %0, i32 4
  store i32 %40, ptr %0, align 4
  store i32 %39, ptr %42, align 4
  store i32 %38, ptr %41, align 4
  ret void
}