| ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+(.[0-9]+)? for VF 1 For instruction:\s*%v0 = load" --filter "Cost of [0-9]+(.[0-9]+)? for VF [0-9]+: (INTERLEAVE-GROUP with factor [0-9]+,|EXPRESSION|REPLICATE|WIDEN)" --filter "^ ir<.* = load from index" --filter "LV: Scalar loop costs:" --filter "LV: Selecting VF:" --filter "Estimated cost per lane" | |
| ; REQUIRES: asserts | |
| ; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize %s --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s | |
| ; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=loop-vectorize %s --vectorizer-maximize-bandwidth --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=MAX | |
| define hidden void @horizontal(ptr nofree noundef readonly captures(none) %0, i32 noundef %1, ptr nofree noundef writeonly captures(none) %2, i32 noundef %3, i32 noundef %4, ptr nofree noundef readonly captures(none) %5, ptr nofree noundef readonly captures(none) %6, i32 noundef %7, ptr nofree noundef readonly captures(none) %8) { | |
| ; CHECK-LABEL: 'horizontal' | |
| ; CHECK: LV: Scalar loop costs: 9. | |
| ; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4:%[0-9]+]]>, ir<%71> | |
| ; CHECK: Cost of 2 for VF 2: WIDEN ir<%65> = load vp<[[VP7:%[0-9]+]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; CHECK: Cost of 3 for VF 2: WIDEN ir<%68> = load vp<[[VP8:%[0-9]+]]>, align 1 | |
| ; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; CHECK: Cost of 1 for VF 2: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; CHECK: Cost of 1 for VF 2: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; CHECK: Cost for VF 2: 11 (Estimated cost per lane: 5.5) | |
| ; CHECK: Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4]]>, ir<%71> | |
| ; CHECK: Cost of 2 for VF 4: WIDEN ir<%65> = load vp<[[VP7]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 4: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; CHECK: Cost of 2 for VF 4: WIDEN ir<%68> = load vp<[[VP8]]>, align 1 | |
| ; CHECK: Cost of 1 for VF 4: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; CHECK: Cost of 1 for VF 4: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; CHECK: Cost of 1 for VF 4: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; CHECK: Cost for VF 4: 11 (Estimated cost per lane: 2.75) | |
| ; CHECK: LV: Selecting VF: 4. | |
| ; | |
| ; MAX-LABEL: 'horizontal' | |
| ; MAX: LV: Scalar loop costs: 9. | |
| ; MAX: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4:%[0-9]+]]>, ir<%71> | |
| ; MAX: Cost of 2 for VF 2: WIDEN ir<%65> = load vp<[[VP7:%[0-9]+]]>, align 2 | |
| ; MAX: Cost of 0 for VF 2: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; MAX: Cost of 3 for VF 2: WIDEN ir<%68> = load vp<[[VP8:%[0-9]+]]>, align 1 | |
| ; MAX: Cost of 0 for VF 2: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; MAX: Cost of 1 for VF 2: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; MAX: Cost of 1 for VF 2: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; MAX: Cost for VF 2: 11 (Estimated cost per lane: 5.5) | |
| ; MAX: Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4]]>, ir<%71> | |
| ; MAX: Cost of 2 for VF 4: WIDEN ir<%65> = load vp<[[VP7]]>, align 2 | |
| ; MAX: Cost of 0 for VF 4: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; MAX: Cost of 2 for VF 4: WIDEN ir<%68> = load vp<[[VP8]]>, align 1 | |
| ; MAX: Cost of 1 for VF 4: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; MAX: Cost of 1 for VF 4: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; MAX: Cost of 1 for VF 4: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; MAX: Cost for VF 4: 11 (Estimated cost per lane: 2.75) | |
| ; MAX: Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4]]>, ir<%71> | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%65> = load vp<[[VP7]]>, align 2 | |
| ; MAX: Cost of 2 for VF 8: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%68> = load vp<[[VP8]]>, align 1 | |
| ; MAX: Cost of 3 for VF 8: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; MAX: Cost for VF 8: 17 (Estimated cost per lane: 2.13) | |
| ; MAX: Cost of 1 for VF 16: WIDEN-REDUCTION-PHI ir<%63> = phi (add) vp<[[VP4]]>, ir<%71> | |
| ; MAX: Cost of 2 for VF 16: WIDEN ir<%65> = load vp<[[VP7]]>, align 2 | |
| ; MAX: Cost of 4 for VF 16: WIDEN-CAST ir<%66> = sext ir<%65> to i32 | |
| ; MAX: Cost of 2 for VF 16: WIDEN ir<%68> = load vp<[[VP8]]>, align 1 | |
| ; MAX: Cost of 6 for VF 16: WIDEN-CAST ir<%69> = zext ir<%68> to i32 | |
| ; MAX: Cost of 4 for VF 16: WIDEN ir<%70> = mul nsw ir<%69>, ir<%66> | |
| ; MAX: Cost of 4 for VF 16: WIDEN ir<%71> = add ir<%70>, ir<%63> | |
| ; MAX: Cost for VF 16: 26 (Estimated cost per lane: 1.63) | |
| ; MAX: LV: Selecting VF: 16. | |
| ; | |
| %10 = getelementptr inbounds nuw i8, ptr %6, i32 4 | |
| %11 = load i16, ptr %10, align 4 | |
| %12 = lshr i16 %11, 1 | |
| %13 = zext nneg i16 %12 to i32 | |
| %14 = sub nsw i32 1, %13 | |
| %15 = getelementptr inbounds nuw i8, ptr %5, i32 4 | |
| %16 = load i16, ptr %15, align 4 | |
| %17 = mul i32 %14, %1 | |
| %18 = getelementptr inbounds i8, ptr %0, i32 %17 | |
| %19 = and i32 %7, 15 | |
| %20 = load ptr, ptr %5, align 4 | |
| %21 = zext i16 %16 to i32 | |
| %22 = mul nuw nsw i32 %19, %21 | |
| %23 = getelementptr inbounds nuw [2 x i8], ptr %20, i32 %22 | |
| %24 = icmp sgt i32 %4, 0 | |
| br i1 %24, label %25, label %42 | |
| 25: ; preds = %9 | |
| %26 = lshr i16 %16, 1 | |
| %27 = zext nneg i16 %26 to i32 | |
| %28 = icmp sgt i32 %3, 0 | |
| br i1 %28, label %29, label %42 | |
| 29: ; preds = %25 | |
| %30 = getelementptr inbounds nuw i8, ptr %8, i32 12 | |
| %31 = load i32, ptr %30, align 4 | |
| %32 = shl nuw i32 1, %31 | |
| %33 = ashr i32 %32, 1 | |
| br label %34 | |
| 34: ; preds = %29, %50 | |
| %35 = phi i32 [ 0, %29 ], [ %51, %50 ] | |
| %36 = mul nsw i32 %35, %1 | |
| %37 = sub i32 %36, %27 | |
| %38 = mul nuw nsw i32 %35, %3 | |
| %39 = getelementptr [2 x i8], ptr %2, i32 %38 | |
| %40 = getelementptr i8, ptr %18, i32 %37 | |
| %41 = getelementptr i8, ptr %40, i32 1 | |
| br label %43 | |
| 42: ; preds = %50, %25, %9 | |
| ret void | |
| 43: ; preds = %34, %53 | |
| %44 = phi i32 [ 0, %34 ], [ %59, %53 ] | |
| %45 = load i16, ptr %15, align 4 | |
| %46 = zext i16 %45 to i32 | |
| %47 = icmp eq i16 %45, 0 | |
| br i1 %47, label %53, label %48 | |
| 48: ; preds = %43 | |
| %49 = getelementptr i8, ptr %41, i32 %44 | |
| br label %61 | |
| 50: ; preds = %53 | |
| %51 = add nuw nsw i32 %35, 1 | |
| %52 = icmp eq i32 %51, %4 | |
| br i1 %52, label %42, label %34 | |
| 53: ; preds = %61, %43 | |
| %54 = phi i32 [ 16384, %43 ], [ %71, %61 ] | |
| %55 = add nsw i32 %33, %54 | |
| %56 = ashr i32 %55, %31 | |
| %57 = trunc i32 %56 to i16 | |
| %58 = getelementptr [2 x i8], ptr %39, i32 %44 | |
| store i16 %57, ptr %58, align 2 | |
| %59 = add nuw nsw i32 %44, 1 | |
| %60 = icmp eq i32 %59, %3 | |
| br i1 %60, label %50, label %43 | |
| 61: ; preds = %48, %61 | |
| %62 = phi i32 [ 0, %48 ], [ %72, %61 ] | |
| %63 = phi i32 [ 16384, %48 ], [ %71, %61 ] | |
| %64 = getelementptr inbounds nuw [2 x i8], ptr %23, i32 %62 | |
| %65 = load i16, ptr %64, align 2 | |
| %66 = sext i16 %65 to i32 | |
| %67 = getelementptr i8, ptr %49, i32 %62 | |
| %68 = load i8, ptr %67, align 1 | |
| %69 = zext i8 %68 to i32 | |
| %70 = mul nsw i32 %69, %66 | |
| %71 = add nsw i32 %70, %63 | |
| %72 = add nuw nsw i32 %62, 1 | |
| %73 = icmp eq i32 %72, %46 | |
| br i1 %73, label %53, label %61 | |
| } | |
| define hidden void @vertical(ptr nofree noundef readonly captures(none) %0, ptr nofree noundef writeonly captures(none) %1, i32 noundef %2, i32 noundef %3, i32 noundef %4, ptr nofree noundef readonly captures(none) %5, i32 noundef %6, ptr nofree noundef readonly captures(none) %7) { | |
| ; CHECK-LABEL: 'vertical' | |
| ; CHECK: LV: Scalar loop costs: 10. | |
| ; CHECK: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%76> = phi (add) vp<[[VP4:%[0-9]+]]>, ir<%86> | |
| ; CHECK: Cost of 2 for VF 2: WIDEN ir<%78> = load vp<[[VP7:%[0-9]+]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%79> = sext ir<%78> to i32 | |
| ; CHECK: Cost of 2 for VF 2: WIDEN ir<%83> = load vp<[[VP8:%[0-9]+]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 2: WIDEN-CAST ir<%84> = sext ir<%83> to i32 | |
| ; CHECK: Cost of 1 for VF 2: WIDEN ir<%85> = mul nsw ir<%84>, ir<%79> | |
| ; CHECK: Cost of 1 for VF 2: WIDEN ir<%86> = add ir<%85>, ir<%76> | |
| ; CHECK: Cost for VF 2: 11 (Estimated cost per lane: 5.5) | |
| ; CHECK: Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%76> = phi (add) vp<[[VP4]]>, ir<%86> | |
| ; CHECK: Cost of 2 for VF 4: WIDEN ir<%78> = load vp<[[VP7]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 4: WIDEN-CAST ir<%79> = sext ir<%78> to i32 | |
| ; CHECK: Cost of 2 for VF 4: WIDEN ir<%83> = load vp<[[VP8]]>, align 2 | |
| ; CHECK: Cost of 0 for VF 4: WIDEN-CAST ir<%84> = sext ir<%83> to i32 | |
| ; CHECK: Cost of 1 for VF 4: WIDEN ir<%85> = mul nsw ir<%84>, ir<%79> | |
| ; CHECK: Cost of 1 for VF 4: WIDEN ir<%86> = add ir<%85>, ir<%76> | |
| ; CHECK: Cost for VF 4: 11 (Estimated cost per lane: 2.75) | |
| ; CHECK: LV: Selecting VF: 4. | |
| ; | |
| ; MAX-LABEL: 'vertical' | |
| ; MAX: LV: Scalar loop costs: 10. | |
| ; MAX: Cost of 1 for VF 2: WIDEN-REDUCTION-PHI ir<%76> = phi (add) vp<[[VP4:%[0-9]+]]>, ir<%86> | |
| ; MAX: Cost of 2 for VF 2: WIDEN ir<%78> = load vp<[[VP7:%[0-9]+]]>, align 2 | |
| ; MAX: Cost of 0 for VF 2: WIDEN-CAST ir<%79> = sext ir<%78> to i32 | |
| ; MAX: Cost of 2 for VF 2: WIDEN ir<%83> = load vp<[[VP8:%[0-9]+]]>, align 2 | |
| ; MAX: Cost of 0 for VF 2: WIDEN-CAST ir<%84> = sext ir<%83> to i32 | |
| ; MAX: Cost of 1 for VF 2: WIDEN ir<%85> = mul nsw ir<%84>, ir<%79> | |
| ; MAX: Cost of 1 for VF 2: WIDEN ir<%86> = add ir<%85>, ir<%76> | |
| ; MAX: Cost for VF 2: 11 (Estimated cost per lane: 5.5) | |
| ; MAX: Cost of 1 for VF 4: WIDEN-REDUCTION-PHI ir<%76> = phi (add) vp<[[VP4]]>, ir<%86> | |
| ; MAX: Cost of 2 for VF 4: WIDEN ir<%78> = load vp<[[VP7]]>, align 2 | |
| ; MAX: Cost of 0 for VF 4: WIDEN-CAST ir<%79> = sext ir<%78> to i32 | |
| ; MAX: Cost of 2 for VF 4: WIDEN ir<%83> = load vp<[[VP8]]>, align 2 | |
| ; MAX: Cost of 0 for VF 4: WIDEN-CAST ir<%84> = sext ir<%83> to i32 | |
| ; MAX: Cost of 1 for VF 4: WIDEN ir<%85> = mul nsw ir<%84>, ir<%79> | |
| ; MAX: Cost of 1 for VF 4: WIDEN ir<%86> = add ir<%85>, ir<%76> | |
| ; MAX: Cost for VF 4: 11 (Estimated cost per lane: 2.75) | |
| ; MAX: Cost of 1 for VF 8: WIDEN-REDUCTION-PHI ir<%76> = phi (add) vp<[[VP4]]>, vp<[[VP9:%[0-9]+]]> (VF scaled by 1/2) | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%78> = load vp<[[VP7]]>, align 2 | |
| ; MAX: Cost of 2 for VF 8: WIDEN ir<%83> = load vp<[[VP8]]>, align 2 | |
| ; MAX: Cost of 1 for VF 8: EXPRESSION vp<[[VP9]]> = ir<%76> + partial.reduce.add (mul nsw (ir<%83> sext to i32), (ir<%78> sext to i32)) | |
| ; MAX: Cost for VF 8: 10 (Estimated cost per lane: 1.25) | |
| ; MAX: LV: Selecting VF: 8. | |
| ; | |
| %9 = getelementptr inbounds nuw i8, ptr %5, i32 4 | |
| %10 = load i16, ptr %9, align 4 | |
| %11 = lshr i16 %10, 1 | |
| %12 = zext nneg i16 %11 to i32 | |
| %13 = add nsw i32 %12, -1 | |
| %14 = getelementptr inbounds nuw i8, ptr %7, i32 12 | |
| %15 = load i32, ptr %14, align 4 | |
| %16 = getelementptr inbounds nuw i8, ptr %7, i32 16 | |
| %17 = load i32, ptr %16, align 4 | |
| %18 = add i32 %15, %17 | |
| %19 = sub i32 14, %18 | |
| %20 = mul nsw i32 %13, %3 | |
| %21 = getelementptr inbounds [2 x i8], ptr %0, i32 %20 | |
| %22 = and i32 %6, 15 | |
| %23 = load ptr, ptr %5, align 4 | |
| %24 = zext i16 %10 to i32 | |
| %25 = mul nuw nsw i32 %22, %24 | |
| %26 = getelementptr inbounds nuw [2 x i8], ptr %23, i32 %25 | |
| %27 = sub nsw i32 22, %15 | |
| %28 = icmp sgt i32 %4, 0 | |
| br i1 %28, label %29, label %41 | |
| 29: ; preds = %8 | |
| %30 = icmp sgt i32 %3, 0 | |
| %31 = shl nuw i32 1, %27 | |
| %32 = shl nuw i32 1, %19 | |
| %33 = ashr i32 %32, 1 | |
| br i1 %30, label %34, label %41 | |
| 34: ; preds = %29 | |
| %35 = sub i32 1, %12 | |
| br label %36 | |
| 36: ; preds = %34, %42 | |
| %37 = phi i32 [ %43, %42 ], [ 0, %34 ] | |
| %38 = add i32 %37, %35 | |
| %39 = mul nsw i32 %37, %2 | |
| %40 = getelementptr i8, ptr %1, i32 %39 | |
| br label %45 | |
| 41: ; preds = %42, %29, %8 | |
| ret void | |
| 42: ; preds = %51 | |
| %43 = add nuw nsw i32 %37, 1 | |
| %44 = icmp eq i32 %43, %4 | |
| br i1 %44, label %41, label %36 | |
| 45: ; preds = %36, %51 | |
| %46 = phi i32 [ 0, %36 ], [ %72, %51 ] | |
| %47 = load i16, ptr %9, align 4 | |
| %48 = zext i16 %47 to i32 | |
| %49 = getelementptr [2 x i8], ptr %21, i32 %46 | |
| %50 = icmp eq i16 %47, 0 | |
| br i1 %50, label %51, label %74 | |
| 51: ; preds = %74, %45 | |
| %52 = phi i32 [ %31, %45 ], [ %86, %74 ] | |
| %53 = load i32, ptr %16, align 4 | |
| %54 = shl nuw i32 1, %53 | |
| %55 = ashr i32 %54, 1 | |
| %56 = add nsw i32 %55, %52 | |
| %57 = ashr i32 %56, %53 | |
| %58 = sub nsw i32 %27, %53 | |
| %59 = shl nsw i32 -1, %58 | |
| %60 = add nsw i32 %58, -1 | |
| %61 = shl nsw i32 -1, %60 | |
| %62 = add i32 %61, %59 | |
| %63 = add i32 %62, %57 | |
| %64 = shl i32 %63, 16 | |
| %65 = ashr exact i32 %64, 16 | |
| %66 = add nsw i32 %65, %33 | |
| %67 = ashr i32 %66, %19 | |
| %68 = tail call i32 @llvm.smax.i32(i32 range(i32 -1073774592, 1073774591) %67, i32 0) | |
| %69 = tail call noundef range(i32 0, 256) i32 @llvm.umin.i32(i32 %68, i32 255) | |
| %70 = trunc nuw i32 %69 to i8 | |
| %71 = getelementptr i8, ptr %40, i32 %46 | |
| store i8 %70, ptr %71, align 1 | |
| %72 = add nuw nsw i32 %46, 1 | |
| %73 = icmp eq i32 %72, %3 | |
| br i1 %73, label %42, label %45 | |
| 74: ; preds = %45, %74 | |
| %75 = phi i32 [ %87, %74 ], [ 0, %45 ] | |
| %76 = phi i32 [ %86, %74 ], [ %31, %45 ] | |
| %77 = getelementptr inbounds nuw [2 x i8], ptr %26, i32 %75 | |
| %78 = load i16, ptr %77, align 2 | |
| %79 = sext i16 %78 to i32 | |
| %80 = add nsw i32 %38, %75 | |
| %81 = mul nsw i32 %80, %3 | |
| %82 = getelementptr [2 x i8], ptr %49, i32 %81 | |
| %83 = load i16, ptr %82, align 2 | |
| %84 = sext i16 %83 to i32 | |
| %85 = mul nsw i32 %84, %79 | |
| %86 = add nsw i32 %85, %76 | |
| %87 = add nuw nsw i32 %75, 1 | |
| %88 = icmp eq i32 %87, %48 | |
| br i1 %88, label %51, label %74 | |
| } | |
| declare i32 @llvm.umin.i32(i32, i32) | |
| declare i32 @llvm.smax.i32(i32, i32) |