| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --replace-value-regex "!llvm.loop ![0-9]+" --version 5 |
| ; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s | FileCheck --check-prefixes=CHECK,VEC %s |
| ; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=1 < %s | FileCheck --check-prefixes=CHECK,INTERLEAVE %s |
| |
| define i32 @postinc(i32 %k) { |
| ; CHECK-LABEL: define i32 @postinc( |
| ; CHECK-SAME: i32 [[K:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret i32 [[INC_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc = add nsw i32 %inc.phi, 1 |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc |
| } |
| |
| define i32 @preinc(i32 %k) { |
| ; CHECK-LABEL: define i32 @preinc( |
| ; CHECK-SAME: i32 [[K:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[N_VEC]], 1 |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret i32 [[INC_PHI_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc = add nsw i32 %inc.phi, 1 |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc.phi |
| } |
| |
| define i32 @constpre() { |
| ; CHECK-LABEL: define i32 @constpre() { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16 |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: br i1 true, label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[MIDDLE_BLOCK]] ], [ 32, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = sub nsw i32 [[INC_PHI]], 2 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 0 |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_PHI_LCSSA:%.*]] = phi i32 [ [[INC_PHI]], %[[FOR_BODY]] ], [ 2, %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret i32 [[INC_PHI_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 32, %entry ], [ %inc, %for.body ] |
| %inc = sub nsw i32 %inc.phi, 2 |
| %cmp = icmp eq i32 %inc, 0 |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc.phi |
| } |
| |
| define ptr @geppre(ptr %ptr) { |
| ; CHECK-LABEL: define ptr @geppre( |
| ; CHECK-SAME: ptr [[PTR:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PTR]], i64 512 |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[IND_END]], i64 -16 |
| ; CHECK-NEXT: br i1 true, label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[PTR]], %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[PTR_PHI:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INC_PTR:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[INC_PTR]] = getelementptr i32, ptr [[PTR_PHI]], i32 4 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 32 |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[PTR_PHI_LCSSA:%.*]] = phi ptr [ [[PTR_PHI]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret ptr [[PTR_PHI_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %ptr.phi = phi ptr [ %ptr, %entry ], [ %inc.ptr, %for.body ] |
| %inc = add nsw i32 %inc.phi, 1 |
| %inc.ptr = getelementptr i32, ptr %ptr.phi, i32 4 |
| %cmp = icmp eq i32 %inc, 32 |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret ptr %ptr.phi |
| } |
| |
| define ptr @both(i32 %k) { |
| ; CHECK-LABEL: define ptr @both( |
| ; CHECK-SAME: i32 [[K:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[BASE:%.*]] = getelementptr inbounds i32, ptr undef, i64 1 |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[K]], -1 |
| ; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 |
| ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i32 |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 4 |
| ; CHECK-NEXT: [[IND_END1:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP3]] |
| ; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 4 |
| ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr undef, i64 [[TMP4]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] |
| ; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[IND_END1]], i64 -4 |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END1]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ] |
| ; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END2]], %[[MIDDLE_BLOCK]] ], [ undef, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC_LAG1:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[TMP:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC_LAG2:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[INC_LAG1]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[TMP]] = getelementptr inbounds i32, ptr [[INC_LAG1]], i64 1 |
| ; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_LAG1_LCSSA:%.*]] = phi ptr [ [[INC_LAG1]], %[[FOR_BODY]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret ptr [[INC_LAG1_LCSSA]] |
| ; |
| entry: |
| %base = getelementptr inbounds i32, ptr undef, i64 1 |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc.lag1 = phi ptr [ %base, %entry ], [ %tmp, %for.body] |
| %inc.lag2 = phi ptr [ undef, %entry ], [ %inc.lag1, %for.body] |
| %tmp = getelementptr inbounds i32, ptr %inc.lag1, i64 1 |
| %inc = add nsw i32 %inc.phi, 1 |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret ptr %inc.lag1 |
| } |
| |
| define i32 @multiphi(i32 %k, ptr %p) { |
| ; CHECK-LABEL: define i32 @multiphi( |
| ; CHECK-SAME: i32 [[K:%.*]], ptr [[P:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: [[PHI2:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: store i32 [[PHI2]], ptr [[P]], align 4 |
| ; CHECK-NEXT: ret i32 [[PHI]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc = add nsw i32 %inc.phi, 1 |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| %phi = phi i32 [ %inc, %for.body ] |
| %phi2 = phi i32 [ %inc, %for.body ] |
| store i32 %phi2, ptr %p |
| ret i32 %phi |
| } |
| |
| define void @PR30742() { |
| ; CHECK-LABEL: define void @PR30742() { |
| ; CHECK-NEXT: [[BB0:.*:]] |
| ; CHECK-NEXT: br label %[[BB1:.*]] |
| ; CHECK: [[BB1_LOOPEXIT:.*]]: |
| ; CHECK-NEXT: br label %[[BB1]] |
| ; CHECK: [[BB1]]: |
| ; CHECK-NEXT: [[TMP00:%.*]] = load i32, ptr undef, align 16 |
| ; CHECK-NEXT: [[TMP01:%.*]] = sub i32 [[TMP00]], undef |
| ; CHECK-NEXT: [[TMP02:%.*]] = icmp slt i32 [[TMP01]], 1 |
| ; CHECK-NEXT: [[TMP03:%.*]] = select i1 [[TMP02]], i32 1, i32 [[TMP01]] |
| ; CHECK-NEXT: [[TMP04:%.*]] = add nsw i32 [[TMP03]], -7 |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[TMP03]], -8 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[TMP03]], -15 |
| ; CHECK-NEXT: [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP1]], i32 0) |
| ; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[SMIN1]] |
| ; CHECK-NEXT: [[TMP3:%.*]] = lshr i32 [[TMP2]], 3 |
| ; CHECK-NEXT: [[TMP4:%.*]] = add nuw nsw i32 [[TMP3]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK4:%.*]] = icmp ult i32 [[TMP4]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK4]], label %[[SCALAR_PH3:.*]], label %[[VECTOR_PH5:.*]] |
| ; CHECK: [[VECTOR_PH5]]: |
| ; CHECK-NEXT: [[N_MOD_VF6:%.*]] = urem i32 [[TMP4]], 2 |
| ; CHECK-NEXT: [[N_VEC7:%.*]] = sub i32 [[TMP4]], [[N_MOD_VF6]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[N_VEC7]], -8 |
| ; CHECK-NEXT: [[IND_END8:%.*]] = add i32 [[TMP04]], [[TMP5]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY10:.*]] |
| ; CHECK: [[VECTOR_BODY10]]: |
| ; CHECK-NEXT: [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH5]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY10]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 2 |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC7]] |
| ; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK2:.*]], label %[[VECTOR_BODY10]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK2]]: |
| ; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i32 [[TMP4]], [[N_VEC7]] |
| ; CHECK-NEXT: [[IND_ESCAPE:%.*]] = sub i32 [[IND_END8]], -8 |
| ; CHECK-NEXT: br i1 [[CMP_N12]], label %[[BB3:.*]], label %[[SCALAR_PH3]] |
| ; CHECK: [[SCALAR_PH3]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL9:%.*]] = phi i32 [ [[IND_END8]], %[[MIDDLE_BLOCK2]] ], [ [[TMP04]], %[[BB1]] ] |
| ; CHECK-NEXT: br label %[[BB2:.*]] |
| ; CHECK: [[BB2]]: |
| ; CHECK-NEXT: [[TMP05:%.*]] = phi i32 [ [[BC_RESUME_VAL9]], %[[SCALAR_PH3]] ], [ [[TMP06:%.*]], %[[BB2]] ] |
| ; CHECK-NEXT: [[TMP06]] = add i32 [[TMP05]], -8 |
| ; CHECK-NEXT: [[TMP07:%.*]] = icmp sgt i32 [[TMP06]], 0 |
| ; CHECK-NEXT: br i1 [[TMP07]], label %[[BB2]], label %[[BB3]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[BB3]]: |
| ; CHECK-NEXT: [[TMP08:%.*]] = phi i32 [ [[TMP05]], %[[BB2]] ], [ [[IND_ESCAPE]], %[[MIDDLE_BLOCK2]] ] |
| ; CHECK-NEXT: [[TMP09:%.*]] = sub i32 [[TMP00]], undef |
| ; CHECK-NEXT: [[TMP10:%.*]] = icmp slt i32 [[TMP09]], 1 |
| ; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], i32 1, i32 [[TMP09]] |
| ; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], -7 |
| ; CHECK-NEXT: [[TMP7:%.*]] = add i32 [[TMP11]], -8 |
| ; CHECK-NEXT: [[TMP8:%.*]] = add nsw i32 [[TMP11]], -15 |
| ; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP8]], i32 0) |
| ; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP7]], [[SMIN]] |
| ; CHECK-NEXT: [[TMP10:%.*]] = lshr i32 [[TMP9]], 3 |
| ; CHECK-NEXT: [[TMP11:%.*]] = add nuw nsw i32 [[TMP10]], 1 |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP11]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP11]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP11]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[N_VEC]], -8 |
| ; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[TMP12]], [[TMP16]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP11]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[BB1_LOOPEXIT]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[BB3]] ] |
| ; CHECK-NEXT: br label %[[BB4:.*]] |
| ; CHECK: [[BB4]]: |
| ; CHECK-NEXT: [[TMP13:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[BB4]] ] |
| ; CHECK-NEXT: [[TMP14]] = add i32 [[TMP13]], -8 |
| ; CHECK-NEXT: [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], 0 |
| ; CHECK-NEXT: br i1 [[TMP15]], label %[[BB4]], label %[[BB1_LOOPEXIT]], {{!llvm.loop ![0-9]+}} |
| ; |
| BB0: |
| br label %BB1 |
| |
| BB1: |
| %tmp00 = load i32, ptr undef, align 16 |
| %tmp01 = sub i32 %tmp00, undef |
| %tmp02 = icmp slt i32 %tmp01, 1 |
| %tmp03 = select i1 %tmp02, i32 1, i32 %tmp01 |
| %tmp04 = add nsw i32 %tmp03, -7 |
| br label %BB2 |
| |
| BB2: |
| %tmp05 = phi i32 [ %tmp04, %BB1 ], [ %tmp06, %BB2 ] |
| %tmp06 = add i32 %tmp05, -8 |
| %tmp07 = icmp sgt i32 %tmp06, 0 |
| br i1 %tmp07, label %BB2, label %BB3 |
| |
| BB3: |
| %tmp08 = phi i32 [ %tmp05, %BB2 ] |
| %tmp09 = sub i32 %tmp00, undef |
| %tmp10 = icmp slt i32 %tmp09, 1 |
| %tmp11 = select i1 %tmp10, i32 1, i32 %tmp09 |
| %tmp11.inc = add nsw i32 %tmp11, -7 |
| br label %BB4 |
| |
| BB4: |
| %tmp13 = phi i32 [ %tmp11.inc, %BB3 ], [ %tmp14, %BB4 ] |
| %tmp14 = add i32 %tmp13, -8 |
| %tmp15 = icmp sgt i32 %tmp14, 0 |
| br i1 %tmp15, label %BB4, label %BB1 |
| } |
| |
| ; |
| define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) { |
| ; VEC-LABEL: define i64 @iv_scalar_steps_and_outside_users( |
| ; VEC-SAME: ptr [[PTR:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 |
| ; VEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP0]] |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0 |
| ; VEC-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP2]], align 4 |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2) |
| ; VEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002 |
| ; VEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1002, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 |
| ; VEC-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]] |
| ; VEC-NEXT: store i64 [[IV]], ptr [[GEP_PTR]], align 4 |
| ; VEC-NEXT: [[EXITCOND:%.*]] = icmp ugt i64 [[IV]], 1000 |
| ; VEC-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ], [ 1001, %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i64 [[IV_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define i64 @iv_scalar_steps_and_outside_users( |
| ; INTERLEAVE-SAME: ptr [[PTR:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP0]] |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP1]] |
| ; INTERLEAVE-NEXT: store i64 [[TMP0]], ptr [[TMP2]], align 4 |
| ; INTERLEAVE-NEXT: store i64 [[TMP1]], ptr [[TMP3]], align 4 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002 |
| ; INTERLEAVE-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1002, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 |
| ; INTERLEAVE-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]] |
| ; INTERLEAVE-NEXT: store i64 [[IV]], ptr [[GEP_PTR]], align 4 |
| ; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp ugt i64 [[IV]], 1000 |
| ; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ], [ 1001, %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i64 [[IV_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %iv.next = add nuw i64 %iv, 1 |
| %gep.ptr = getelementptr inbounds i64, ptr %ptr, i64 %iv |
| store i64 %iv, ptr %gep.ptr |
| %exitcond = icmp ugt i64 %iv, 1000 |
| br i1 %exitcond, label %exit, label %loop |
| |
| exit: |
| %iv.lcssa = phi i64 [ %iv, %loop ] |
| ret i64 %iv.lcssa |
| } |
| |
| |
| ; %iv.2 is dead in the vector loop and only used outside the loop. |
| define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) { |
| ; VEC-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside( |
| ; VEC-SAME: ptr [[PTR:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 |
| ; VEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP0]] |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0 |
| ; VEC-NEXT: store <2 x i64> [[VEC_IND]], ptr [[TMP2]], align 4 |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2) |
| ; VEC-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002 |
| ; VEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1002, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 2004, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 |
| ; VEC-NEXT: [[IV_2_NEXT]] = add nuw i32 [[IV_2]], 2 |
| ; VEC-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]] |
| ; VEC-NEXT: store i64 [[IV]], ptr [[GEP_PTR]], align 4 |
| ; VEC-NEXT: [[EXITCOND:%.*]] = icmp ugt i64 [[IV]], 1000 |
| ; VEC-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP]] ], [ 2002, %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i32 [[IV_2_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside( |
| ; INTERLEAVE-SAME: ptr [[PTR:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP0]] |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[TMP1]] |
| ; INTERLEAVE-NEXT: store i64 [[TMP0]], ptr [[TMP2]], align 4 |
| ; INTERLEAVE-NEXT: store i64 [[TMP1]], ptr [[TMP3]], align 4 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1002 |
| ; INTERLEAVE-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1002, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 2004, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 |
| ; INTERLEAVE-NEXT: [[IV_2_NEXT]] = add nuw i32 [[IV_2]], 2 |
| ; INTERLEAVE-NEXT: [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]] |
| ; INTERLEAVE-NEXT: store i64 [[IV]], ptr [[GEP_PTR]], align 4 |
| ; INTERLEAVE-NEXT: [[EXITCOND:%.*]] = icmp ugt i64 [[IV]], 1000 |
| ; INTERLEAVE-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP]] ], [ 2002, %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i32 [[IV_2_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ] |
| %iv.next = add nuw i64 %iv, 1 |
| %iv.2.next = add nuw i32 %iv.2, 2 |
| %gep.ptr = getelementptr inbounds i64, ptr %ptr, i64 %iv |
| store i64 %iv, ptr %gep.ptr |
| %exitcond = icmp ugt i64 %iv, 1000 |
| br i1 %exitcond, label %exit, label %loop |
| |
| exit: |
| %iv.2.lcssa = phi i32 [ %iv.2, %loop ] |
| ret i32 %iv.2.lcssa |
| } |
| |
| define i32 @postinc_sub(i32 %k) { |
| ; CHECK-LABEL: define i32 @postinc_sub( |
| ; CHECK-SAME: i32 [[K:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP0]], %[[MIDDLE_BLOCK]] ], [ [[K]], %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = sub nsw i32 [[INC_PHI]], 1 |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 0 |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret i32 [[INC_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ %k, %entry ], [ %inc, %for.body ] |
| %inc = sub nsw i32 %inc.phi, 1 |
| %cmp = icmp eq i32 %inc, 0 |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc |
| } |
| |
| define i32 @postinc_swapped_ops(i32 %k) { |
| ; CHECK-LABEL: define i32 @postinc_swapped_ops( |
| ; CHECK-SAME: i32 [[K:%.*]]) { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; CHECK: [[VECTOR_PH]]: |
| ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; CHECK: [[VECTOR_BODY]]: |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; CHECK-NEXT: [[TMP0:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[TMP0]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[MIDDLE_BLOCK]]: |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; CHECK: [[SCALAR_PH]]: |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; CHECK-NEXT: br label %[[FOR_BODY:.*]] |
| ; CHECK: [[FOR_BODY]]: |
| ; CHECK-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; CHECK-NEXT: [[INC]] = add nsw i32 1, [[INC_PHI]] |
| ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; CHECK: [[FOR_END]]: |
| ; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY]] ], [ [[N_VEC]], %[[MIDDLE_BLOCK]] ] |
| ; CHECK-NEXT: ret i32 [[INC_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc = add nsw i32 1, %inc.phi |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc |
| } |
| |
| define i32 @postinc_not_iv_backedge_value(i32 %k) { |
| ; VEC-LABEL: define i32 @postinc_not_iv_backedge_value( |
| ; VEC-SAME: i32 [[K:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; VEC-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP0:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2) |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2) |
| ; VEC-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[TMP0]], i32 1 |
| ; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[FOR_BODY:.*]] |
| ; VEC: [[FOR_BODY]]: |
| ; VEC-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; VEC-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; VEC-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2 |
| ; VEC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; VEC-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[FOR_END]]: |
| ; VEC-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i32 [[INC_2_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define i32 @postinc_not_iv_backedge_value( |
| ; INTERLEAVE-SAME: i32 [[K:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 2 |
| ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[K]], 2 |
| ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub i32 [[K]], [[N_MOD_VF]] |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[K]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[FOR_BODY:.*]] |
| ; INTERLEAVE: [[FOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[INC]] = add nsw i32 [[INC_PHI]], 1 |
| ; INTERLEAVE-NEXT: [[INC_2:%.*]] = add i32 [[INC_PHI]], 2 |
| ; INTERLEAVE-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], [[K]] |
| ; INTERLEAVE-NEXT: br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[FOR_END]]: |
| ; INTERLEAVE-NEXT: [[INC_2_LCSSA:%.*]] = phi i32 [ [[INC_2]], %[[FOR_BODY]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i32 [[INC_2_LCSSA]] |
| ; |
| entry: |
| br label %for.body |
| |
| for.body: |
| %inc.phi = phi i32 [ 0, %entry ], [ %inc, %for.body ] |
| %inc = add nsw i32 %inc.phi, 1 |
| %inc.2 = add i32 %inc.phi, 2 |
| %cmp = icmp eq i32 %inc, %k |
| br i1 %cmp, label %for.end, label %for.body |
| |
| for.end: |
| ret i32 %inc.2 |
| } |
| |
| define float @fp_postinc_use_fadd(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) { |
| ; VEC-LABEL: define float @fp_postinc_use_fadd( |
| ; VEC-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 |
| ; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 |
| ; VEC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; VEC-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float |
| ; VEC-NEXT: [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]] |
| ; VEC-NEXT: [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]] |
| ; VEC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[DOTSPLAT2]] |
| ; VEC-NEXT: [[INDUCTION:%.*]] = fadd fast <2 x float> [[DOTSPLAT]], [[TMP2]] |
| ; VEC-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00 |
| ; VEC-NEXT: [[DOTSPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT4:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 |
| ; VEC-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] |
| ; VEC-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 |
| ; VEC-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP6]], align 4 |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = fadd fast <2 x float> [[VEC_IND]], [[DOTSPLAT4]] |
| ; VEC-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] |
| ; VEC-NEXT: store float [[FP_IV]], ptr [[GEP_A]], align 4 |
| ; VEC-NEXT: [[ADD]] = fadd fast float [[FP_IV]], [[FPINC]] |
| ; VEC-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; VEC-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; VEC-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret float [[ADD_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define float @fp_postinc_use_fadd( |
| ; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 |
| ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 |
| ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]] |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = fadd fast float [[INIT]], [[TMP0]] |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]] |
| ; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = fadd fast float [[INIT]], [[TMP4]] |
| ; INTERLEAVE-NEXT: [[TMP5:%.*]] = fmul fast float 0.000000e+00, [[FPINC]] |
| ; INTERLEAVE-NEXT: [[TMP6:%.*]] = fadd fast float [[OFFSET_IDX]], [[TMP5]] |
| ; INTERLEAVE-NEXT: [[TMP7:%.*]] = fmul fast float 1.000000e+00, [[FPINC]] |
| ; INTERLEAVE-NEXT: [[TMP8:%.*]] = fadd fast float [[OFFSET_IDX]], [[TMP7]] |
| ; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP2]] |
| ; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]] |
| ; INTERLEAVE-NEXT: store float [[TMP6]], ptr [[TMP9]], align 4 |
| ; INTERLEAVE-NEXT: store float [[TMP8]], ptr [[TMP10]], align 4 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] |
| ; INTERLEAVE-NEXT: store float [[FP_IV]], ptr [[GEP_A]], align 4 |
| ; INTERLEAVE-NEXT: [[ADD]] = fadd fast float [[FP_IV]], [[FPINC]] |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; INTERLEAVE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; INTERLEAVE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ %init, %entry ], [ %add, %loop ] |
| %gep.A = getelementptr inbounds float, ptr %A, i64 %iv |
| store float %fp.iv, ptr %gep.A, align 4 |
| %add = fadd fast float %fp.iv, %fpinc |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %N |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret float %add |
| } |
| |
| define float @fp_postinc_use_fsub(float %init, ptr noalias nocapture %A, i64 %N, float %fpinc) { |
| ; VEC-LABEL: define float @fp_postinc_use_fsub( |
| ; VEC-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 |
| ; VEC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 |
| ; VEC-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; VEC-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float |
| ; VEC-NEXT: [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]] |
| ; VEC-NEXT: [[TMP1:%.*]] = fsub fast float [[INIT]], [[TMP0]] |
| ; VEC-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[INIT]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <2 x float> poison, float [[FPINC]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT1]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> <float 0.000000e+00, float 1.000000e+00>, [[DOTSPLAT2]] |
| ; VEC-NEXT: [[INDUCTION:%.*]] = fsub fast <2 x float> [[DOTSPLAT]], [[TMP2]] |
| ; VEC-NEXT: [[TMP3:%.*]] = fmul fast float [[FPINC]], 2.000000e+00 |
| ; VEC-NEXT: [[DOTSPLATINSERT3:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i64 0 |
| ; VEC-NEXT: [[DOTSPLAT4:%.*]] = shufflevector <2 x float> [[DOTSPLATINSERT3]], <2 x float> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x float> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 |
| ; VEC-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP4]] |
| ; VEC-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 |
| ; VEC-NEXT: store <2 x float> [[VEC_IND]], ptr [[TMP6]], align 4 |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = fsub fast <2 x float> [[VEC_IND]], [[DOTSPLAT4]] |
| ; VEC-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; VEC-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: [[BC_RESUME_VAL5:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] |
| ; VEC-NEXT: store float [[FP_IV]], ptr [[GEP_A]], align 4 |
| ; VEC-NEXT: [[ADD]] = fsub fast float [[FP_IV]], [[FPINC]] |
| ; VEC-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; VEC-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; VEC-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret float [[ADD_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define float @fp_postinc_use_fsub( |
| ; INTERLEAVE-SAME: float [[INIT:%.*]], ptr noalias nocapture [[A:%.*]], i64 [[N:%.*]], float [[FPINC:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2 |
| ; INTERLEAVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2 |
| ; INTERLEAVE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] |
| ; INTERLEAVE-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = fmul fast float [[FPINC]], [[DOTCAST]] |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = fsub fast float [[INIT]], [[TMP0]] |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[DOTCAST1:%.*]] = sitofp i64 [[INDEX]] to float |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = fmul fast float [[FPINC]], [[DOTCAST1]] |
| ; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = fsub fast float [[INIT]], [[TMP4]] |
| ; INTERLEAVE-NEXT: [[TMP5:%.*]] = fmul fast float 0.000000e+00, [[FPINC]] |
| ; INTERLEAVE-NEXT: [[TMP6:%.*]] = fsub fast float [[OFFSET_IDX]], [[TMP5]] |
| ; INTERLEAVE-NEXT: [[TMP7:%.*]] = fmul fast float 1.000000e+00, [[FPINC]] |
| ; INTERLEAVE-NEXT: [[TMP8:%.*]] = fsub fast float [[OFFSET_IDX]], [[TMP7]] |
| ; INTERLEAVE-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP2]] |
| ; INTERLEAVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP3]] |
| ; INTERLEAVE-NEXT: store float [[TMP6]], ptr [[TMP9]], align 4 |
| ; INTERLEAVE-NEXT: store float [[TMP8]], ptr [[TMP10]], align 4 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] |
| ; INTERLEAVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL2:%.*]] = phi float [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[INIT]], %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[FP_IV:%.*]] = phi float [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] |
| ; INTERLEAVE-NEXT: store float [[FP_IV]], ptr [[GEP_A]], align 4 |
| ; INTERLEAVE-NEXT: [[ADD]] = fsub fast float [[FP_IV]], [[FPINC]] |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 |
| ; INTERLEAVE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] |
| ; INTERLEAVE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[LOOP]] ], [ [[TMP1]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret float [[ADD_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] |
| %fp.iv = phi float [ %init, %entry ], [ %add, %loop ] |
| %gep.A = getelementptr inbounds float, ptr %A, i64 %iv |
| store float %fp.iv, ptr %gep.A, align 4 |
| %add = fsub fast float %fp.iv, %fpinc |
| %iv.next = add nuw nsw i64 %iv, 1 |
| %ec = icmp eq i64 %iv.next, %N |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret float %add |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/121745. |
| define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) { |
| ; VEC-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification( |
| ; VEC-SAME: ptr [[DST:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32 |
| ; VEC-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1 |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 |
| ; VEC-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 1 |
| ; VEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]] |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0 |
| ; VEC-NEXT: store <2 x i16> zeroinitializer, ptr [[TMP2]], align 2 |
| ; VEC-NEXT: [[TMP4:%.*]] = add i32 [[STEP_2]], [[TMP0]] |
| ; VEC-NEXT: [[TMP5:%.*]] = add i32 [[STEP_2]], [[TMP6]] |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; VEC-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8 |
| ; VEC-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: br i1 true, label %[[E_EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]] |
| ; VEC-NEXT: store i16 0, ptr [[GEP_DST]], align 2 |
| ; VEC-NEXT: [[IV_NEXT]] = add i32 [[STEP_2]], [[IV]] |
| ; VEC-NEXT: [[CMP_I:%.*]] = icmp slt i32 [[IV_NEXT]], 8 |
| ; VEC-NEXT: br i1 [[CMP_I]], label %[[LOOP]], label %[[E_EXIT]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[E_EXIT]]: |
| ; VEC-NEXT: [[RES:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i32 [[RES]] |
| ; |
| ; INTERLEAVE-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification( |
| ; INTERLEAVE-SAME: ptr [[DST:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32 |
| ; INTERLEAVE-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1 |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]] |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]] |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[TMP2]], align 2 |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[TMP3]], align 2 |
| ; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i32 [[STEP_2]], [[TMP1]] |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8 |
| ; INTERLEAVE-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 true, label %[[E_EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]] |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[GEP_DST]], align 2 |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add i32 [[STEP_2]], [[IV]] |
| ; INTERLEAVE-NEXT: [[CMP_I:%.*]] = icmp slt i32 [[IV_NEXT]], 8 |
| ; INTERLEAVE-NEXT: br i1 [[CMP_I]], label %[[LOOP]], label %[[E_EXIT]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[E_EXIT]]: |
| ; INTERLEAVE-NEXT: [[RES:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i32 [[RES]] |
| ; |
| entry: |
| %step.1 = sext i8 0 to i32 |
| %step.2 = add nsw i32 %step.1, 1 |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] |
| %gep.dst = getelementptr inbounds i16, ptr %dst, i32 %iv |
| store i16 0, ptr %gep.dst, align 2 |
| %iv.next = add i32 %step.2, %iv |
| %cmp.i = icmp slt i32 %iv.next, 8 |
| br i1 %cmp.i, label %loop, label %e.exit |
| |
| e.exit: |
| %res = phi i32 [ %iv.next, %loop ] |
| ret i32 %res |
| } |
| |
| define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) { |
| ; VEC-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2( |
| ; VEC-SAME: ptr [[DST:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32 |
| ; VEC-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1 |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[STEP_2]], i64 0 |
| ; VEC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], 2 |
| ; VEC-NEXT: [[TMP0:%.*]] = add i32 [[OFFSET_IDX]], 0 |
| ; VEC-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 2 |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]] |
| ; VEC-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]] |
| ; VEC-NEXT: store i16 0, ptr [[TMP2]], align 2 |
| ; VEC-NEXT: store i16 0, ptr [[TMP3]], align 2 |
| ; VEC-NEXT: [[TMP4:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 1) |
| ; VEC-NEXT: [[TMP5:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], [[TMP4]] |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 4) |
| ; VEC-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4 |
| ; VEC-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1 |
| ; VEC-NEXT: br i1 true, label %[[E_EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]] |
| ; VEC-NEXT: store i16 0, ptr [[GEP_DST]], align 2 |
| ; VEC-NEXT: [[INC:%.*]] = add i32 [[IV]], 1 |
| ; VEC-NEXT: [[IV_NEXT]] = add i32 [[STEP_2]], [[INC]] |
| ; VEC-NEXT: [[CMP_I:%.*]] = icmp slt i32 [[IV_NEXT]], 8 |
| ; VEC-NEXT: br i1 [[CMP_I]], label %[[LOOP]], label %[[E_EXIT]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[E_EXIT]]: |
| ; VEC-NEXT: [[RES:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i32 [[RES]] |
| ; |
| ; INTERLEAVE-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2( |
| ; INTERLEAVE-SAME: ptr [[DST:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: [[STEP_1:%.*]] = sext i8 0 to i32 |
| ; INTERLEAVE-NEXT: [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1 |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = mul i32 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i32 [[OFFSET_IDX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]] |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP1]] |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[TMP2]], align 2 |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[TMP3]], align 2 |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = add i32 [[TMP1]], 1 |
| ; INTERLEAVE-NEXT: [[TMP5:%.*]] = add i32 [[STEP_2]], [[TMP4]] |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4 |
| ; INTERLEAVE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 true, label %[[E_EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]] |
| ; INTERLEAVE-NEXT: store i16 0, ptr [[GEP_DST]], align 2 |
| ; INTERLEAVE-NEXT: [[INC:%.*]] = add i32 [[IV]], 1 |
| ; INTERLEAVE-NEXT: [[IV_NEXT]] = add i32 [[STEP_2]], [[INC]] |
| ; INTERLEAVE-NEXT: [[CMP_I:%.*]] = icmp slt i32 [[IV_NEXT]], 8 |
| ; INTERLEAVE-NEXT: br i1 [[CMP_I]], label %[[LOOP]], label %[[E_EXIT]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[E_EXIT]]: |
| ; INTERLEAVE-NEXT: [[RES:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i32 [[RES]] |
| ; |
| entry: |
| %step.1 = sext i8 0 to i32 |
| %step.2 = add nsw i32 %step.1, 1 |
| br label %loop |
| |
| loop: |
| %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] |
| %gep.dst = getelementptr inbounds i16, ptr %dst, i32 %iv |
| store i16 0, ptr %gep.dst, align 2 |
| %inc = add i32 %iv, 1 |
| %iv.next = add i32 %step.2, %inc |
| %cmp.i = icmp slt i32 %iv.next, 8 |
| br i1 %cmp.i, label %loop, label %e.exit |
| |
| e.exit: |
| %res = phi i32 [ %iv.next, %loop ] |
| ret i32 %res |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/122496. |
| define i32 @iv_ext_used_outside( ptr %dst) { |
| ; VEC-LABEL: define i32 @iv_ext_used_outside( |
| ; VEC-SAME: ptr [[DST:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; VEC-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 |
| ; VEC-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 |
| ; VEC-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP0]] |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i32 0 |
| ; VEC-NEXT: store <2 x i32> zeroinitializer, ptr [[TMP2]], align 4 |
| ; VEC-NEXT: [[TMP5:%.*]] = add nuw nsw <2 x i16> [[VEC_IND]], splat (i16 1) |
| ; VEC-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP5]], i32 0 |
| ; VEC-NEXT: [[TMP4:%.*]] = zext nneg i16 [[TMP3]] to i32 |
| ; VEC-NEXT: [[TMP8:%.*]] = extractelement <2 x i16> [[TMP5]], i32 1 |
| ; VEC-NEXT: [[TMP7:%.*]] = zext nneg i16 [[TMP8]] to i32 |
| ; VEC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; VEC-NEXT: [[VEC_IND_NEXT]] = add <2 x i16> [[VEC_IND]], splat (i16 2) |
| ; VEC-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 |
| ; VEC-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV_1:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_1_EXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[IV_1]] |
| ; VEC-NEXT: store i32 0, ptr [[GEP]], align 4 |
| ; VEC-NEXT: [[IV_1_NEXT]] = add nuw nsw i16 [[IV_1]], 1 |
| ; VEC-NEXT: [[IV_1_EXT]] = zext nneg i16 [[IV_1_NEXT]] to i32 |
| ; VEC-NEXT: [[EC:%.*]] = icmp samesign ult i16 [[IV_1]], 128 |
| ; VEC-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[IV_1_EXT_LCSSA:%.*]] = phi i32 [ [[IV_1_EXT]], %[[LOOP]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i32 [[IV_1_EXT_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define i32 @iv_ext_used_outside( |
| ; INTERLEAVE-SAME: ptr [[DST:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] |
| ; INTERLEAVE-NEXT: [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16 |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], 0 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 1 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP0]] |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[TMP1]] |
| ; INTERLEAVE-NEXT: store i32 0, ptr [[TMP2]], align 4 |
| ; INTERLEAVE-NEXT: store i32 0, ptr [[TMP3]], align 4 |
| ; INTERLEAVE-NEXT: [[TMP4:%.*]] = add nuw nsw i16 [[TMP1]], 1 |
| ; INTERLEAVE-NEXT: [[TMP5:%.*]] = zext nneg i16 [[TMP4]] to i32 |
| ; INTERLEAVE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 |
| ; INTERLEAVE-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128 |
| ; INTERLEAVE-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i16 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i32 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV_1:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_1_EXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i16 [[IV_1]] |
| ; INTERLEAVE-NEXT: store i32 0, ptr [[GEP]], align 4 |
| ; INTERLEAVE-NEXT: [[IV_1_NEXT]] = add nuw nsw i16 [[IV_1]], 1 |
| ; INTERLEAVE-NEXT: [[IV_1_EXT]] = zext nneg i16 [[IV_1_NEXT]] to i32 |
| ; INTERLEAVE-NEXT: [[EC:%.*]] = icmp samesign ult i16 [[IV_1]], 128 |
| ; INTERLEAVE-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[IV_1_EXT_LCSSA:%.*]] = phi i32 [ [[IV_1_EXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i32 [[IV_1_EXT_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv.1 = phi i16 [ 0, %entry ], [ %iv.1.next, %loop ] |
| %iv.2 = phi i32 [ 0, %entry ], [ %iv.1.ext, %loop ] |
| %gep = getelementptr inbounds nuw i32, ptr %dst, i16 %iv.1 |
| store i32 0, ptr %gep, align 4 |
| %iv.1.next = add nuw nsw i16 %iv.1, 1 |
| %iv.1.ext = zext nneg i16 %iv.1.next to i32 |
| %ec = icmp samesign ult i16 %iv.1, 128 |
| br i1 %ec, label %loop, label %exit |
| |
| exit: |
| %iv.1.ext.lcssa = phi i32 [ %iv.1.ext, %loop ] |
| ret i32 %iv.1.ext.lcssa |
| } |
| |
| ; Test case for https://github.com/llvm/llvm-project/issues/122602. |
| define i64 @test_iv_increment_incremented(ptr %dst) { |
| ; VEC-LABEL: define i64 @test_iv_increment_incremented( |
| ; VEC-SAME: ptr [[DST:%.*]]) { |
| ; VEC-NEXT: [[ENTRY:.*]]: |
| ; VEC-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; VEC: [[VECTOR_PH]]: |
| ; VEC-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; VEC: [[VECTOR_BODY]]: |
| ; VEC-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3 |
| ; VEC-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[TMP0]], i32 0 |
| ; VEC-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 -1 |
| ; VEC-NEXT: store <2 x i16> splat (i16 1), ptr [[TMP2]], align 2 |
| ; VEC-NEXT: [[TMP3:%.*]] = add i64 2, -1 |
| ; VEC-NEXT: [[TMP5:%.*]] = add i64 1, -1 |
| ; VEC-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], 1 |
| ; VEC-NEXT: [[TMP6:%.*]] = add i64 [[TMP5]], 1 |
| ; VEC-NEXT: br label %[[MIDDLE_BLOCK:.*]] |
| ; VEC: [[MIDDLE_BLOCK]]: |
| ; VEC-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; VEC: [[SCALAR_PH]]: |
| ; VEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ] |
| ; VEC-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ] |
| ; VEC-NEXT: br label %[[LOOP:.*]] |
| ; VEC: [[LOOP]]: |
| ; VEC-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ] |
| ; VEC-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV_1]] |
| ; VEC-NEXT: store i16 1, ptr [[GEP]], align 2 |
| ; VEC-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], -1 |
| ; VEC-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_2_NEXT]], 0 |
| ; VEC-NEXT: [[IV_1_NEXT]] = add i64 [[IV_2_NEXT]], 1 |
| ; VEC-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; VEC: [[EXIT]]: |
| ; VEC-NEXT: [[IV_1_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_1_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ] |
| ; VEC-NEXT: ret i64 [[IV_1_NEXT_LCSSA]] |
| ; |
| ; INTERLEAVE-LABEL: define i64 @test_iv_increment_incremented( |
| ; INTERLEAVE-SAME: ptr [[DST:%.*]]) { |
| ; INTERLEAVE-NEXT: [[ENTRY:.*]]: |
| ; INTERLEAVE-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] |
| ; INTERLEAVE: [[VECTOR_PH]]: |
| ; INTERLEAVE-NEXT: br label %[[VECTOR_BODY:.*]] |
| ; INTERLEAVE: [[VECTOR_BODY]]: |
| ; INTERLEAVE-NEXT: [[TMP0:%.*]] = getelementptr i16, ptr [[DST]], i64 3 |
| ; INTERLEAVE-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], i64 2 |
| ; INTERLEAVE-NEXT: store i16 1, ptr [[TMP0]], align 2 |
| ; INTERLEAVE-NEXT: store i16 1, ptr [[TMP1]], align 2 |
| ; INTERLEAVE-NEXT: [[TMP2:%.*]] = add i64 1, -1 |
| ; INTERLEAVE-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 1 |
| ; INTERLEAVE-NEXT: br label %[[MIDDLE_BLOCK:.*]] |
| ; INTERLEAVE: [[MIDDLE_BLOCK]]: |
| ; INTERLEAVE-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] |
| ; INTERLEAVE: [[SCALAR_PH]]: |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ] |
| ; INTERLEAVE-NEXT: br label %[[LOOP:.*]] |
| ; INTERLEAVE: [[LOOP]]: |
| ; INTERLEAVE-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ] |
| ; INTERLEAVE-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV_1]] |
| ; INTERLEAVE-NEXT: store i16 1, ptr [[GEP]], align 2 |
| ; INTERLEAVE-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], -1 |
| ; INTERLEAVE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_2_NEXT]], 0 |
| ; INTERLEAVE-NEXT: [[IV_1_NEXT]] = add i64 [[IV_2_NEXT]], 1 |
| ; INTERLEAVE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], {{!llvm.loop ![0-9]+}} |
| ; INTERLEAVE: [[EXIT]]: |
| ; INTERLEAVE-NEXT: [[IV_1_NEXT_LCSSA:%.*]] = phi i64 [ [[IV_1_NEXT]], %[[LOOP]] ], [ [[TMP3]], %[[MIDDLE_BLOCK]] ] |
| ; INTERLEAVE-NEXT: ret i64 [[IV_1_NEXT_LCSSA]] |
| ; |
| entry: |
| br label %loop |
| |
| loop: |
| %iv.1 = phi i64 [ 3, %entry ], [ %iv.1.next, %loop ] |
| %iv.2 = phi i64 [ 2, %entry ], [ %iv.2.next, %loop ] |
| %gep = getelementptr i16, ptr %dst, i64 %iv.1 |
| store i16 1, ptr %gep, align 2 |
| %iv.2.next = add i64 %iv.2, -1 |
| %ec = icmp eq i64 %iv.2.next, 0 |
| %iv.1.next = add i64 %iv.2.next, 1 |
| br i1 %ec, label %exit, label %loop |
| |
| exit: |
| ret i64 %iv.1.next |
| } |
