| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 | |
| ; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s | |
| ; Truncates of an induction plus a loop-invariant offset are narrowed by adding | |
| ; the offset to the start value of the narrowed induction. The offset does not | |
| ; have to be a constant or the induction step. | |
| ; trunc(iv + 1), where the offset is also the step. | |
| define void @trunc_update(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_update( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 1, i8 2, i8 3, i8 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP1]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV_NEXT]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %t = trunc i64 %iv.next to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + 5), where the offset differs from the step. | |
| define void @trunc_of_iv_plus_const(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_const( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 5, i8 6, i8 7, i8 8>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP1]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 5 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, 5 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv - 3). | |
| define void @trunc_of_iv_minus_const(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_minus_const( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 -3, i8 -2, i8 -1, i8 0>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP1]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = sub i64 [[IV]], 3 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = sub i64 %iv, 3 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + 7) with a step of 3. The address uses a separate unit-stride IV. | |
| define void @trunc_of_iv_plus_const_non_unit_step(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_const_non_unit_step( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N_VEC]], 3 | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 7, i8 10, i8 13, i8 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP2]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 12) | |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 3 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 7 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IDX]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[IDX_NEXT]] = add nuw i64 [[IDX]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IDX_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %idx = phi i64 [ 0, %entry ], [ %idx.next, %loop ] | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 3 | |
| %off = add i64 %iv, 7 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %idx | |
| store i8 %t, ptr %gep, align 1 | |
| %idx.next = add nuw i64 %idx, 1 | |
| %ec = icmp eq i64 %idx.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv - 7) with a step of -2. | |
| define void @trunc_of_iv_minus_const_negative_step(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_minus_const_negative_step( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N_VEC]], -2 | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add i64 1000, [[TMP1]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 -31, i8 -33, i8 -35, i8 -37>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP3]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 -8) | |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 1000, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], -2 | |
| ; CHECK-NEXT: [[OFF:%.*]] = sub i64 [[IV]], 7 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IDX]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[IDX_NEXT]] = add nuw i64 [[IDX]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IDX_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %idx = phi i64 [ 0, %entry ], [ %idx.next, %loop ] | |
| %iv = phi i64 [ 1000, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add i64 %iv, -2 | |
| %off = sub i64 %iv, 7 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %idx | |
| store i8 %t, ptr %gep, align 1 | |
| %idx.next = add nuw i64 %idx, 1 | |
| %ec = icmp eq i64 %idx.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + 10) where the offset makes the narrowed induction wrap. | |
| define void @trunc_of_iv_plus_const_wraps(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_const_wraps( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 250, [[N_VEC]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 4, i8 5, i8 6, i8 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP2]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 250, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 10 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IDX]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[IDX_NEXT]] = add nuw i64 [[IDX]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IDX_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %idx = phi i64 [ 0, %entry ], [ %idx.next, %loop ] | |
| %iv = phi i64 [ 250, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, 10 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %idx | |
| store i8 %t, ptr %gep, align 1 | |
| %idx.next = add nuw i64 %idx, 1 | |
| %ec = icmp eq i64 %idx.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv), without an offset, is narrowed as before. | |
| define void @trunc_of_iv(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP1]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP15:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %t = trunc i64 %iv to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + %off.v), where the offset is loop-invariant but not a constant. | |
| ; The start of the narrowed induction is computed in the vector preheader. | |
| define void @trunc_of_iv_plus_variable(ptr %dst, i64 %n, i64 %off.v) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_variable( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[OFF_V:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[OFF_V]] to i8 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP1]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3> | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP2]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], [[OFF_V]] | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP17:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, %off.v | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + 5), where the start value is not a constant. The start of the | |
| ; narrowed induction is computed in the vector preheader. | |
| define void @trunc_of_iv_plus_const_variable_start(ptr %dst, i64 %n, i64 %start) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_const_variable_start( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[START:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[START]], [[N_VEC]] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[START]], 5 | |
| ; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i8 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP3]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3> | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP4]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 5 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IDX]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[IDX_NEXT]] = add nuw i64 [[IDX]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IDX_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %idx = phi i64 [ 0, %entry ], [ %idx.next, %loop ] | |
| %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, 5 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %idx | |
| store i8 %t, ptr %gep, align 1 | |
| %idx.next = add nuw i64 %idx, 1 | |
| %ec = icmp eq i64 %idx.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; The offset applies to an already narrowed induction, whose type is narrower | |
| ; than its start value, so it is not folded. | |
| define void @trunc_of_narrowed_iv_plus_const(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_narrowed_iv_plus_const( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] | |
| ; CHECK: [[VECTOR_MEMCHECK]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[N]], 1 | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = shl i64 [[N]], 2 | |
| ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]] | |
| ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP3]] | |
| ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST]], [[TMP1]] | |
| ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] | |
| ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 5) | |
| ; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP5]] to <4 x i16> | |
| ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i16> [[TMP6]], ptr [[TMP7]], align 2, !alias.scope [[META20:![0-9]+]], !noalias [[META23:![0-9]+]] | |
| ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP8]], align 4, !alias.scope [[META23]] | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) | |
| ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV]] to i32 | |
| ; CHECK-NEXT: [[U:%.*]] = add i32 [[T]], 5 | |
| ; CHECK-NEXT: [[V:%.*]] = trunc i32 [[U]] to i16 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i16 [[V]], ptr [[GEP]], align 2 | |
| ; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i32 [[U]], ptr [[GEP2]], align 4 | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP26:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %t = trunc i64 %iv to i32 | |
| %u = add i32 %t, 5 | |
| %v = trunc i32 %u to i16 | |
| %gep = getelementptr inbounds i16, ptr %dst, i64 %iv | |
| store i16 %v, ptr %gep, align 2 | |
| %gep2 = getelementptr inbounds i32, ptr %dst, i64 %iv | |
| store i32 %u, ptr %gep2, align 4 | |
| %iv.next = add nuw i64 %iv, 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; Scalar users of the narrowed induction also start at the offset. | |
| define void @trunc_of_iv_plus_const_scalar_use(ptr %dst, ptr %dst2, ptr %src, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_const_scalar_use( | |
| ; CHECK-SAME: ptr [[DST:%.*]], ptr [[DST2:%.*]], ptr [[SRC:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] | |
| ; CHECK: [[VECTOR_SCEVCHECK]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -1 | |
| ; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32 | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add i32 5, [[TMP1]] | |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 5 | |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[TMP0]], 4294967295 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] | |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] | |
| ; CHECK: [[VECTOR_MEMCHECK]]: | |
| ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[N]], 2 | |
| ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]] | |
| ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[DST2]], i64 [[N]] | |
| ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 5 | |
| ; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[N]], 5 | |
| ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP10]] | |
| ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[TMP8]] | |
| ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[DST2]], [[TMP7]] | |
| ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] | |
| ; CHECK-NEXT: [[BOUND011:%.*]] = icmp ult ptr [[DST]], [[TMP11]] | |
| ; CHECK-NEXT: [[BOUND112:%.*]] = icmp ult ptr [[TMP9]], [[TMP7]] | |
| ; CHECK-NEXT: [[FOUND_CONFLICT13:%.*]] = and i1 [[BOUND011]], [[BOUND112]] | |
| ; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT13]] | |
| ; CHECK-NEXT: [[BOUND014:%.*]] = icmp ult ptr [[DST2]], [[TMP11]] | |
| ; CHECK-NEXT: [[BOUND115:%.*]] = icmp ult ptr [[TMP9]], [[TMP8]] | |
| ; CHECK-NEXT: [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]] | |
| ; CHECK-NEXT: [[CONFLICT_RDX17:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT16]] | |
| ; CHECK-NEXT: br i1 [[CONFLICT_RDX17]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP12:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP12]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 5, i32 6, i32 7, i32 8>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP13:%.*]] = add i64 5, [[INDEX]] | |
| ; CHECK-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP13]] to i32 | |
| ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP15]], align 4, !alias.scope [[META27:![0-9]+]], !noalias [[META30:![0-9]+]] | |
| ; CHECK-NEXT: [[TMP16:%.*]] = zext i32 [[TMP14]] to i64 | |
| ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP16]] | |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1, !alias.scope [[META33:![0-9]+]] | |
| ; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[DST2]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[WIDE_LOAD]], ptr [[TMP18]], align 1, !alias.scope [[META34:![0-9]+]], !noalias [[META33]] | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) | |
| ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 5 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i32 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i32 [[T]], ptr [[GEP]], align 4 | |
| ; CHECK-NEXT: [[Z:%.*]] = zext i32 [[T]] to i64 | |
| ; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[Z]] | |
| ; CHECK-NEXT: [[V:%.*]] = load i8, ptr [[SRC_GEP]], align 1 | |
| ; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[DST2]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[V]], ptr [[GEP2]], align 1 | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP36:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %off = add i64 %iv, 5 | |
| %t = trunc i64 %off to i32 | |
| %gep = getelementptr inbounds i32, ptr %dst, i64 %iv | |
| store i32 %t, ptr %gep, align 4 | |
| %z = zext i32 %t to i64 | |
| %src.gep = getelementptr inbounds i8, ptr %src, i64 %z | |
| %v = load i8, ptr %src.gep, align 1 | |
| %gep2 = getelementptr inbounds i8, ptr %dst2, i64 %iv | |
| store i8 %v, ptr %gep2, align 1 | |
| %iv.next = add nuw i64 %iv, 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; trunc(iv + %step), the increment by a non-constant step. The narrowed | |
| ; induction must start one step ahead, at %step rather than 0. | |
| define void @trunc_of_iv_inc_variable_step(ptr %dst, i64 %n, i64 %step) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_inc_variable_step( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[STEP:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[N_VEC]], [[STEP]] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[STEP]] to i32 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]] | |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], [[TMP3]] | |
| ; CHECK-NEXT: [[TMP4:%.*]] = shl i32 [[TMP2]], 2 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP5]], align 4 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]] | |
| ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IDX_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], [[STEP]] | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[IV_NEXT]] to i32 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]] | |
| ; CHECK-NEXT: store i32 [[T]], ptr [[GEP]], align 4 | |
| ; CHECK-NEXT: [[IDX_NEXT]] = add nuw i64 [[IDX]], 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IDX_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP38:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %idx = phi i64 [ 0, %entry ], [ %idx.next, %loop ] | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add i64 %iv, %step | |
| %t = trunc i64 %iv.next to i32 | |
| %gep = getelementptr inbounds i32, ptr %dst, i64 %idx | |
| store i32 %t, ptr %gep, align 4 | |
| %idx.next = add nuw i64 %idx, 1 | |
| %ec = icmp eq i64 %idx.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; Negative test: the offset changes in the loop, so the IV plus the offset is | |
| ; not an induction. | |
| define void @trunc_of_iv_plus_loop_variant(ptr noalias %dst, ptr noalias %src, i64 %n) { | |
| ; CHECK-LABEL: define void @trunc_of_iv_plus_loop_variant( | |
| ; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]] | |
| ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8 | |
| ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i64> [[VEC_IND]], [[WIDE_LOAD]] | |
| ; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i64> [[TMP2]] to <4 x i8> | |
| ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4) | |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]] | |
| ; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[SRC_GEP]], align 8 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], [[V]] | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP40:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: ret void | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %src.gep = getelementptr inbounds i64, ptr %src, i64 %iv | |
| %v = load i64, ptr %src.gep, align 8 | |
| %off = add i64 %iv, %v | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| ret void | |
| } | |
| ; The narrowed induction is also used outside the loop. Its exit value must | |
| ; include the offset. | |
| define i8 @trunc_of_iv_plus_const_live_out(ptr %dst, i64 %n) { | |
| ; CHECK-LABEL: define i8 @trunc_of_iv_plus_const_live_out( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 5, i8 6, i8 7, i8 8>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP1]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[TMP3:%.*]] = sub nuw i64 [[N_VEC]], 1 | |
| ; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i8 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = add i8 5, [[TMP4]] | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], 5 | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP42:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: [[T_LCSSA:%.*]] = phi i8 [ [[T]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ] | |
| ; CHECK-NEXT: ret i8 [[T_LCSSA]] | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, 5 | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| %t.lcssa = phi i8 [ %t, %loop ] | |
| ret i8 %t.lcssa | |
| } | |
| ; Same with a loop-invariant offset. | |
| define i8 @trunc_of_iv_plus_variable_live_out(ptr %dst, i64 %n, i64 %off.v) { | |
| ; CHECK-LABEL: define i8 @trunc_of_iv_plus_variable_live_out( | |
| ; CHECK-SAME: ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[OFF_V:%.*]]) { | |
| ; CHECK-NEXT: [[ENTRY:.*]]: | |
| ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 | |
| ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] | |
| ; CHECK: [[VECTOR_PH]]: | |
| ; CHECK-NEXT: [[TMP0:%.*]] = and i64 [[N]], 3 | |
| ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP0]] | |
| ; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[OFF_V]] to i8 | |
| ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP1]], i64 0 | |
| ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer | |
| ; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i8> [[BROADCAST_SPLAT]], <i8 0, i8 1, i8 2, i8 3> | |
| ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] | |
| ; CHECK: [[VECTOR_BODY]]: | |
| ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] | |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]] | |
| ; CHECK-NEXT: store <4 x i8> [[VEC_IND]], ptr [[TMP2]], align 1 | |
| ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 | |
| ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4) | |
| ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]] | |
| ; CHECK: [[MIDDLE_BLOCK]]: | |
| ; CHECK-NEXT: [[TMP4:%.*]] = sub nuw i64 [[N_VEC]], 1 | |
| ; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i8 | |
| ; CHECK-NEXT: [[TMP6:%.*]] = add i8 [[TMP1]], [[TMP5]] | |
| ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] | |
| ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] | |
| ; CHECK: [[SCALAR_PH]]: | |
| ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] | |
| ; CHECK-NEXT: br label %[[LOOP:.*]] | |
| ; CHECK: [[LOOP]]: | |
| ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] | |
| ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 | |
| ; CHECK-NEXT: [[OFF:%.*]] = add i64 [[IV]], [[OFF_V]] | |
| ; CHECK-NEXT: [[T:%.*]] = trunc i64 [[OFF]] to i8 | |
| ; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]] | |
| ; CHECK-NEXT: store i8 [[T]], ptr [[GEP]], align 1 | |
| ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] | |
| ; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP44:![0-9]+]] | |
| ; CHECK: [[EXIT]]: | |
| ; CHECK-NEXT: [[T_LCSSA:%.*]] = phi i8 [ [[T]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ] | |
| ; CHECK-NEXT: ret i8 [[T_LCSSA]] | |
| ; | |
| entry: | |
| br label %loop | |
| loop: | |
| %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] | |
| %iv.next = add nuw i64 %iv, 1 | |
| %off = add i64 %iv, %off.v | |
| %t = trunc i64 %off to i8 | |
| %gep = getelementptr inbounds i8, ptr %dst, i64 %iv | |
| store i8 %t, ptr %gep, align 1 | |
| %ec = icmp eq i64 %iv.next, %n | |
| br i1 %ec, label %exit, label %loop | |
| exit: | |
| %t.lcssa = phi i8 [ %t, %loop ] | |
| ret i8 %t.lcssa | |
| } |