| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 |
| ; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -verify-loop-info -verify-dom-info -S < %s | FileCheck %s |
| ; RUN: opt -mtriple=aarch64 -mattr=+sve -passes='loop(loop-idiom-vectorize)' -disable-loop-idiom-vectorize-find-first-byte -S < %s | FileCheck -check-prefix=DISABLE %s |
| |
| ; Base case based on `libcxx/include/__algorithm/find_first_of.h': |
| ; char* find_first_of(char *first, char *last, char *s_first, char *s_last) { |
| ; for (; first != last; ++first) |
| ; for (char *it = s_first; it != s_last; ++it) |
| ; if (*first == *it) |
| ; return first; |
| ; return last; |
| ; } |
| define ptr @find_first_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_of_i8( |
| ; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; CHECK: [[HEADER_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[MEM_CHECK:.*]] |
| ; CHECK: [[MEM_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64 |
| ; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16) |
| ; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]] |
| ; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]] |
| ; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]] |
| ; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0:![0-9]+]] |
| ; CHECK: [[FIND_FIRST_VEC_HEADER]]: |
| ; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64 |
| ; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]]) |
| ; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]] |
| ; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]] |
| ; CHECK: [[MATCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]]) |
| ; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]] |
| ; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer |
| ; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]] |
| ; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0) |
| ; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]]) |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]]) |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]] |
| ; CHECK: [[CALCULATE_MATCH]]: |
| ; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true) |
| ; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]] |
| ; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]] |
| ; CHECK: [[NEEDLE_CHECK_VEC]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]] |
| ; CHECK: [[SEARCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]] |
| ; CHECK: [[SCALAR_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[HEADER:.*]] |
| ; CHECK: [[HEADER]]: |
| ; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ] |
| ; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; CHECK-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; CHECK: [[NEEDLE_CHECK:.*]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; CHECK: [[MATCH_CHECK]]: |
| ; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]] |
| ; CHECK: [[SEARCH_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1 |
| ; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]] |
| ; CHECK: [[EXIT_LOOPEXIT]]: |
| ; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ] |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT_LOOPEXIT1]]: |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ] |
| ; CHECK-NEXT: ret ptr [[RES]] |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i8( |
| ; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0:[0-9]+]] { |
| ; DISABLE-NEXT: [[ENTRY:.*]]: |
| ; DISABLE-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; DISABLE-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; DISABLE-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; DISABLE: [[HEADER_PREHEADER]]: |
| ; DISABLE-NEXT: br label %[[HEADER:.*]] |
| ; DISABLE: [[HEADER]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ] |
| ; DISABLE-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; DISABLE-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; DISABLE: [[NEEDLE_CHECK:.*]]: |
| ; DISABLE-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; DISABLE-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; DISABLE: [[MATCH_CHECK]]: |
| ; DISABLE-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; DISABLE-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; DISABLE-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; DISABLE-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]] |
| ; DISABLE: [[SEARCH_CHECK]]: |
| ; DISABLE-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1 |
| ; DISABLE-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; DISABLE-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]] |
| ; DISABLE: [[EXIT_LOOPEXIT]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ] |
| ; DISABLE-NEXT: br label %[[EXIT]] |
| ; DISABLE: [[EXIT_LOOPEXIT1]]: |
| ; DISABLE-NEXT: br label %[[EXIT]] |
| ; DISABLE: [[EXIT]]: |
| ; DISABLE-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ] |
| ; DISABLE-NEXT: ret ptr [[RES]] |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit, label %header |
| |
| exit: |
| %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ] |
| ret ptr %res |
| } |
| |
| ; Equivalent to @find_first_of_i8 but with i16. |
| ; This is accepted and generates a similar loop. |
| define ptr @find_first_of_i16(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_of_i16( |
| ; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; CHECK: [[HEADER_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[MEM_CHECK:.*]] |
| ; CHECK: [[MEM_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64 |
| ; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8) |
| ; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]] |
| ; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]] |
| ; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]] |
| ; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]] |
| ; CHECK: [[FIND_FIRST_VEC_HEADER]]: |
| ; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64 |
| ; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]]) |
| ; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP0]], [[SEARCH_PRED]] |
| ; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 1 [[PSEARCH]], <vscale x 8 x i1> [[SEARCH_MASKED]], <vscale x 8 x i16> zeroinitializer) |
| ; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]] |
| ; CHECK: [[MATCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]]) |
| ; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 8 x i1> [[TMP0]], [[NEEDLE_PRED]] |
| ; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 1 [[PNEEDLE]], <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> zeroinitializer) |
| ; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[NEEDLE0]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[NEEDLE0_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer |
| ; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 8 x i1> [[NEEDLE_MASKED]], <vscale x 8 x i16> [[NEEDLE_LOAD_VEC]], <vscale x 8 x i16> [[NEEDLE0_SPLAT]] |
| ; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <8 x i16> @llvm.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> [[NEEDLE_SPLAT]], i64 0) |
| ; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 8 x i1> @llvm.experimental.vector.match.nxv8i16.v8i16(<vscale x 8 x i16> [[SEARCH_LOAD_VEC]], <8 x i16> [[NEEDLE_VEC]], <vscale x 8 x i1> [[SEARCH_MASKED]]) |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[MATCH_PRED]]) |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]] |
| ; CHECK: [[CALCULATE_MATCH]]: |
| ; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 8 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[MATCH_VEC]], i1 true) |
| ; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i16, ptr [[MATCH_START]], i64 [[MATCH_IDX]] |
| ; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]] |
| ; CHECK: [[NEEDLE_CHECK_VEC]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i16, ptr [[PNEEDLE]], i64 8 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]] |
| ; CHECK: [[SEARCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i16, ptr [[PSEARCH]], i64 8 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_LOOPEXIT1:.*]] |
| ; CHECK: [[SCALAR_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[HEADER:.*]] |
| ; CHECK: [[HEADER]]: |
| ; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ] |
| ; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1 |
| ; CHECK-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; CHECK: [[NEEDLE_CHECK:.*]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; CHECK: [[MATCH_CHECK]]: |
| ; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1 |
| ; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT]], label %[[NEEDLE_CHECK]] |
| ; CHECK: [[SEARCH_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1 |
| ; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1]], label %[[HEADER]] |
| ; CHECK: [[EXIT_LOOPEXIT]]: |
| ; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ] |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT_LOOPEXIT1]]: |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ] |
| ; CHECK-NEXT: ret ptr [[RES]] |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i16( |
| ; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; DISABLE-NEXT: [[ENTRY:.*]]: |
| ; DISABLE-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; DISABLE-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; DISABLE-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; DISABLE: [[HEADER_PREHEADER]]: |
| ; DISABLE-NEXT: br label %[[HEADER:.*]] |
| ; DISABLE: [[HEADER]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ] |
| ; DISABLE-NEXT: [[SEARCH_LOAD:%.*]] = load i16, ptr [[SEARCH_PTR]], align 1 |
| ; DISABLE-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; DISABLE: [[NEEDLE_CHECK:.*]]: |
| ; DISABLE-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i16, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; DISABLE-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; DISABLE: [[MATCH_CHECK]]: |
| ; DISABLE-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; DISABLE-NEXT: [[NEEDLE_LOAD:%.*]] = load i16, ptr [[NEEDLE_PTR]], align 1 |
| ; DISABLE-NEXT: [[MATCH_CMP:%.*]] = icmp eq i16 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; DISABLE-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[NEEDLE_CHECK]] |
| ; DISABLE: [[SEARCH_CHECK]]: |
| ; DISABLE-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i16, ptr [[SEARCH_PTR]], i64 1 |
| ; DISABLE-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; DISABLE-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_LOOPEXIT1:.*]], label %[[HEADER]] |
| ; DISABLE: [[EXIT_LOOPEXIT]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ] |
| ; DISABLE-NEXT: br label %[[EXIT]] |
| ; DISABLE: [[EXIT_LOOPEXIT1]]: |
| ; DISABLE-NEXT: br label %[[EXIT]] |
| ; DISABLE: [[EXIT]]: |
| ; DISABLE-NEXT: [[RES:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_PTR_LCSSA]], %[[EXIT_LOOPEXIT]] ], [ [[SEARCH_END]], %[[EXIT_LOOPEXIT1]] ] |
| ; DISABLE-NEXT: ret ptr [[RES]] |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i16, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i16, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i16, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i16 %search_load, %needle_load |
| br i1 %match_cmp, label %exit, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i16, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit, label %header |
| |
| exit: |
| %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ] |
| ret ptr %res |
| } |
| |
| ; Same as @find_first_of_i8 but with two intermediate exit blocks for the |
| ; "success" (exit_succ) and "failure" (exit_fail) paths. |
| define ptr @find_first_of_i8_multi_exit(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit( |
| ; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; CHECK-NEXT: [[ENTRY:.*]]: |
| ; CHECK-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; CHECK-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; CHECK-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; CHECK-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; CHECK: [[HEADER_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[MEM_CHECK:.*]] |
| ; CHECK: [[MEM_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64 |
| ; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16) |
| ; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]] |
| ; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]] |
| ; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]] |
| ; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]] |
| ; CHECK: [[FIND_FIRST_VEC_HEADER]]: |
| ; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64 |
| ; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]]) |
| ; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]] |
| ; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]] |
| ; CHECK: [[MATCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]]) |
| ; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]] |
| ; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer |
| ; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]] |
| ; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0) |
| ; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]]) |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]]) |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]] |
| ; CHECK: [[CALCULATE_MATCH]]: |
| ; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true) |
| ; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]] |
| ; CHECK-NEXT: br label %[[EXIT_SUCC:.*]] |
| ; CHECK: [[NEEDLE_CHECK_VEC]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]] |
| ; CHECK: [[SEARCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[EXIT_FAIL_LOOPEXIT:.*]] |
| ; CHECK: [[SCALAR_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[HEADER:.*]] |
| ; CHECK: [[HEADER]]: |
| ; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ] |
| ; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; CHECK-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; CHECK: [[NEEDLE_CHECK:.*]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; CHECK: [[MATCH_CHECK]]: |
| ; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_SUCC]], label %[[NEEDLE_CHECK]] |
| ; CHECK: [[SEARCH_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1 |
| ; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT]], label %[[HEADER]] |
| ; CHECK: [[EXIT_SUCC]]: |
| ; CHECK-NEXT: [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ] |
| ; CHECK-NEXT: br label %[[EXIT:.*]] |
| ; CHECK: [[EXIT_FAIL_LOOPEXIT]]: |
| ; CHECK-NEXT: br label %[[EXIT_FAIL]] |
| ; CHECK: [[EXIT_FAIL]]: |
| ; CHECK-NEXT: [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ] |
| ; CHECK-NEXT: br label %[[EXIT]] |
| ; CHECK: [[EXIT]]: |
| ; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ] |
| ; CHECK-NEXT: ret ptr [[RES]] |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit( |
| ; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; DISABLE-NEXT: [[ENTRY:.*]]: |
| ; DISABLE-NEXT: [[SEARCH_TEST:%.*]] = icmp eq ptr [[SEARCH_START]], [[SEARCH_END]] |
| ; DISABLE-NEXT: [[NEEDLE_TEST:%.*]] = icmp eq ptr [[NEEDLE_START]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: [[COMBINED_TEST:%.*]] = or i1 [[SEARCH_TEST]], [[NEEDLE_TEST]] |
| ; DISABLE-NEXT: br i1 [[COMBINED_TEST]], label %[[EXIT_FAIL:.*]], label %[[HEADER_PREHEADER:.*]] |
| ; DISABLE: [[HEADER_PREHEADER]]: |
| ; DISABLE-NEXT: br label %[[HEADER:.*]] |
| ; DISABLE: [[HEADER]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[HEADER_PREHEADER]] ] |
| ; DISABLE-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; DISABLE-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; DISABLE: [[NEEDLE_CHECK:.*]]: |
| ; DISABLE-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr inbounds i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; DISABLE-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; DISABLE: [[MATCH_CHECK]]: |
| ; DISABLE-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; DISABLE-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; DISABLE-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; DISABLE-NEXT: br i1 [[MATCH_CMP]], label %[[EXIT_SUCC:.*]], label %[[NEEDLE_CHECK]] |
| ; DISABLE: [[SEARCH_CHECK]]: |
| ; DISABLE-NEXT: [[SEARCH_NEXT]] = getelementptr inbounds i8, ptr [[SEARCH_PTR]], i64 1 |
| ; DISABLE-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; DISABLE-NEXT: br i1 [[SEARCH_CMP]], label %[[EXIT_FAIL_LOOPEXIT:.*]], label %[[HEADER]] |
| ; DISABLE: [[EXIT_SUCC]]: |
| ; DISABLE-NEXT: [[RES_SUCC:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ] |
| ; DISABLE-NEXT: br label %[[EXIT:.*]] |
| ; DISABLE: [[EXIT_FAIL_LOOPEXIT]]: |
| ; DISABLE-NEXT: br label %[[EXIT_FAIL]] |
| ; DISABLE: [[EXIT_FAIL]]: |
| ; DISABLE-NEXT: [[RES_FAIL:%.*]] = phi ptr [ [[SEARCH_END]], %[[ENTRY]] ], [ [[SEARCH_END]], %[[EXIT_FAIL_LOOPEXIT]] ] |
| ; DISABLE-NEXT: br label %[[EXIT]] |
| ; DISABLE: [[EXIT]]: |
| ; DISABLE-NEXT: [[RES:%.*]] = phi ptr [ [[RES_SUCC]], %[[EXIT_SUCC]] ], [ [[RES_FAIL]], %[[EXIT_FAIL]] ] |
| ; DISABLE-NEXT: ret ptr [[RES]] |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit_fail, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit_succ, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit_fail, label %header |
| |
| exit_succ: |
| %res_succ = phi ptr [ %search_ptr, %match_check ] |
| br label %exit |
| |
| exit_fail: |
| %res_fail = phi ptr [ %search_end, %entry ], [ %search_end, %search_check ] |
| br label %exit |
| |
| exit: |
| %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ] |
| ret ptr %res |
| } |
| |
| define ptr @ensure_not_found_successors_fixed(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @ensure_not_found_successors_fixed( |
| ; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: br label %[[MEM_CHECK:.*]] |
| ; CHECK: [[MEM_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64 |
| ; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16) |
| ; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]] |
| ; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]] |
| ; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]] |
| ; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]] |
| ; CHECK: [[FIND_FIRST_VEC_HEADER]]: |
| ; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64 |
| ; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]]) |
| ; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]] |
| ; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]] |
| ; CHECK: [[MATCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]]) |
| ; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]] |
| ; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer |
| ; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]] |
| ; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0) |
| ; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]]) |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]]) |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]] |
| ; CHECK: [[CALCULATE_MATCH]]: |
| ; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true) |
| ; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]] |
| ; CHECK-NEXT: br label %[[FOUND_MATCH:.*]] |
| ; CHECK: [[NEEDLE_CHECK_VEC]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]] |
| ; CHECK: [[SEARCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[NOT_FOUND:.*]] |
| ; CHECK: [[SCALAR_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[HEADER:.*]] |
| ; CHECK: [[HEADER]]: |
| ; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ] |
| ; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; CHECK-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; CHECK: [[NEEDLE_CHECK:.*]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; CHECK: [[MATCH_CHECK]]: |
| ; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[FOUND_MATCH]], label %[[NEEDLE_CHECK]] |
| ; CHECK: [[SEARCH_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1 |
| ; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[NOT_FOUND]], label %[[HEADER]] |
| ; CHECK: [[FOUND_MATCH]]: |
| ; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ] |
| ; CHECK-NEXT: ret ptr [[SEARCH_PTR_LCSSA]] |
| ; CHECK: [[NOT_FOUND]]: |
| ; CHECK-NEXT: [[UNUSED:%.*]] = phi i64 [ 0, %[[SEARCH_CHECK]] ], [ 0, %[[SEARCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: ret ptr null |
| ; |
| ; DISABLE-LABEL: define ptr @ensure_not_found_successors_fixed( |
| ; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; DISABLE-NEXT: [[ENTRY:.*]]: |
| ; DISABLE-NEXT: br label %[[HEADER:.*]] |
| ; DISABLE: [[HEADER]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[ENTRY]] ] |
| ; DISABLE-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; DISABLE-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; DISABLE: [[NEEDLE_CHECK:.*]]: |
| ; DISABLE-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; DISABLE-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; DISABLE: [[MATCH_CHECK]]: |
| ; DISABLE-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; DISABLE-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; DISABLE-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; DISABLE-NEXT: br i1 [[MATCH_CMP]], label %[[FOUND_MATCH:.*]], label %[[NEEDLE_CHECK]] |
| ; DISABLE: [[SEARCH_CHECK]]: |
| ; DISABLE-NEXT: [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1 |
| ; DISABLE-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; DISABLE-NEXT: br i1 [[SEARCH_CMP]], label %[[NOT_FOUND:.*]], label %[[HEADER]] |
| ; DISABLE: [[FOUND_MATCH]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ] |
| ; DISABLE-NEXT: ret ptr [[SEARCH_PTR_LCSSA]] |
| ; DISABLE: [[NOT_FOUND]]: |
| ; DISABLE-NEXT: [[UNUSED:%.*]] = phi i64 [ 0, %[[SEARCH_CHECK]] ] |
| ; DISABLE-NEXT: ret ptr null |
| ; |
| entry: |
| br label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %found_match, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %not_found, label %header |
| |
| found_match: |
| ret ptr %search_ptr |
| |
| not_found: |
| %unused = phi i64 [ 0, %search_check ] |
| ret ptr null |
| } |
| |
| ; This is a small variation of the previous test case, where we have another value |
| ; in a successor phi: |
| ; |
| ; entry: |
| ; %dummy_val = getelementptr i8, ptr %search_start, i64 1 |
| ; br label %header |
| ; ... |
| ; not_found: |
| ; %unused = phi ptr [ %dummy_val, %search_check ] |
| ; |
| define ptr @ensure_not_found_successors_fixed2(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @ensure_not_found_successors_fixed2( |
| ; CHECK-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; CHECK-NEXT: [[ENTRY:.*:]] |
| ; CHECK-NEXT: [[DUMMY_VAL:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 1 |
| ; CHECK-NEXT: br label %[[MEM_CHECK:.*]] |
| ; CHECK: [[MEM_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_START_INT:%.*]] = ptrtoint ptr [[SEARCH_START]] to i64 |
| ; CHECK-NEXT: [[SEARCH_END_INT:%.*]] = ptrtoint ptr [[SEARCH_END]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_START_INT:%.*]] = ptrtoint ptr [[NEEDLE_START]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_END_INT:%.*]] = ptrtoint ptr [[NEEDLE_END]] to i64 |
| ; CHECK-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 16) |
| ; CHECK-NEXT: [[SEARCH_START_PAGE:%.*]] = lshr i64 [[SEARCH_START_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_END_PAGE:%.*]] = lshr i64 [[SEARCH_END_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_START_PAGE:%.*]] = lshr i64 [[NEEDLE_START_INT]], 12 |
| ; CHECK-NEXT: [[NEEDLE_END_PAGE:%.*]] = lshr i64 [[NEEDLE_END_INT]], 12 |
| ; CHECK-NEXT: [[SEARCH_PAGE_CMP:%.*]] = icmp ne i64 [[SEARCH_START_PAGE]], [[SEARCH_END_PAGE]] |
| ; CHECK-NEXT: [[NEEDLE_PAGE_CMP:%.*]] = icmp ne i64 [[NEEDLE_START_PAGE]], [[NEEDLE_END_PAGE]] |
| ; CHECK-NEXT: [[COMBINED_PAGE_CMP:%.*]] = or i1 [[SEARCH_PAGE_CMP]], [[NEEDLE_PAGE_CMP]] |
| ; CHECK-NEXT: br i1 [[COMBINED_PAGE_CMP]], label %[[SCALAR_PREHEADER:.*]], label %[[FIND_FIRST_VEC_HEADER:.*]], !prof [[PROF0]] |
| ; CHECK: [[FIND_FIRST_VEC_HEADER]]: |
| ; CHECK-NEXT: [[PSEARCH:%.*]] = phi ptr [ [[SEARCH_START]], %[[MEM_CHECK]] ], [ [[SEARCH_NEXT_VEC:%.*]], %[[SEARCH_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PSEARCH]] to i64 |
| ; CHECK-NEXT: [[SEARCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP1]], i64 [[SEARCH_END_INT]]) |
| ; CHECK-NEXT: [[SEARCH_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[SEARCH_PRED]] |
| ; CHECK-NEXT: [[SEARCH_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PSEARCH]], <vscale x 16 x i1> [[SEARCH_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: br label %[[MATCH_CHECK_VEC:.*]] |
| ; CHECK: [[MATCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[PNEEDLE:%.*]] = phi ptr [ [[NEEDLE_START]], %[[FIND_FIRST_VEC_HEADER]] ], [ [[NEEDLE_NEXT_VEC:%.*]], %[[NEEDLE_CHECK_VEC:.*]] ] |
| ; CHECK-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[PNEEDLE]] to i64 |
| ; CHECK-NEXT: [[NEEDLE_PRED:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP2]], i64 [[NEEDLE_END_INT]]) |
| ; CHECK-NEXT: [[NEEDLE_MASKED:%.*]] = and <vscale x 16 x i1> [[TMP0]], [[NEEDLE_PRED]] |
| ; CHECK-NEXT: [[NEEDLE_LOAD_VEC:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[PNEEDLE]], <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> zeroinitializer) |
| ; CHECK-NEXT: [[NEEDLE0:%.*]] = extractelement <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[NEEDLE0]], i64 0 |
| ; CHECK-NEXT: [[NEEDLE0_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[NEEDLE0_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer |
| ; CHECK-NEXT: [[NEEDLE_SPLAT:%.*]] = select <vscale x 16 x i1> [[NEEDLE_MASKED]], <vscale x 16 x i8> [[NEEDLE_LOAD_VEC]], <vscale x 16 x i8> [[NEEDLE0_SPLAT]] |
| ; CHECK-NEXT: [[NEEDLE_VEC:%.*]] = call <16 x i8> @llvm.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> [[NEEDLE_SPLAT]], i64 0) |
| ; CHECK-NEXT: [[MATCH_PRED:%.*]] = call <vscale x 16 x i1> @llvm.experimental.vector.match.nxv16i8.v16i8(<vscale x 16 x i8> [[SEARCH_LOAD_VEC]], <16 x i8> [[NEEDLE_VEC]], <vscale x 16 x i1> [[SEARCH_MASKED]]) |
| ; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[MATCH_PRED]]) |
| ; CHECK-NEXT: br i1 [[TMP3]], label %[[CALCULATE_MATCH:.*]], label %[[NEEDLE_CHECK_VEC]] |
| ; CHECK: [[CALCULATE_MATCH]]: |
| ; CHECK-NEXT: [[MATCH_START:%.*]] = phi ptr [ [[PSEARCH]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_VEC:%.*]] = phi <vscale x 16 x i1> [ [[MATCH_PRED]], %[[MATCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: [[MATCH_IDX:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[MATCH_VEC]], i1 true) |
| ; CHECK-NEXT: [[MATCH_RES:%.*]] = getelementptr i8, ptr [[MATCH_START]], i64 [[MATCH_IDX]] |
| ; CHECK-NEXT: br label %[[FOUND_MATCH:.*]] |
| ; CHECK: [[NEEDLE_CHECK_VEC]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT_VEC]] = getelementptr i8, ptr [[PNEEDLE]], i64 16 |
| ; CHECK-NEXT: [[TMP4:%.*]] = icmp ult ptr [[NEEDLE_NEXT_VEC]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[TMP4]], label %[[MATCH_CHECK_VEC]], label %[[SEARCH_CHECK_VEC]] |
| ; CHECK: [[SEARCH_CHECK_VEC]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT_VEC]] = getelementptr i8, ptr [[PSEARCH]], i64 16 |
| ; CHECK-NEXT: [[TMP5:%.*]] = icmp ult ptr [[SEARCH_NEXT_VEC]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[TMP5]], label %[[FIND_FIRST_VEC_HEADER]], label %[[NOT_FOUND:.*]] |
| ; CHECK: [[SCALAR_PREHEADER]]: |
| ; CHECK-NEXT: br label %[[HEADER:.*]] |
| ; CHECK: [[HEADER]]: |
| ; CHECK-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[SCALAR_PREHEADER]] ] |
| ; CHECK-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; CHECK-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; CHECK: [[NEEDLE_CHECK:.*]]: |
| ; CHECK-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; CHECK-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; CHECK-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; CHECK: [[MATCH_CHECK]]: |
| ; CHECK-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; CHECK-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; CHECK-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; CHECK-NEXT: br i1 [[MATCH_CMP]], label %[[FOUND_MATCH]], label %[[NEEDLE_CHECK]] |
| ; CHECK: [[SEARCH_CHECK]]: |
| ; CHECK-NEXT: [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1 |
| ; CHECK-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; CHECK-NEXT: br i1 [[SEARCH_CMP]], label %[[NOT_FOUND]], label %[[HEADER]] |
| ; CHECK: [[FOUND_MATCH]]: |
| ; CHECK-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ], [ [[MATCH_RES]], %[[CALCULATE_MATCH]] ] |
| ; CHECK-NEXT: ret ptr [[SEARCH_PTR_LCSSA]] |
| ; CHECK: [[NOT_FOUND]]: |
| ; CHECK-NEXT: [[UNUSED:%.*]] = phi ptr [ [[DUMMY_VAL]], %[[SEARCH_CHECK]] ], [ [[DUMMY_VAL]], %[[SEARCH_CHECK_VEC]] ] |
| ; CHECK-NEXT: ret ptr null |
| ; |
| ; DISABLE-LABEL: define ptr @ensure_not_found_successors_fixed2( |
| ; DISABLE-SAME: ptr [[SEARCH_START:%.*]], ptr [[SEARCH_END:%.*]], ptr [[NEEDLE_START:%.*]], ptr [[NEEDLE_END:%.*]]) #[[ATTR0]] { |
| ; DISABLE-NEXT: [[ENTRY:.*]]: |
| ; DISABLE-NEXT: [[DUMMY_VAL:%.*]] = getelementptr i8, ptr [[SEARCH_START]], i64 1 |
| ; DISABLE-NEXT: br label %[[HEADER:.*]] |
| ; DISABLE: [[HEADER]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR:%.*]] = phi ptr [ [[SEARCH_NEXT:%.*]], %[[SEARCH_CHECK:.*]] ], [ [[SEARCH_START]], %[[ENTRY]] ] |
| ; DISABLE-NEXT: [[SEARCH_LOAD:%.*]] = load i8, ptr [[SEARCH_PTR]], align 1 |
| ; DISABLE-NEXT: br label %[[MATCH_CHECK:.*]] |
| ; DISABLE: [[NEEDLE_CHECK:.*]]: |
| ; DISABLE-NEXT: [[NEEDLE_NEXT:%.*]] = getelementptr i8, ptr [[NEEDLE_PTR:%.*]], i64 1 |
| ; DISABLE-NEXT: [[NEEDLE_CMP:%.*]] = icmp eq ptr [[NEEDLE_NEXT]], [[NEEDLE_END]] |
| ; DISABLE-NEXT: br i1 [[NEEDLE_CMP]], label %[[SEARCH_CHECK]], label %[[MATCH_CHECK]] |
| ; DISABLE: [[MATCH_CHECK]]: |
| ; DISABLE-NEXT: [[NEEDLE_PTR]] = phi ptr [ [[NEEDLE_START]], %[[HEADER]] ], [ [[NEEDLE_NEXT]], %[[NEEDLE_CHECK]] ] |
| ; DISABLE-NEXT: [[NEEDLE_LOAD:%.*]] = load i8, ptr [[NEEDLE_PTR]], align 1 |
| ; DISABLE-NEXT: [[MATCH_CMP:%.*]] = icmp eq i8 [[SEARCH_LOAD]], [[NEEDLE_LOAD]] |
| ; DISABLE-NEXT: br i1 [[MATCH_CMP]], label %[[FOUND_MATCH:.*]], label %[[NEEDLE_CHECK]] |
| ; DISABLE: [[SEARCH_CHECK]]: |
| ; DISABLE-NEXT: [[SEARCH_NEXT]] = getelementptr i8, ptr [[SEARCH_PTR]], i64 1 |
| ; DISABLE-NEXT: [[SEARCH_CMP:%.*]] = icmp eq ptr [[SEARCH_NEXT]], [[SEARCH_END]] |
| ; DISABLE-NEXT: br i1 [[SEARCH_CMP]], label %[[NOT_FOUND:.*]], label %[[HEADER]] |
| ; DISABLE: [[FOUND_MATCH]]: |
| ; DISABLE-NEXT: [[SEARCH_PTR_LCSSA:%.*]] = phi ptr [ [[SEARCH_PTR]], %[[MATCH_CHECK]] ] |
| ; DISABLE-NEXT: ret ptr [[SEARCH_PTR_LCSSA]] |
| ; DISABLE: [[NOT_FOUND]]: |
| ; DISABLE-NEXT: [[UNUSED:%.*]] = phi ptr [ [[DUMMY_VAL]], %[[SEARCH_CHECK]] ] |
| ; DISABLE-NEXT: ret ptr null |
| ; |
| entry: |
| %dummy_val = getelementptr i8, ptr %search_start, i64 1 |
| br label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %found_match, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %not_found, label %header |
| |
| found_match: |
| ret ptr %search_ptr |
| |
| not_found: |
| %unused = phi ptr [ %dummy_val, %search_check ] |
| ret ptr null |
| } |
| |
| ; From here on we only test for the presence/absence of the intrinsic. |
| ; UTC_ARGS: --disable |
| |
| ; Same as @find_first_of_i8 but with `ne' comparison. |
| ; This is rejected for now, but should eventually be supported. |
| define ptr @find_first_not_of_i8(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_not_of_i8( |
| ; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_not_of_i8( |
| ; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp ne i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit, label %header |
| |
| exit: |
| %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ] |
| ret ptr %res |
| } |
| |
| ; This is the same as @find_first_of_i8 but without SVE2, which we require to |
| ; perform the conversion. |
| define ptr @find_first_of_i8_nosve2(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) { |
| ; CHECK-LABEL: define ptr @find_first_of_i8_nosve2( |
| ; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i8_nosve2( |
| ; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit, label %header |
| |
| exit: |
| %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ] |
| ret ptr %res |
| } |
| |
| ; Same as @find_first_of_i8 but here we use the inner PHI outside the loop nest. |
| ; This isn't supported. |
| define ptr @find_first_of_i8_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_of_i8_outside_use( |
| ; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i8_outside_use( |
| ; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit, label %header |
| |
| exit: |
| %res = phi ptr [ %search_end, %entry ], [ %search_ptr, %match_check ], [ %search_end, %search_check ] |
| %use = phi ptr [ %needle_end, %entry ], [ %needle_ptr, %match_check ], [ %needle_end, %search_check ] |
| ret ptr %res |
| } |
| |
| ; Same as @find_first_of_i8_multi_exit but `search_ptr' is used in `exit_fail' |
| ; which should block the transform. |
| define ptr @find_first_of_i8_multi_exit_outside_use(ptr %search_start, ptr %search_end, ptr %needle_start, ptr %needle_end) #0 { |
| ; CHECK-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use( |
| ; CHECK-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| ; DISABLE-LABEL: define ptr @find_first_of_i8_multi_exit_outside_use( |
| ; DISABLE-NOT: {{%.*}} @llvm.experimental.vector.match{{.*}} |
| ; |
| entry: |
| %search_test = icmp eq ptr %search_start, %search_end |
| %needle_test = icmp eq ptr %needle_start, %needle_end |
| %combined_test = or i1 %search_test, %needle_test |
| br i1 %combined_test, label %exit_fail, label %header |
| |
| header: |
| %search_ptr = phi ptr [ %search_next, %search_check ], [ %search_start, %entry ] |
| %search_load = load i8, ptr %search_ptr, align 1 |
| br label %match_check |
| |
| needle_check: |
| %needle_next = getelementptr inbounds i8, ptr %needle_ptr, i64 1 |
| %needle_cmp = icmp eq ptr %needle_next, %needle_end |
| br i1 %needle_cmp, label %search_check, label %match_check |
| |
| match_check: |
| %needle_ptr = phi ptr [ %needle_start, %header ], [ %needle_next, %needle_check ] |
| %needle_load = load i8, ptr %needle_ptr, align 1 |
| %match_cmp = icmp eq i8 %search_load, %needle_load |
| br i1 %match_cmp, label %exit_succ, label %needle_check |
| |
| search_check: |
| %search_next = getelementptr inbounds i8, ptr %search_ptr, i64 1 |
| %search_cmp = icmp eq ptr %search_next, %search_end |
| br i1 %search_cmp, label %exit_fail, label %header |
| |
| exit_succ: |
| %res_succ = phi ptr [ %search_ptr, %match_check ] |
| br label %exit |
| |
| exit_fail: |
| %res_fail = phi ptr [ %search_end, %entry ], [ %search_ptr, %search_check ] |
| br label %exit |
| |
| exit: |
| %res = phi ptr [ %res_succ, %exit_succ ], [ %res_fail, %exit_fail ] |
| ret ptr %res |
| } |
| |
| attributes #0 = { "target-features"="+sve2" } |
| ;. |
| ; CHECK: [[PROF0]] = !{!"branch_weights", i32 10, i32 90} |
| ;. |
