| #include <algorithm> | |
| #include <cstdlib> | |
| #include <functional> | |
| #include <initializer_list> | |
| #include <iostream> | |
| #include <limits> | |
| #include <memory> | |
| #include <random> | |
| #include <stdint.h> | |
| #include <utility> | |
| #include "common.h" | |
| // Tests for memory runtime checks generated by the vectorizer. Runs scalar and | |
| // vectorized versions of a loop requiring runtime checks on the same inputs | |
| // with pointers to the same buffer using various offsets between reads and | |
| // writes. Fails if they do not produce the same results. | |
| template <typename Ty> | |
| static void check(const std::unique_ptr<Ty[]> &Reference, | |
| const std::unique_ptr<Ty[]> &Tmp, unsigned NumElements, | |
| int Offset) { | |
| if (!std::equal(&Reference[0], &Reference[0] + NumElements, &Tmp[0])) { | |
| std::cerr << "Miscompare with offset " << Offset << "\n"; | |
| exit(1); | |
| } | |
| } | |
| // Helper to call \p f with \p args and acts as optimization barrier for \p f. | |
| template <typename F, typename... Args> | |
| __attribute__((optnone)) static void callThroughOptnone(F &&f, Args &&...args) { | |
| f(std::forward<Args>(args)...); | |
| } | |
| template <typename Ty> using Fn2Ty = std::function<void(Ty *, Ty *, unsigned)>; | |
| // Run both \p ScalarFn and \p VectorFn on the same inputs with pointers to the | |
| // same buffer. Fail if they do not produce the same results. | |
| template <typename Ty> | |
| static void checkOverlappingMemoryOneRuntimeCheck(Fn2Ty<Ty> ScalarFn, | |
| Fn2Ty<Ty> VectorFn, | |
| const char *Name) { | |
| std::cout << "Checking " << Name << "\n"; | |
| unsigned N = 100; | |
| // Make sure we have enough extra elements so we can be liberal with offsets. | |
| unsigned NumArrayElements = N * 8; | |
| std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]); | |
| auto CheckWithOffset = [&](int Offset) { | |
| init_data(Input1, NumArrayElements); | |
| for (unsigned i = 0; i < NumArrayElements; i++) { | |
| Reference[i] = Input1[i]; | |
| ToCheck[i] = Input1[i]; | |
| } | |
| // Run scalar function to generate reference output. | |
| Ty *ReferenceStart = &Reference[NumArrayElements / 2]; | |
| ScalarFn(ReferenceStart + Offset, ReferenceStart, N); | |
| // Run vector function to generate output to check. | |
| Ty *StartPtr = &ToCheck[NumArrayElements / 2]; | |
| callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, N); | |
| // Compare scalar and vector output. | |
| check(Reference, ToCheck, NumArrayElements, Offset); | |
| }; | |
| for (int i = -100; i <= 100; i++) | |
| CheckWithOffset(i); | |
| } | |
| template <typename Ty> | |
| using Fn3Ty = std::function<void(Ty *, Ty *, Ty *, unsigned)>; | |
| template <typename Ty> | |
| static void checkOverlappingMemoryTwoRuntimeChecks(Fn3Ty<Ty> ScalarFn, | |
| Fn3Ty<Ty> VectorFn, | |
| const char *Name) { | |
| std::cout << "Checking " << Name << "\n"; | |
| unsigned N = 100; | |
| // Make sure we have enough extra elements so we can be liberal with offsets. | |
| unsigned NumArrayElements = N * 8; | |
| std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> Input2(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]); | |
| auto CheckWithOffsetSecond = [&](int Offset) { | |
| init_data(Input1, NumArrayElements); | |
| init_data(Input2, NumArrayElements); | |
| for (unsigned i = 0; i < NumArrayElements; i++) { | |
| Reference[i] = Input1[i]; | |
| ToCheck[i] = Input1[i]; | |
| } | |
| // Run scalar function to generate reference output. | |
| Ty *ReferenceStart = &Reference[NumArrayElements / 2]; | |
| ScalarFn(ReferenceStart + Offset, &Input2[0], ReferenceStart, N); | |
| // Run vector function to generate output to check. | |
| Ty *StartPtr = &ToCheck[NumArrayElements / 2]; | |
| callThroughOptnone(VectorFn, StartPtr + Offset, &Input2[0], StartPtr, N); | |
| // Compare scalar and vector output. | |
| check(Reference, ToCheck, NumArrayElements, Offset); | |
| }; | |
| for (int i = -100; i <= 100; i++) | |
| CheckWithOffsetSecond(i); | |
| } | |
| template <typename Ty> | |
| using Fn4Ty = std::function<void(Ty *, Ty *, unsigned, unsigned)>; | |
| template <typename Ty> | |
| static void checkOverlappingMemoryTwoRuntimeChecksNested(Fn4Ty<Ty> ScalarFn, | |
| Fn4Ty<Ty> VectorFn, | |
| const int OuterTC, | |
| const int InnerTC, | |
| const char *Name) { | |
| std::cout << "Checking " << Name << "\n"; | |
| // Make sure we have enough extra elements so we can be liberal with offsets. | |
| const unsigned NumArrayElements = (InnerTC * (OuterTC + 1)) * 8; | |
| std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]); | |
| auto CheckWithOffsetSecond = [&](int Offset) { | |
| init_data(Input1, NumArrayElements); | |
| for (unsigned i = 0; i < NumArrayElements; i++) { | |
| Reference[i] = Input1[i]; | |
| ToCheck[i] = Input1[i]; | |
| } | |
| // Run scalar function to generate reference output. | |
| Ty *ReferenceStart = &Reference[NumArrayElements / 2]; | |
| ScalarFn(ReferenceStart + Offset, ReferenceStart, OuterTC, InnerTC); | |
| // Run vector function to generate output to check. | |
| Ty *StartPtr = &ToCheck[NumArrayElements / 2]; | |
| callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, OuterTC, InnerTC); | |
| // Compare scalar and vector output. | |
| check(Reference, ToCheck, NumArrayElements, Offset); | |
| }; | |
| // With a nested loop, sometimes the runtime checks will fail and sometimes | |
| // succeed. For example, with large offsets you'd expect for the first and | |
| // last one or two executions of the inner loop there is no overlap. | |
| for (int i = -(2 * (InnerTC + 1)); i <= (2 * (InnerTC + 1)); i++) | |
| CheckWithOffsetSecond(i); | |
| } | |
| #define DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(Loop) \ | |
| auto ScalarFn = [](auto *Out, auto *In, unsigned TC, auto S1, auto S2) { \ | |
| _Pragma("clang loop vectorize(disable) interleave_count(1)") Loop \ | |
| }; \ | |
| auto VectorFn = [](auto *Out, auto *In, unsigned TC, auto S1, auto S2) { \ | |
| _Pragma("clang loop vectorize(enable)") Loop \ | |
| }; | |
| template <typename Ty, typename StrideTy> | |
| using StencilFnTy = | |
| std::function<void(Ty *, Ty *, unsigned, StrideTy, StrideTy)>; | |
| template <typename Ty, typename StrideTy> | |
| static void checkOverlappingMemoryStencil(StencilFnTy<Ty, StrideTy> ScalarFn, | |
| StencilFnTy<Ty, StrideTy> VectorFn, | |
| const char *Name, StrideTy S1, | |
| StrideTy S2) { | |
| std::cout << "Checking " << Name << ", strides " << +S1 << " and " << +S2 | |
| << "\n"; | |
| unsigned N = 100; | |
| // Conservative bound on the read offsets for the stencil loops below. | |
| unsigned Reach = 2 * (std::abs(long(S1)) + std::abs(long(S2))) + 1; | |
| // Cover all possible overlap and include disjoint accesses. Test at least | |
| // offsets -100 to 100. | |
| int MaxOffset = std::max(100u, Reach + N + 2); | |
| // The buffer must hold Out at every offset and every read of In. | |
| unsigned Margin = MaxOffset + N; | |
| unsigned NumArrayElements = 2 * Margin + 1; | |
| std::unique_ptr<Ty[]> Input(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]); | |
| std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]); | |
| init_data(Input, NumArrayElements); | |
| auto CheckWithOffset = [&](int Offset, unsigned TC) { | |
| for (unsigned i = 0; i < NumArrayElements; i++) { | |
| Reference[i] = Input[i]; | |
| ToCheck[i] = Input[i]; | |
| } | |
| // Run scalar function to generate reference output. | |
| Ty *ReferenceStart = &Reference[Margin]; | |
| ScalarFn(ReferenceStart + Offset, ReferenceStart, TC, S1, S2); | |
| // Run vector function to generate output to check. | |
| Ty *StartPtr = &ToCheck[Margin]; | |
| callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, TC, S1, S2); | |
| // Compare scalar and vector output. | |
| check(Reference, ToCheck, NumArrayElements, Offset); | |
| }; | |
| // Include empty and short loops as well as a non-multiple tail. The strides | |
| // and trip count stay runtime arguments of the vector function. | |
| for (unsigned TC : {0u, 1u, 2u, 7u, N - 1, N}) | |
| for (int Offset = -MaxOffset; Offset <= MaxOffset; Offset++) | |
| CheckWithOffset(Offset, TC); | |
| } | |
| // Check one element type with one stride. The loops get 0 as their second stride | |
| // and do not use it. | |
| template <typename Ty, typename StrideTy, typename ScalarFnTy, | |
| typename VectorFnTy> | |
| static void checkOneStride(ScalarFnTy ScalarFn, VectorFnTy VectorFn, | |
| const char *Name, | |
| std::initializer_list<StrideTy> Strides) { | |
| for (StrideTy S : Strides) | |
| checkOverlappingMemoryStencil<Ty, StrideTy>(ScalarFn, VectorFn, Name, S, | |
| StrideTy(0)); | |
| } | |
| // Check one element type with two strides. | |
| template <typename Ty, typename StrideTy, typename ScalarFnTy, | |
| typename VectorFnTy> | |
| static void checkTwoStrides( | |
| ScalarFnTy ScalarFn, VectorFnTy VectorFn, const char *Name, | |
| std::initializer_list<std::pair<StrideTy, StrideTy>> Strides) { | |
| for (auto [S1, S2] : Strides) | |
| checkOverlappingMemoryStencil<Ty, StrideTy>(ScalarFn, VectorFn, Name, S1, S2); | |
| } | |
| int main(void) { | |
| rng = std::mt19937(15); | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = 0; i < TC; i++) | |
| A[i] = B[i] + 10; | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, step 1, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>(ScalarFn, VectorFn, | |
| "1 read, 1 write, step 1, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, step 1, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = 0; i < TC; i++) | |
| A[i] = B[i + 3] + 10; | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = 3; i < TC; i++) | |
| A[i] = B[i - 3] + 10; | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = TC; i > 0; i--) | |
| A[i] = B[i] + 10;); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = TC; i > 2; i -= 2) | |
| A[i] = B[i] + 10; | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = 0, j = 0; i < TC; i++) { | |
| A[i] = B[j] + 10; | |
| j += 2; | |
| } | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, | |
| "1 read, 1 write, 2 inductions, different steps, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, | |
| "1 read, 1 write, 2 inductions, different steps, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, | |
| "1 read, 1 write, 2 inductions, different steps, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| _Pragma("clang loop unroll(disable)") | |
| for (unsigned i = 0; i < TC; i += 2) | |
| A[i] = B[i] + 10; | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, induction increment 2, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write, induction increment 2, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, | |
| "1 read, 1 write, induction increment 2, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN2(, | |
| for (unsigned i = 0, sum = 0; i < TC; i++) { | |
| sum += A[i] + 10; | |
| B[TC / 2] = sum; | |
| } | |
| ); | |
| checkOverlappingMemoryOneRuntimeCheck<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint8_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint32_t"); | |
| checkOverlappingMemoryOneRuntimeCheck<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN3( | |
| for (unsigned i = 0; i < TC; i++) | |
| A[i] = B[i] + C[i] + 10; | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint32_t>( | |
| ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint8_t>( | |
| ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint64_t>( | |
| ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint64_t"); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_FN3( | |
| for (unsigned i = 0; i < TC; i++) { | |
| auto X = C[i] + 10; | |
| A[i] = X; | |
| B[i] = X + 9; | |
| } | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint8_t>( | |
| ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint32_t>( | |
| ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecks<uint64_t>( | |
| ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint64_t"); | |
| } | |
| { | |
| DEFINE_NESTED_SCALAR_AND_VECTOR_FN4( | |
| auto X = B[(i * OuterTC) + j]; | |
| A[(i * (OuterTC + 1)) + j] = X; | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint64_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint64_t"); | |
| } | |
| { | |
| DEFINE_NESTED_SCALAR_AND_VECTOR_FN4( | |
| auto X = B[(i * OuterTC) + j]; | |
| A[(i * (OuterTC + 1)) + j] += X; | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint64_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint64_t"); | |
| } | |
| { | |
| DEFINE_NESTED_SCALAR_AND_VECTOR_FN5( | |
| auto X = B[(i * OuterTC) + j]; | |
| A[(i * (OuterTC + 1)) + j] = X; | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint64_t"); | |
| } | |
| { | |
| DEFINE_NESTED_SCALAR_AND_VECTOR_FN5( | |
| auto X = B[(i * OuterTC) + j]; | |
| A[(i * (OuterTC + 1)) + j] += X; | |
| ); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint8_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint32_t"); | |
| checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>( | |
| ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint64_t"); | |
| } | |
| const std::initializer_list<long> Strides = {-16, -1, 0, 1, 2, 7, 16, 64, 100}; | |
| const std::initializer_list<std::pair<long, long>> StridePairs = { | |
| {1, 2}, {2, 1}, {7, 16}, {16, 7}, {16, 100}, {100, 16}, {64, 64}, | |
| {-7, 16}, {16, -7}, {-7, -16}, {0, 16}, {16, 0}, {0, 0}}; | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - S1] + In[i - 1] + In[i] + In[i + 1] + In[i + S1]; | |
| ); | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "1D 5-point stencil, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "1D 5-point stencil, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "1D 5-point stencil, uint64_t", Strides); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - 2 * S1] + In[i - S1] + In[i] + In[i + S1] + | |
| In[i + 2 * S1]; | |
| ); | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "stencil with scaled stride, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "stencil with scaled stride, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "stencil with scaled stride, uint64_t", Strides); | |
| } | |
| { | |
| // A 3D-style stencil. S1 is the row size and S2 is the plane size. | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - S2 - S1] + In[i - S2] + In[i - S2 + S1] + In[i - S1] + | |
| In[i] + In[i + S1] + In[i + S2 - S1] + In[i + S2] + | |
| In[i + S2 + S1]; | |
| ); | |
| checkTwoStrides<uint8_t>( | |
| ScalarFn, VectorFn, "3D-style stencil with two strides, uint8_t", | |
| StridePairs); | |
| checkTwoStrides<uint32_t>( | |
| ScalarFn, VectorFn, "3D-style stencil with two strides, uint32_t", | |
| StridePairs); | |
| checkTwoStrides<uint64_t>( | |
| ScalarFn, VectorFn, "3D-style stencil with two strides, uint64_t", | |
| StridePairs); | |
| } | |
| { | |
| // Either In[i - 2 * S1] or In[i - 2 * S2] can be the lowest read. It | |
| // depends on the strides at runtime. In the same way, either In[i + 2 * S1] | |
| // or In[i + 2 * S2] can be the highest read. | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - 2 * S2] + In[i - 2 * S1] + In[i - S2] + In[i - S1] + | |
| In[i] + In[i + S1] + In[i + S2] + In[i + 2 * S1] + | |
| In[i + 2 * S2]; | |
| ); | |
| checkTwoStrides<uint8_t>( | |
| ScalarFn, VectorFn, "stencil with two scaled strides, uint8_t", | |
| StridePairs); | |
| checkTwoStrides<uint32_t>( | |
| ScalarFn, VectorFn, "stencil with two scaled strides, uint32_t", | |
| StridePairs); | |
| checkTwoStrides<uint64_t>( | |
| ScalarFn, VectorFn, "stencil with two scaled strides, uint64_t", | |
| StridePairs); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| // Convert before subtracting so TC == 0 starts at -1. | |
| for (int64_t i = int64_t(TC) - 1; i >= 0; --i) | |
| Out[i] = In[i - S1] + In[i - 1] + In[i] + In[i + 1] + In[i + S1]; | |
| ); | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "reverse stencil, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "reverse stencil, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "reverse stencil, uint64_t", Strides); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i] + In[i + S1] + In[i + 2 * S1]; | |
| ); | |
| // Positive and negative S1 give forward-only and backward-only windows. | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "one-sided stencil, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "one-sided stencil, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "one-sided stencil, uint64_t", Strides); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i] + In[i - S1] + In[i - 2 * S1]; | |
| ); | |
| // Positive S1 gives a backward-only window. | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "negative one-sided stencil, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "negative one-sided stencil, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "negative one-sided stencil, uint64_t", Strides); | |
| } | |
| { | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - 2 * S1 - 1] + In[i - S1 - 1] + In[i] + | |
| In[i + S1 + 1] + In[i + 2 * S1 + 1] + In[i + S2 - 1]; | |
| ); | |
| checkTwoStrides<uint8_t>( | |
| ScalarFn, VectorFn, "stencil with mixed offsets, uint8_t", StridePairs); | |
| checkTwoStrides<uint32_t>( | |
| ScalarFn, VectorFn, "stencil with mixed offsets, uint32_t", | |
| StridePairs); | |
| checkTwoStrides<uint64_t>( | |
| ScalarFn, VectorFn, "stencil with mixed offsets, uint64_t", | |
| StridePairs); | |
| } | |
| { | |
| // Predicated reads make stencil merging bail out at compile time. | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) { | |
| if (i % 3 != 0) | |
| Out[i] = In[i - S1] + In[i] + In[i + S1]; | |
| } | |
| ); | |
| checkOneStride<uint8_t>( | |
| ScalarFn, VectorFn, "stencil with predicated reads, uint8_t", Strides); | |
| checkOneStride<uint32_t>( | |
| ScalarFn, VectorFn, "stencil with predicated reads, uint32_t", Strides); | |
| checkOneStride<uint64_t>( | |
| ScalarFn, VectorFn, "stencil with predicated reads, uint64_t", Strides); | |
| } | |
| { | |
| // Use signed strides of different widths with ordinary integer promotions. | |
| DEFINE_SCALAR_AND_VECTOR_STENCIL_FN( | |
| for (int64_t i = 0; i < TC; i++) | |
| Out[i] = In[i - 2 * S1] + In[i - S1] + In[i] + In[i + S1] + | |
| In[i + 2 * S1]; | |
| ); | |
| const std::initializer_list<int8_t> I8Strides = { | |
| -128, -16, -1, 0, 1, 16, 100, 127}; | |
| checkOneStride<uint8_t, int8_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint8_t", | |
| I8Strides); | |
| checkOneStride<uint32_t, int8_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint32_t", | |
| I8Strides); | |
| checkOneStride<uint64_t, int8_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint64_t", | |
| I8Strides); | |
| const std::initializer_list<int32_t> I32Strides = {-16, -1, 0, 1, 16, 100}; | |
| checkOneStride<uint8_t, int32_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint8_t", | |
| I32Strides); | |
| checkOneStride<uint32_t, int32_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint32_t", | |
| I32Strides); | |
| checkOneStride<uint64_t, int32_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint64_t", | |
| I32Strides); | |
| const std::initializer_list<int64_t> I64Strides = {-16, -1, 0, 1, 16, 100}; | |
| checkOneStride<uint8_t, int64_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint8_t", | |
| I64Strides); | |
| checkOneStride<uint32_t, int64_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint32_t", | |
| I64Strides); | |
| checkOneStride<uint64_t, int64_t>( | |
| ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint64_t", | |
| I64Strides); | |
| } | |
| return 0; | |
| } |