blob: 7fdc6f7c94047f4511a5500b21e27f5c059ed4b3 [file] [edit]
#include <algorithm>
#include <cstdlib>
#include <functional>
#include <initializer_list>
#include <iostream>
#include <limits>
#include <memory>
#include <random>
#include <stdint.h>
#include <utility>
#include "common.h"
// Tests for memory runtime checks generated by the vectorizer. Runs scalar and
// vectorized versions of a loop requiring runtime checks on the same inputs
// with pointers to the same buffer using various offsets between reads and
// writes. Fails if they do not produce the same results.
template <typename Ty>
static void check(const std::unique_ptr<Ty[]> &Reference,
const std::unique_ptr<Ty[]> &Tmp, unsigned NumElements,
int Offset) {
if (!std::equal(&Reference[0], &Reference[0] + NumElements, &Tmp[0])) {
std::cerr << "Miscompare with offset " << Offset << "\n";
exit(1);
}
}
// Helper to call \p f with \p args and acts as optimization barrier for \p f.
template <typename F, typename... Args>
__attribute__((optnone)) static void callThroughOptnone(F &&f, Args &&...args) {
f(std::forward<Args>(args)...);
}
template <typename Ty> using Fn2Ty = std::function<void(Ty *, Ty *, unsigned)>;
// Run both \p ScalarFn and \p VectorFn on the same inputs with pointers to the
// same buffer. Fail if they do not produce the same results.
template <typename Ty>
static void checkOverlappingMemoryOneRuntimeCheck(Fn2Ty<Ty> ScalarFn,
Fn2Ty<Ty> VectorFn,
const char *Name) {
std::cout << "Checking " << Name << "\n";
unsigned N = 100;
// Make sure we have enough extra elements so we can be liberal with offsets.
unsigned NumArrayElements = N * 8;
std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]);
auto CheckWithOffset = [&](int Offset) {
init_data(Input1, NumArrayElements);
for (unsigned i = 0; i < NumArrayElements; i++) {
Reference[i] = Input1[i];
ToCheck[i] = Input1[i];
}
// Run scalar function to generate reference output.
Ty *ReferenceStart = &Reference[NumArrayElements / 2];
ScalarFn(ReferenceStart + Offset, ReferenceStart, N);
// Run vector function to generate output to check.
Ty *StartPtr = &ToCheck[NumArrayElements / 2];
callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, N);
// Compare scalar and vector output.
check(Reference, ToCheck, NumArrayElements, Offset);
};
for (int i = -100; i <= 100; i++)
CheckWithOffset(i);
}
template <typename Ty>
using Fn3Ty = std::function<void(Ty *, Ty *, Ty *, unsigned)>;
template <typename Ty>
static void checkOverlappingMemoryTwoRuntimeChecks(Fn3Ty<Ty> ScalarFn,
Fn3Ty<Ty> VectorFn,
const char *Name) {
std::cout << "Checking " << Name << "\n";
unsigned N = 100;
// Make sure we have enough extra elements so we can be liberal with offsets.
unsigned NumArrayElements = N * 8;
std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> Input2(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]);
auto CheckWithOffsetSecond = [&](int Offset) {
init_data(Input1, NumArrayElements);
init_data(Input2, NumArrayElements);
for (unsigned i = 0; i < NumArrayElements; i++) {
Reference[i] = Input1[i];
ToCheck[i] = Input1[i];
}
// Run scalar function to generate reference output.
Ty *ReferenceStart = &Reference[NumArrayElements / 2];
ScalarFn(ReferenceStart + Offset, &Input2[0], ReferenceStart, N);
// Run vector function to generate output to check.
Ty *StartPtr = &ToCheck[NumArrayElements / 2];
callThroughOptnone(VectorFn, StartPtr + Offset, &Input2[0], StartPtr, N);
// Compare scalar and vector output.
check(Reference, ToCheck, NumArrayElements, Offset);
};
for (int i = -100; i <= 100; i++)
CheckWithOffsetSecond(i);
}
template <typename Ty>
using Fn4Ty = std::function<void(Ty *, Ty *, unsigned, unsigned)>;
template <typename Ty>
static void checkOverlappingMemoryTwoRuntimeChecksNested(Fn4Ty<Ty> ScalarFn,
Fn4Ty<Ty> VectorFn,
const int OuterTC,
const int InnerTC,
const char *Name) {
std::cout << "Checking " << Name << "\n";
// Make sure we have enough extra elements so we can be liberal with offsets.
const unsigned NumArrayElements = (InnerTC * (OuterTC + 1)) * 8;
std::unique_ptr<Ty[]> Input1(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]);
auto CheckWithOffsetSecond = [&](int Offset) {
init_data(Input1, NumArrayElements);
for (unsigned i = 0; i < NumArrayElements; i++) {
Reference[i] = Input1[i];
ToCheck[i] = Input1[i];
}
// Run scalar function to generate reference output.
Ty *ReferenceStart = &Reference[NumArrayElements / 2];
ScalarFn(ReferenceStart + Offset, ReferenceStart, OuterTC, InnerTC);
// Run vector function to generate output to check.
Ty *StartPtr = &ToCheck[NumArrayElements / 2];
callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, OuterTC, InnerTC);
// Compare scalar and vector output.
check(Reference, ToCheck, NumArrayElements, Offset);
};
// With a nested loop, sometimes the runtime checks will fail and sometimes
// succeed. For example, with large offsets you'd expect for the first and
// last one or two executions of the inner loop there is no overlap.
for (int i = -(2 * (InnerTC + 1)); i <= (2 * (InnerTC + 1)); i++)
CheckWithOffsetSecond(i);
}
#define DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(Loop) \
auto ScalarFn = [](auto *Out, auto *In, unsigned TC, auto S1, auto S2) { \
_Pragma("clang loop vectorize(disable) interleave_count(1)") Loop \
}; \
auto VectorFn = [](auto *Out, auto *In, unsigned TC, auto S1, auto S2) { \
_Pragma("clang loop vectorize(enable)") Loop \
};
template <typename Ty, typename StrideTy>
using StencilFnTy =
std::function<void(Ty *, Ty *, unsigned, StrideTy, StrideTy)>;
template <typename Ty, typename StrideTy>
static void checkOverlappingMemoryStencil(StencilFnTy<Ty, StrideTy> ScalarFn,
StencilFnTy<Ty, StrideTy> VectorFn,
const char *Name, StrideTy S1,
StrideTy S2) {
std::cout << "Checking " << Name << ", strides " << +S1 << " and " << +S2
<< "\n";
unsigned N = 100;
// Conservative bound on the read offsets for the stencil loops below.
unsigned Reach = 2 * (std::abs(long(S1)) + std::abs(long(S2))) + 1;
// Cover all possible overlap and include disjoint accesses. Test at least
// offsets -100 to 100.
int MaxOffset = std::max(100u, Reach + N + 2);
// The buffer must hold Out at every offset and every read of In.
unsigned Margin = MaxOffset + N;
unsigned NumArrayElements = 2 * Margin + 1;
std::unique_ptr<Ty[]> Input(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> Reference(new Ty[NumArrayElements]);
std::unique_ptr<Ty[]> ToCheck(new Ty[NumArrayElements]);
init_data(Input, NumArrayElements);
auto CheckWithOffset = [&](int Offset, unsigned TC) {
for (unsigned i = 0; i < NumArrayElements; i++) {
Reference[i] = Input[i];
ToCheck[i] = Input[i];
}
// Run scalar function to generate reference output.
Ty *ReferenceStart = &Reference[Margin];
ScalarFn(ReferenceStart + Offset, ReferenceStart, TC, S1, S2);
// Run vector function to generate output to check.
Ty *StartPtr = &ToCheck[Margin];
callThroughOptnone(VectorFn, StartPtr + Offset, StartPtr, TC, S1, S2);
// Compare scalar and vector output.
check(Reference, ToCheck, NumArrayElements, Offset);
};
// Include empty and short loops as well as a non-multiple tail. The strides
// and trip count stay runtime arguments of the vector function.
for (unsigned TC : {0u, 1u, 2u, 7u, N - 1, N})
for (int Offset = -MaxOffset; Offset <= MaxOffset; Offset++)
CheckWithOffset(Offset, TC);
}
// Check one element type with one stride. The loops get 0 as their second stride
// and do not use it.
template <typename Ty, typename StrideTy, typename ScalarFnTy,
typename VectorFnTy>
static void checkOneStride(ScalarFnTy ScalarFn, VectorFnTy VectorFn,
const char *Name,
std::initializer_list<StrideTy> Strides) {
for (StrideTy S : Strides)
checkOverlappingMemoryStencil<Ty, StrideTy>(ScalarFn, VectorFn, Name, S,
StrideTy(0));
}
// Check one element type with two strides.
template <typename Ty, typename StrideTy, typename ScalarFnTy,
typename VectorFnTy>
static void checkTwoStrides(
ScalarFnTy ScalarFn, VectorFnTy VectorFn, const char *Name,
std::initializer_list<std::pair<StrideTy, StrideTy>> Strides) {
for (auto [S1, S2] : Strides)
checkOverlappingMemoryStencil<Ty, StrideTy>(ScalarFn, VectorFn, Name, S1, S2);
}
int main(void) {
rng = std::mt19937(15);
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = 0; i < TC; i++)
A[i] = B[i] + 10;
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, step 1, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(ScalarFn, VectorFn,
"1 read, 1 write, step 1, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write, step 1, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = 0; i < TC; i++)
A[i] = B[i + 3] + 10;
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset 3, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = 3; i < TC; i++)
A[i] = B[i - 3] + 10;
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write, offset -3, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = TC; i > 0; i--)
A[i] = B[i] + 10;);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = TC; i > 2; i -= 2)
A[i] = B[i] + 10;
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write, index count down 2, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = 0, j = 0; i < TC; i++) {
A[i] = B[j] + 10;
j += 2;
}
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn,
"1 read, 1 write, 2 inductions, different steps, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn,
"1 read, 1 write, 2 inductions, different steps, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn,
"1 read, 1 write, 2 inductions, different steps, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
_Pragma("clang loop unroll(disable)")
for (unsigned i = 0; i < TC; i += 2)
A[i] = B[i] + 10;
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write, induction increment 2, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write, induction increment 2, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn,
"1 read, 1 write, induction increment 2, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN2(,
for (unsigned i = 0, sum = 0; i < TC; i++) {
sum += A[i] + 10;
B[TC / 2] = sum;
}
);
checkOverlappingMemoryOneRuntimeCheck<uint8_t>(
ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint8_t");
checkOverlappingMemoryOneRuntimeCheck<uint32_t>(
ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint32_t");
checkOverlappingMemoryOneRuntimeCheck<uint64_t>(
ScalarFn, VectorFn, "1 read, 1 write to invariant address, step 1, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN3(
for (unsigned i = 0; i < TC; i++)
A[i] = B[i] + C[i] + 10;
);
checkOverlappingMemoryTwoRuntimeChecks<uint32_t>(
ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint32_t");
checkOverlappingMemoryTwoRuntimeChecks<uint8_t>(
ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint8_t");
checkOverlappingMemoryTwoRuntimeChecks<uint64_t>(
ScalarFn, VectorFn, "2 reads, 1 write, simple indices, uint64_t");
}
{
DEFINE_SCALAR_AND_VECTOR_FN3(
for (unsigned i = 0; i < TC; i++) {
auto X = C[i] + 10;
A[i] = X;
B[i] = X + 9;
}
);
checkOverlappingMemoryTwoRuntimeChecks<uint8_t>(
ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint8_t");
checkOverlappingMemoryTwoRuntimeChecks<uint32_t>(
ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint32_t");
checkOverlappingMemoryTwoRuntimeChecks<uint64_t>(
ScalarFn, VectorFn, "1 read, 2 writes, simple indices, uint64_t");
}
{
DEFINE_NESTED_SCALAR_AND_VECTOR_FN4(
auto X = B[(i * OuterTC) + j];
A[(i * (OuterTC + 1)) + j] = X;
);
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (matching trip counts), uint64_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 50, "1 read, 1 write, nested loop (different trip counts), uint64_t");
}
{
DEFINE_NESTED_SCALAR_AND_VECTOR_FN4(
auto X = B[(i * OuterTC) + j];
A[(i * (OuterTC + 1)) + j] += X;
);
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (matching trip counts), uint64_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 50, "2 reads, 1 write, nested loop (different trip counts), uint64_t");
}
{
DEFINE_NESTED_SCALAR_AND_VECTOR_FN5(
auto X = B[(i * OuterTC) + j];
A[(i * (OuterTC + 1)) + j] = X;
);
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 100, "1 read, 1 write, nested loop (decreasing outer iv, matching trip counts), uint64_t");
}
{
DEFINE_NESTED_SCALAR_AND_VECTOR_FN5(
auto X = B[(i * OuterTC) + j];
A[(i * (OuterTC + 1)) + j] += X;
);
checkOverlappingMemoryTwoRuntimeChecksNested<uint8_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint8_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint32_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint32_t");
checkOverlappingMemoryTwoRuntimeChecksNested<uint64_t>(
ScalarFn, VectorFn, 100, 100, "2 reads, 1 write, nested loop (decreasing outer iv, matching trip counts), uint64_t");
}
const std::initializer_list<long> Strides = {-16, -1, 0, 1, 2, 7, 16, 64, 100};
const std::initializer_list<std::pair<long, long>> StridePairs = {
{1, 2}, {2, 1}, {7, 16}, {16, 7}, {16, 100}, {100, 16}, {64, 64},
{-7, 16}, {16, -7}, {-7, -16}, {0, 16}, {16, 0}, {0, 0}};
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - S1] + In[i - 1] + In[i] + In[i + 1] + In[i + S1];
);
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "1D 5-point stencil, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "1D 5-point stencil, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "1D 5-point stencil, uint64_t", Strides);
}
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - 2 * S1] + In[i - S1] + In[i] + In[i + S1] +
In[i + 2 * S1];
);
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "stencil with scaled stride, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "stencil with scaled stride, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "stencil with scaled stride, uint64_t", Strides);
}
{
// A 3D-style stencil. S1 is the row size and S2 is the plane size.
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - S2 - S1] + In[i - S2] + In[i - S2 + S1] + In[i - S1] +
In[i] + In[i + S1] + In[i + S2 - S1] + In[i + S2] +
In[i + S2 + S1];
);
checkTwoStrides<uint8_t>(
ScalarFn, VectorFn, "3D-style stencil with two strides, uint8_t",
StridePairs);
checkTwoStrides<uint32_t>(
ScalarFn, VectorFn, "3D-style stencil with two strides, uint32_t",
StridePairs);
checkTwoStrides<uint64_t>(
ScalarFn, VectorFn, "3D-style stencil with two strides, uint64_t",
StridePairs);
}
{
// Either In[i - 2 * S1] or In[i - 2 * S2] can be the lowest read. It
// depends on the strides at runtime. In the same way, either In[i + 2 * S1]
// or In[i + 2 * S2] can be the highest read.
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - 2 * S2] + In[i - 2 * S1] + In[i - S2] + In[i - S1] +
In[i] + In[i + S1] + In[i + S2] + In[i + 2 * S1] +
In[i + 2 * S2];
);
checkTwoStrides<uint8_t>(
ScalarFn, VectorFn, "stencil with two scaled strides, uint8_t",
StridePairs);
checkTwoStrides<uint32_t>(
ScalarFn, VectorFn, "stencil with two scaled strides, uint32_t",
StridePairs);
checkTwoStrides<uint64_t>(
ScalarFn, VectorFn, "stencil with two scaled strides, uint64_t",
StridePairs);
}
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
// Convert before subtracting so TC == 0 starts at -1.
for (int64_t i = int64_t(TC) - 1; i >= 0; --i)
Out[i] = In[i - S1] + In[i - 1] + In[i] + In[i + 1] + In[i + S1];
);
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "reverse stencil, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "reverse stencil, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "reverse stencil, uint64_t", Strides);
}
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i] + In[i + S1] + In[i + 2 * S1];
);
// Positive and negative S1 give forward-only and backward-only windows.
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "one-sided stencil, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "one-sided stencil, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "one-sided stencil, uint64_t", Strides);
}
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i] + In[i - S1] + In[i - 2 * S1];
);
// Positive S1 gives a backward-only window.
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "negative one-sided stencil, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "negative one-sided stencil, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "negative one-sided stencil, uint64_t", Strides);
}
{
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - 2 * S1 - 1] + In[i - S1 - 1] + In[i] +
In[i + S1 + 1] + In[i + 2 * S1 + 1] + In[i + S2 - 1];
);
checkTwoStrides<uint8_t>(
ScalarFn, VectorFn, "stencil with mixed offsets, uint8_t", StridePairs);
checkTwoStrides<uint32_t>(
ScalarFn, VectorFn, "stencil with mixed offsets, uint32_t",
StridePairs);
checkTwoStrides<uint64_t>(
ScalarFn, VectorFn, "stencil with mixed offsets, uint64_t",
StridePairs);
}
{
// Predicated reads make stencil merging bail out at compile time.
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++) {
if (i % 3 != 0)
Out[i] = In[i - S1] + In[i] + In[i + S1];
}
);
checkOneStride<uint8_t>(
ScalarFn, VectorFn, "stencil with predicated reads, uint8_t", Strides);
checkOneStride<uint32_t>(
ScalarFn, VectorFn, "stencil with predicated reads, uint32_t", Strides);
checkOneStride<uint64_t>(
ScalarFn, VectorFn, "stencil with predicated reads, uint64_t", Strides);
}
{
// Use signed strides of different widths with ordinary integer promotions.
DEFINE_SCALAR_AND_VECTOR_STENCIL_FN(
for (int64_t i = 0; i < TC; i++)
Out[i] = In[i - 2 * S1] + In[i - S1] + In[i] + In[i + S1] +
In[i + 2 * S1];
);
const std::initializer_list<int8_t> I8Strides = {
-128, -16, -1, 0, 1, 16, 100, 127};
checkOneStride<uint8_t, int8_t>(
ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint8_t",
I8Strides);
checkOneStride<uint32_t, int8_t>(
ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint32_t",
I8Strides);
checkOneStride<uint64_t, int8_t>(
ScalarFn, VectorFn, "scaled stencil, int8_t stride, uint64_t",
I8Strides);
const std::initializer_list<int32_t> I32Strides = {-16, -1, 0, 1, 16, 100};
checkOneStride<uint8_t, int32_t>(
ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint8_t",
I32Strides);
checkOneStride<uint32_t, int32_t>(
ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint32_t",
I32Strides);
checkOneStride<uint64_t, int32_t>(
ScalarFn, VectorFn, "scaled stencil, int32_t stride, uint64_t",
I32Strides);
const std::initializer_list<int64_t> I64Strides = {-16, -1, 0, 1, 16, 100};
checkOneStride<uint8_t, int64_t>(
ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint8_t",
I64Strides);
checkOneStride<uint32_t, int64_t>(
ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint32_t",
I64Strides);
checkOneStride<uint64_t, int64_t>(
ScalarFn, VectorFn, "scaled stencil, int64_t stride, uint64_t",
I64Strides);
}
return 0;
}