SingleSource/UnitTests/Vectorizer/VPlanNativePath/outer-loop-vect.cpp

#include <cstddef>
#include <cstdint>
#include <iostream>
#include <memory>

#include "../common.h"

// Tests for outer-loop vectorization in LLVM's VPlan-native path.

#define DEFINE_SCALAR_AND_VECTOR_FN_FOR_OLV(Args, Loop)                       \
  auto ScalarFn = [] Args {                                                   \
    _Pragma("clang loop vectorize(disable) interleave_count(1)") Loop         \
  };                                                                          \
  auto VectorFn = [] Args {                                                   \
    _Pragma("clang loop vectorize(enable)") Loop                              \
  };

#define DEFINE_SCALAR_AND_VECTOR_FN_FOR_NESTED_OLV(Args, Loop)                \
  auto ScalarFn = [] Args {                                                   \
    for (size_t I = 0; I < N; I++) {                                          \
      _Pragma("clang loop vectorize(disable) interleave_count(1)") Loop       \
    }                                                                         \
  };                                                                          \
  auto VectorFn = [] Args {                                                   \
    for (size_t I = 0; I < N; I++) {                                          \
      _Pragma("clang loop vectorize(enable)") Loop                            \
    }                                                                         \
  };

int main() {
  rng = std::mt19937(15);

  {
    // A matrix-multiplication where the second loop of the triple loop nest
    // is vectorized (the macro adds the outer-most loop).
    DEFINE_SCALAR_AND_VECTOR_FN_FOR_NESTED_OLV(
      (size_t N, size_t M, size_t L,
       int32_t *__restrict__ A, const int32_t *B, const int32_t *C),
      for (size_t J = 0; J < L; J++) {
        int32_t X = 0;
        for (size_t K = 0; K < M; K++)
          X += B[I * M + K] * C[K * L + J];
        A[I * L + J] = X;
      });

    std::cout << "Checking matrix-multiplication\n";

    size_t N = 100, M = 100, L = 100;
    std::unique_ptr<int32_t[]> A_Reference(new int32_t[N * L]);
    std::unique_ptr<int32_t[]> A_ToCheck(new int32_t[N * L]);
    std::unique_ptr<int32_t[]> B(new int32_t[N * M]);
    std::unique_ptr<int32_t[]> C(new int32_t[M * L]);
    init_data(B, N * M);
    init_data(C, M * L);

    ScalarFn(N, M, L, &A_Reference[0], &B[0], &C[0]);
    VectorFn(N, M, L, &A_ToCheck[0], &B[0], &C[0]);
    check(A_Reference, A_ToCheck, N*L);
  }

  {
    // A test where the vectorized loop itself has an auxiliary IV.
    DEFINE_SCALAR_AND_VECTOR_FN_FOR_OLV(
      (size_t N, int32_t *__restrict__ A, const int32_t *B),
      for (size_t I = 0, AuxIV = 333; I < N; I++, AuxIV += 12) {
        int32_t X = B[I];
        for (size_t J = 0; J < N; J++) {
          X += AuxIV * B[J];
        }
        A[I] = AuxIV + X;
      });

    std::cout << "Checking loop with auxiliary IV\n";

    size_t N = 123;
    std::unique_ptr<int32_t[]> A_Reference(new int32_t[N]);
    std::unique_ptr<int32_t[]> A_ToCheck(new int32_t[N]);
    std::unique_ptr<int32_t[]> B(new int32_t[N]);
    init_data(B, N);

    ScalarFn(N, &A_Reference[0], &B[0]);
    VectorFn(N, &A_ToCheck[0], &B[0]);
    check(A_Reference, A_ToCheck, N);
  }

  {
    // A test for irregular memory accesses patterns.
    DEFINE_SCALAR_AND_VECTOR_FN_FOR_OLV(
      (size_t N, size_t M,
       int32_t *__restrict__ A, const int32_t *B, const int32_t *C),
      for (size_t I = 0; I < N; I++) {
        int32_t X = 0;
        for (size_t J = 0; J < M / 2; J++) {
          int32_t Idx = C[J * 2];
          X += B[Idx % N];
        }
        A[I] = X;
      })

    std::cout << "Checking loop with indirect memory accesses\n";

    size_t N = 123, M = 456;
    std::unique_ptr<int32_t[]> A_Reference(new int32_t[N]);
    std::unique_ptr<int32_t[]> A_ToCheck(new int32_t[N]);
    std::unique_ptr<int32_t[]> B(new int32_t[N]);
    std::unique_ptr<int32_t[]> C(new int32_t[M]);
    init_data(B, N);
    init_data(C, M);

    ScalarFn(N, M, &A_Reference[0], &B[0], &C[0]);
    VectorFn(N, M, &A_ToCheck[0], &B[0], &C[0]);
    check(A_Reference, A_ToCheck, N);
  }

  {
    // A test where the vectorized loop contains a loop which contains
    // another loop itself.
    DEFINE_SCALAR_AND_VECTOR_FN_FOR_OLV(
      (size_t N, size_t M, size_t L, int32_t *__restrict__ A),
      for (size_t I = 0; I < N; I++) {
        for (size_t J = 0; J < M; J++) {
          for (size_t K = 0; K < L; K++) {
            A[I * (M * L) + J * L + K] = I * J * K;
          }
        }
      });

    std::cout << "Checking triple-loop-nest\n";

    size_t N = 123, M = 45, L = 67;
    std::unique_ptr<int32_t[]> A_Reference(new int32_t[N * M * L]);
    std::unique_ptr<int32_t[]> A_ToCheck(new int32_t[N * M * L]);

    ScalarFn(N, M, L, &A_Reference[0]);
    VectorFn(N, M, L, &A_ToCheck[0]);
    check(A_Reference, A_ToCheck, N * M * L);
  }

}