| // This program tests performance impact of Interleaving Count with varying loop | |
| // iteration count for different types of loops, such as loops with or | |
| // without reductions inside it, loops with different vectorization widths. | |
| #include <iostream> | |
| #include <memory> | |
| #include <random> | |
| #include "benchmark/benchmark.h" | |
| #define ELEMENTS 2048 | |
| #define ALIGNED16 __attribute__((aligned(16))) | |
| static std::mt19937 rng; | |
| unsigned int g_sum = 0; | |
| int A[ELEMENTS] ALIGNED16; | |
| int B[ELEMENTS] ALIGNED16; | |
| int C[ELEMENTS] ALIGNED16; | |
| int D[ELEMENTS] ALIGNED16; | |
| int E[ELEMENTS] ALIGNED16; | |
| int F[ELEMENTS] ALIGNED16; | |
| // Initialize arrays with random numbers. | |
| static void init_data(unsigned N) { | |
| std::uniform_int_distribution<int> distrib(std::numeric_limits<int>::min(), | |
| std::numeric_limits<int>::max()); | |
| for (unsigned I = 0; I < N; I++) { | |
| A[I] = distrib(rng); | |
| B[I] = distrib(rng); | |
| C[I] = distrib(rng); | |
| D[I] = distrib(rng); | |
| E[I] = distrib(rng); | |
| F[I] = distrib(rng); | |
| } | |
| } | |
| static void __attribute__((always_inline)) | |
| runBenchForLoopInterleaving(benchmark::State &state, int (*Fn)(int), | |
| int Iterations) { | |
| std::uniform_int_distribution<int> distrib(std::numeric_limits<int>::min(), | |
| std::numeric_limits<int>::max()); | |
| init_data(ELEMENTS); | |
| for (auto _ : state) { | |
| benchmark::DoNotOptimize(A); | |
| benchmark::DoNotOptimize(B); | |
| benchmark::DoNotOptimize(C); | |
| benchmark::DoNotOptimize(D); | |
| benchmark::DoNotOptimize(E); | |
| benchmark::DoNotOptimize(F); | |
| benchmark::ClobberMemory(); | |
| g_sum += Fn(Iterations); | |
| } | |
| } | |
| #define STRINGIFY(a) #a | |
| // Loops without Reduction with different vectorization configurations | |
| static int __attribute__((noinline)) loopNoReductionAutoVec(int Iterations) { | |
| #pragma clang loop unroll(disable) | |
| for (int J = 0; J < Iterations; J++) { | |
| A[J] = B[J] + C[J]; | |
| } | |
| return 0; | |
| } | |
| static int __attribute__((noinline)) bigLoopNoReductionAutoVec(int Iterations) { | |
| #pragma clang loop unroll(disable) | |
| for (int J = 0; J < Iterations; J++) { | |
| A[J] = B[J] + C[J]; | |
| D[J]++; | |
| E[J] *= 2; | |
| F[J] /= 5; | |
| } | |
| return 0; | |
| } | |
| #define loopNoReductionWithVecHint(vw, ic) \ | |
| static int __attribute__((noinline)) \ | |
| loopWithVW##vw##IC##ic(int Iterations) { \ | |
| _Pragma(STRINGIFY(clang loop vectorize_width(vw) interleave_count( \ | |
| ic))) for (int J = 0; J < Iterations; J++) { \ | |
| A[J] = B[J] + C[J]; \ | |
| } \ | |
| return 0; \ | |
| } | |
| #define bigLoopNoReductionWithVecHint(vw, ic) \ | |
| static int __attribute__((noinline)) \ | |
| bigLoopWithVW##vw##IC##ic(int Iterations) { \ | |
| _Pragma(STRINGIFY(clang loop vectorize_width(vw) interleave_count( \ | |
| ic))) for (int J = 0; J < Iterations; J++) { \ | |
| A[J] = B[J] + C[J]; \ | |
| D[J]++; \ | |
| E[J] *= 2; \ | |
| F[J] /= 5; \ | |
| } \ | |
| return 0; \ | |
| } | |
| // Loops with Reduction with different vectorization configurations | |
| static int __attribute__((noinline)) loopWithReductionAutoVec(int Iterations) { | |
| unsigned sum = 0; | |
| #pragma clang loop unroll(disable) | |
| for (int J = 0; J < Iterations; J++) { | |
| sum += A[J]; | |
| } | |
| return sum; | |
| } | |
| static int __attribute__((noinline)) | |
| bigLoopWithReductionAutoVec(int Iterations) { | |
| unsigned sum = 0; | |
| #pragma clang loop unroll(disable) | |
| for (int J = 0; J < Iterations; J++) { | |
| sum += A[J]; | |
| D[J]++; | |
| E[J] *= 2; | |
| F[J] /= 5; | |
| } | |
| return sum; | |
| } | |
| #define loopWithReductionWithVecHint(vw, ic) \ | |
| static int __attribute__((noinline)) \ | |
| loopWithReductionWithVW##vw##IC##ic(int Iterations) { \ | |
| unsigned sum = 0; \ | |
| _Pragma(STRINGIFY(clang loop vectorize_width(vw) interleave_count( \ | |
| ic))) for (int J = 0; J < Iterations; J++) { \ | |
| sum += A[J]; \ | |
| } \ | |
| return sum; \ | |
| } | |
| #define bigLoopWithReductionWithVecHint(vw, ic) \ | |
| static int __attribute__((noinline)) \ | |
| bigLoopWithReductionWithVW##vw##IC##ic(int Iterations) { \ | |
| unsigned sum = 0; \ | |
| _Pragma(STRINGIFY(clang loop vectorize_width(vw) interleave_count( \ | |
| ic))) for (int J = 0; J < Iterations; J++) { \ | |
| sum += A[J]; \ | |
| D[J]++; \ | |
| E[J] *= 2; \ | |
| F[J] /= 5; \ | |
| } \ | |
| return sum; \ | |
| } | |
| // We are evaluating 4 types of loops for different vectorization configurations | |
| // 1) Loops without reductions | |
| // 2) Loops with reductions | |
| // 3) Bigger loop bodies without reductions | |
| // 4) Bigger loop bodies with some reductions | |
| // For each, we are evaluating the following vectorization configurations of | |
| // vectorization width (VW), interleaving count (IC): | |
| // 1) automatically selected by the compiler (without vectorization hint) | |
| // 2) VW=4, IC=1 | |
| // 3) VW=4, IC=2 | |
| // 4) VW=4, IC=4 | |
| // 5) VW=1, IC=1 | |
| // 6) VW=1, IC=2 | |
| // 7) VW=1, IC=4 | |
| // Of these, configurations 5-7 are skipped for loop type 1 & 3). | |
| // Creating a function for the above configurations with different Vectorization | |
| // Hints: | |
| loopNoReductionWithVecHint(4, 1); | |
| loopNoReductionWithVecHint(4, 2); | |
| loopNoReductionWithVecHint(4, 4); | |
| loopWithReductionWithVecHint(4, 1); | |
| loopWithReductionWithVecHint(4, 2); | |
| loopWithReductionWithVecHint(4, 4); | |
| loopWithReductionWithVecHint(1, 1); | |
| loopWithReductionWithVecHint(1, 2); | |
| loopWithReductionWithVecHint(1, 4); | |
| bigLoopNoReductionWithVecHint(4, 1); | |
| bigLoopNoReductionWithVecHint(4, 2); | |
| bigLoopNoReductionWithVecHint(4, 4); | |
| bigLoopWithReductionWithVecHint(4, 1); | |
| bigLoopWithReductionWithVecHint(4, 2); | |
| bigLoopWithReductionWithVecHint(4, 4); | |
| bigLoopWithReductionWithVecHint(1, 1); | |
| bigLoopWithReductionWithVecHint(1, 2); | |
| bigLoopWithReductionWithVecHint(1, 4); | |
| #define ADD_BENCHMARK(Itr) \ | |
| void benchAutoVecForLoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopNoReductionAutoVec, Itr); \ | |
| } \ | |
| BENCHMARK(benchAutoVecForLoopTC##Itr); \ | |
| void benchForIC1VW4LoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithVW4IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW4LoopTC##Itr); \ | |
| void benchForIC2VW4LoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithVW4IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW4LoopTC##Itr); \ | |
| void benchForIC4VW4LoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithVW4IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW4LoopTC##Itr); \ | |
| void benchForLoopWithReductionAutoVecTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionAutoVec, Itr); \ | |
| } \ | |
| BENCHMARK(benchForLoopWithReductionAutoVecTC##Itr); \ | |
| void benchForIC1VW4LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW4IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW4LoopWithReductionTC##Itr); \ | |
| void benchForIC2VW4LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW4IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW4LoopWithReductionTC##Itr); \ | |
| void benchForIC4VW4LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW4IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW4LoopWithReductionTC##Itr); \ | |
| void benchForIC1VW1LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW1IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW1LoopWithReductionTC##Itr); \ | |
| void benchForIC2VW1LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW1IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW1LoopWithReductionTC##Itr); \ | |
| void benchForIC4VW1LoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopWithReductionWithVW1IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW1LoopWithReductionTC##Itr); \ | |
| void benchAutoVecForBigLoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &loopNoReductionAutoVec, Itr); \ | |
| } \ | |
| BENCHMARK(benchAutoVecForBigLoopTC##Itr); \ | |
| void benchForIC1VW4BigLoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithVW4IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW4BigLoopTC##Itr); \ | |
| void benchForIC2VW4BigLoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithVW4IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW4BigLoopTC##Itr); \ | |
| void benchForIC4VW4BigLoopTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithVW4IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW4BigLoopTC##Itr); \ | |
| void benchForBigLoopWithReductionAutoVecTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionAutoVec, Itr); \ | |
| } \ | |
| BENCHMARK(benchForBigLoopWithReductionAutoVecTC##Itr); \ | |
| void benchForIC1VW4BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW4IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW4BigLoopWithReductionTC##Itr); \ | |
| void benchForIC2VW4BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW4IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW4BigLoopWithReductionTC##Itr); \ | |
| void benchForIC4VW4BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW4IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW4BigLoopWithReductionTC##Itr); \ | |
| void benchForIC1VW1BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW1IC1, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC1VW1BigLoopWithReductionTC##Itr); \ | |
| void benchForIC2VW1BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW1IC2, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC2VW1BigLoopWithReductionTC##Itr); \ | |
| void benchForIC4VW1BigLoopWithReductionTC##Itr(benchmark::State &state) { \ | |
| runBenchForLoopInterleaving(state, &bigLoopWithReductionWithVW1IC4, Itr); \ | |
| } \ | |
| BENCHMARK(benchForIC4VW1BigLoopWithReductionTC##Itr); | |
| #ifdef ALL_LOOP_IC_TESTS | |
| ADD_BENCHMARK(1) | |
| ADD_BENCHMARK(2) | |
| ADD_BENCHMARK(3) | |
| ADD_BENCHMARK(4) | |
| ADD_BENCHMARK(5) | |
| ADD_BENCHMARK(6) | |
| ADD_BENCHMARK(7) | |
| ADD_BENCHMARK(8) | |
| ADD_BENCHMARK(9) | |
| ADD_BENCHMARK(10) | |
| ADD_BENCHMARK(11) | |
| ADD_BENCHMARK(12) | |
| ADD_BENCHMARK(13) | |
| ADD_BENCHMARK(14) | |
| ADD_BENCHMARK(15) | |
| ADD_BENCHMARK(16) | |
| ADD_BENCHMARK(17) | |
| ADD_BENCHMARK(18) | |
| ADD_BENCHMARK(19) | |
| ADD_BENCHMARK(20) | |
| ADD_BENCHMARK(21) | |
| ADD_BENCHMARK(22) | |
| ADD_BENCHMARK(23) | |
| ADD_BENCHMARK(24) | |
| ADD_BENCHMARK(25) | |
| ADD_BENCHMARK(26) | |
| ADD_BENCHMARK(27) | |
| ADD_BENCHMARK(28) | |
| ADD_BENCHMARK(29) | |
| ADD_BENCHMARK(30) | |
| ADD_BENCHMARK(31) | |
| ADD_BENCHMARK(32) | |
| ADD_BENCHMARK(33) | |
| ADD_BENCHMARK(34) | |
| ADD_BENCHMARK(35) | |
| ADD_BENCHMARK(36) | |
| ADD_BENCHMARK(37) | |
| ADD_BENCHMARK(38) | |
| ADD_BENCHMARK(39) | |
| ADD_BENCHMARK(40) | |
| ADD_BENCHMARK(41) | |
| ADD_BENCHMARK(42) | |
| ADD_BENCHMARK(43) | |
| ADD_BENCHMARK(44) | |
| ADD_BENCHMARK(45) | |
| ADD_BENCHMARK(46) | |
| ADD_BENCHMARK(47) | |
| ADD_BENCHMARK(48) | |
| ADD_BENCHMARK(49) | |
| ADD_BENCHMARK(50) | |
| ADD_BENCHMARK(51) | |
| ADD_BENCHMARK(52) | |
| ADD_BENCHMARK(53) | |
| ADD_BENCHMARK(54) | |
| ADD_BENCHMARK(55) | |
| ADD_BENCHMARK(56) | |
| ADD_BENCHMARK(57) | |
| ADD_BENCHMARK(58) | |
| ADD_BENCHMARK(59) | |
| ADD_BENCHMARK(60) | |
| ADD_BENCHMARK(61) | |
| ADD_BENCHMARK(62) | |
| ADD_BENCHMARK(63) | |
| ADD_BENCHMARK(64) | |
| ADD_BENCHMARK(65) | |
| ADD_BENCHMARK(66) | |
| ADD_BENCHMARK(67) | |
| ADD_BENCHMARK(68) | |
| ADD_BENCHMARK(69) | |
| ADD_BENCHMARK(70) | |
| ADD_BENCHMARK(71) | |
| ADD_BENCHMARK(72) | |
| ADD_BENCHMARK(73) | |
| ADD_BENCHMARK(74) | |
| ADD_BENCHMARK(75) | |
| ADD_BENCHMARK(76) | |
| ADD_BENCHMARK(77) | |
| ADD_BENCHMARK(78) | |
| ADD_BENCHMARK(79) | |
| ADD_BENCHMARK(80) | |
| ADD_BENCHMARK(81) | |
| ADD_BENCHMARK(82) | |
| ADD_BENCHMARK(83) | |
| ADD_BENCHMARK(84) | |
| ADD_BENCHMARK(85) | |
| ADD_BENCHMARK(86) | |
| ADD_BENCHMARK(87) | |
| ADD_BENCHMARK(88) | |
| ADD_BENCHMARK(89) | |
| ADD_BENCHMARK(90) | |
| ADD_BENCHMARK(91) | |
| ADD_BENCHMARK(92) | |
| ADD_BENCHMARK(93) | |
| ADD_BENCHMARK(94) | |
| ADD_BENCHMARK(95) | |
| ADD_BENCHMARK(96) | |
| ADD_BENCHMARK(97) | |
| ADD_BENCHMARK(98) | |
| ADD_BENCHMARK(99) | |
| ADD_BENCHMARK(100) | |
| ADD_BENCHMARK(101) | |
| ADD_BENCHMARK(102) | |
| ADD_BENCHMARK(103) | |
| ADD_BENCHMARK(104) | |
| ADD_BENCHMARK(105) | |
| ADD_BENCHMARK(106) | |
| ADD_BENCHMARK(107) | |
| ADD_BENCHMARK(108) | |
| ADD_BENCHMARK(109) | |
| ADD_BENCHMARK(110) | |
| ADD_BENCHMARK(111) | |
| ADD_BENCHMARK(112) | |
| ADD_BENCHMARK(113) | |
| ADD_BENCHMARK(114) | |
| ADD_BENCHMARK(115) | |
| ADD_BENCHMARK(116) | |
| ADD_BENCHMARK(117) | |
| ADD_BENCHMARK(118) | |
| ADD_BENCHMARK(119) | |
| ADD_BENCHMARK(120) | |
| ADD_BENCHMARK(121) | |
| ADD_BENCHMARK(122) | |
| ADD_BENCHMARK(123) | |
| ADD_BENCHMARK(124) | |
| ADD_BENCHMARK(125) | |
| ADD_BENCHMARK(126) | |
| ADD_BENCHMARK(127) | |
| ADD_BENCHMARK(128) | |
| #else | |
| ADD_BENCHMARK(1) | |
| ADD_BENCHMARK(2) | |
| ADD_BENCHMARK(3) | |
| ADD_BENCHMARK(4) | |
| ADD_BENCHMARK(7) | |
| ADD_BENCHMARK(8) | |
| ADD_BENCHMARK(15) | |
| ADD_BENCHMARK(16) | |
| ADD_BENCHMARK(31) | |
| ADD_BENCHMARK(32) | |
| ADD_BENCHMARK(63) | |
| ADD_BENCHMARK(64) | |
| ADD_BENCHMARK(127) | |
| ADD_BENCHMARK(128) | |
| #endif |