[Dexter] Add support for Windows to regression test suite.
This patch addresses the issue of the regression suite not running on windows hardware. It changes the following things:
add new dexter regression suite command to lit.cfg.py that makes use of the clang-cl_vs2015 and dbgend builder and debuggers.
sprinkle the new regression suite command through the feature and tool tests that require them.
mark certain problem tests on windows
There's a couple of tests that fail (or pass) in unexpected ways on Windows.
Problem tests are both the penalty and perfect expect_watch_type.cpp tests. Type information reporting parity is not possible a this time in dexter due to the nature of how different debuggers report type information back to their users.
reviewers: Orlando
Differential Revision: https://reviews.llvm.org/D76609
GitOrigin-RevId: 89025da9f676aebff7daf055824d6fd102a70c34
diff --git a/dexter/feature_tests/commands/penalty/expect_program_state.cpp b/dexter/feature_tests/commands/penalty/expect_program_state.cpp
index 4769068..2a2355c 100644
--- a/dexter/feature_tests/commands/penalty/expect_program_state.cpp
+++ b/dexter/feature_tests/commands/penalty/expect_program_state.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectProgramState correctly applies a penalty when
// an expected program state is never found.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_program_state.cpp:
int GCD(int lhs, int rhs)
diff --git a/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp b/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
index 9612c06..a65e7f5 100644
--- a/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
+++ b/dexter/feature_tests/commands/penalty/expect_step_kinds.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectStepKind correctly applies a penalty when
// unexpected step kinds are encountered.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_step_kinds.cpp:
int abs(int i){
diff --git a/dexter/feature_tests/commands/penalty/expect_step_order.cpp b/dexter/feature_tests/commands/penalty/expect_step_order.cpp
index e52a7ff..b02dd02 100644
--- a/dexter/feature_tests/commands/penalty/expect_step_order.cpp
+++ b/dexter/feature_tests/commands/penalty/expect_step_order.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectStepOrder correctly applies a penalty for steps
// found out of expected order.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_step_order.cpp:
int main()
diff --git a/dexter/feature_tests/commands/penalty/expect_watch_type.cpp b/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
index 145eeed..c81de5f 100644
--- a/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
+++ b/dexter/feature_tests/commands/penalty/expect_watch_type.cpp
@@ -2,11 +2,11 @@
// Check that \DexExpectWatchType applies penalties when expected
// types are not found and unexpected types are.
//
-// REQUIRES: system-linux, lldb
+// NOTE: This test passes but not in the expected way on Windows.
+// TODO: Reduce this test's coverage and be more specific about
+// expected behaviour.
//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_watch_type.cpp:
template<class T>
diff --git a/dexter/feature_tests/commands/penalty/expect_watch_value.cpp b/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
index b4c60fa..cc7d2de 100644
--- a/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
+++ b/dexter/feature_tests/commands/penalty/expect_watch_value.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectWatchValue correctly applies a penalty when
// expected values are not found.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_watch_value.cpp:
int main()
diff --git a/dexter/feature_tests/commands/penalty/unreachable.cpp b/dexter/feature_tests/commands/penalty/unreachable.cpp
index 5cea1c4..3d3e76c 100644
--- a/dexter/feature_tests/commands/penalty/unreachable.cpp
+++ b/dexter/feature_tests/commands/penalty/unreachable.cpp
@@ -2,11 +2,7 @@
// Check that \DexUnreachable correctly applies a penalty if the command
// line is stepped on.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: not %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: not %dexter_regression_test -- %s | FileCheck %s
// CHECK: unreachable.cpp:
int
diff --git a/dexter/feature_tests/commands/perfect/expect_program_state.cpp b/dexter/feature_tests/commands/perfect/expect_program_state.cpp
index ebb4766..7137998 100644
--- a/dexter/feature_tests/commands/perfect/expect_program_state.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_program_state.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectWatchValue applies no penalties when expected
// program states are found.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -glldb" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_program_state.cpp:
int GCD(int lhs, int rhs)
diff --git a/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp b/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
index 3ff0e9f..6f4e85f 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_kind/direction.cpp
@@ -5,11 +5,10 @@
// 'VERTICAL_FORWARD' for every step onto a greater source line number in
// the same function.
//
-// REQUIRES: system-linux, lldb
+// TODO: The dbgeng debugger does not support column step reporting at present.
+// XFAIL: system-windows
//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: direction.cpp:
int func(int i) {
diff --git a/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp b/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
index d5ab3ff..c845c4d 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_kind/func.cpp
@@ -3,11 +3,7 @@
// trivial test. Expect one 'FUNC' per call to a function which is defined
// in one of the source files in the test directory.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: func.cpp:
int func(int i) {
diff --git a/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp b/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
index 6dff5c8..b496b3b 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_kind/func_external.cpp
@@ -2,13 +2,11 @@
// Check that \DexExpectStepKind correctly counts 'FUNC_EXTERNAL' steps
// for a trivial test. Expect one 'FUNC_EXTERNAL' per external call.
//
-// REQUIRES: system-linux, lldb
-// XFAIL: system-linux
-// This fails right now on my linux machine, needs examining as to why.
+// XFAIL:*
+// This fails right now on my linux and windows machine, needs examining as to
+// why.
//
-// RUN: %dexter --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: func_external.cpp:
#include <cstdlib>
diff --git a/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp b/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
index f2b54e4..31dc6c6 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_kind/recursive.cpp
@@ -3,11 +3,7 @@
// Specifically, ensure recursive calls count towards 'FUNC' and not
// 'VERTICAL_BACKWARD'.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: recursive.cpp:
int func(int i) {
diff --git a/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp b/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
index 1f90607..63bb197 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_kind/small_loop.cpp
@@ -3,11 +3,7 @@
// where the last source line in the loop is a call. Expect steps out
// of a function to a line before the call to count as 'VERTICAL_BACKWARD'.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: small_loop.cpp:
int func(int i){
diff --git a/dexter/feature_tests/commands/perfect/expect_step_order.cpp b/dexter/feature_tests/commands/perfect/expect_step_order.cpp
index a89a0ce..e24d771 100644
--- a/dexter/feature_tests/commands/perfect/expect_step_order.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_step_order.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectStepOrder applies no penalty when the expected
// order is found.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_step_order.cpp:
int main()
diff --git a/dexter/feature_tests/commands/perfect/expect_watch_type.cpp b/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
index 4159383..d3ef79b 100644
--- a/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_watch_type.cpp
@@ -2,11 +2,11 @@
// Check that \DexExpectWatchType applies no penalties when expected
// types are found.
//
-// REQUIRES: system-linux, lldb
+// TODO: On Windows WITH dbgeng, This test takes a long time to run and doesn't evaluate type values
+// in the same manner as LLDB.
+// XFAIL: system-windows
//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_watch_type.cpp:
template<class T>
diff --git a/dexter/feature_tests/commands/perfect/expect_watch_value.cpp b/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
index 035e411..b86cc31 100644
--- a/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
+++ b/dexter/feature_tests/commands/perfect/expect_watch_value.cpp
@@ -2,11 +2,7 @@
// Check that \DexExpectWatchValue applies no penalties when expected
// values are found.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: expect_watch_value.cpp:
unsigned long Factorial(int n) {
diff --git a/dexter/feature_tests/commands/perfect/unreachable.cpp b/dexter/feature_tests/commands/perfect/unreachable.cpp
index b158b48..cf889f8 100644
--- a/dexter/feature_tests/commands/perfect/unreachable.cpp
+++ b/dexter/feature_tests/commands/perfect/unreachable.cpp
@@ -2,11 +2,7 @@
// Check that \DexUnreachable has no effect if the command line is never
// stepped on.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" -- %s \
-// RUN: | FileCheck %s
+// RUN: %dexter_regression_test -- %s | FileCheck %s
// CHECK: unreachable.cpp:
int main()
diff --git a/dexter/feature_tests/subtools/view.cpp b/dexter/feature_tests/subtools/view.cpp
index 43598db..03e0b94 100644
--- a/dexter/feature_tests/subtools/view.cpp
+++ b/dexter/feature_tests/subtools/view.cpp
@@ -1,11 +1,7 @@
// Purpose:
// Check the `view` subtool works with typical inputs.
//
-// REQUIRES: system-linux, lldb
-//
-// RUN: %dexter_base test --fail-lt 1.0 -w \
-// RUN: --builder 'clang' --debugger 'lldb' --cflags "-O0 -g" \
-// RUN: --results %t -- %s
+// RUN: %dexter_regression_test --results %t -- %s
//
// RUN: %dexter_base view %t/view.cpp.dextIR | FileCheck %s
// CHECK: ## BEGIN
diff --git a/lit.cfg.py b/lit.cfg.py
index 2d93b8d..fb2f723 100644
--- a/lit.cfg.py
+++ b/lit.cfg.py
@@ -112,6 +112,35 @@
dexter_base_cmd = '"{}" "{}"'.format(config.python3_executable, dexter_path)
tools.append(ToolSubst('%dexter_base', dexter_base_cmd))
+# Set up commands for DexTer regression tests.
+# Builder, debugger, optimisation level and several other flags differ
+# depending on whether we're running a unix like or windows os.
+if platform.system() == 'Windows':
+ dexter_regression_test_builder = '--builder clang-cl_vs2015'
+ dexter_regression_test_debugger = '--debugger dbgeng'
+ dexter_regression_test_cflags = '--cflags "/Zi /Od"'
+ dexter_regression_test_ldflags = '--ldflags "/Zi"'
+else:
+ dexter_regression_test_builder = '--builder clang'
+ dexter_regression_test_debugger = "--debugger lldb"
+ dexter_regression_test_cflags = '--cflags "-O0 -glldb"'
+ dexter_regression_test_ldflags = ''
+
+# Typical command would take the form:
+# ./path_to_py/python.exe ./path_to_dex/dexter.py test --fail-lt 1.0 -w --builder clang --debugger lldb --cflags '-O0 -g'
+dexter_regression_test_command = ' '.join(
+ # "python3", "dexter.py", test, fail_mode, builder, debugger, cflags, ldflags
+ ["{}".format(config.python3_executable),
+ "{}".format(dexter_path),
+ 'test',
+ '--fail-lt 1.0 -w',
+ dexter_regression_test_builder,
+ dexter_regression_test_debugger,
+ dexter_regression_test_cflags,
+ dexter_regression_test_ldflags])
+
+tools.append(ToolSubst('%dexter_regression_test', dexter_regression_test_command))
+
tool_dirs = [config.llvm_tools_dir]
llvm_config.add_tool_substitutions(tools, tool_dirs)
