Add lit config for dir with standalone tests
Some test systems do not use lit for test discovery but only for its
substitution and test selection because they use another way of managing
test collections, e.g. CTest. This forces those tests to be invoked with
lit --no-indirectly-run-check. When a mix of lit version is in use, it
requires to detect the availability of that option.
This commit provides a new config option standalone_tests to signal a
directory made of tests meant to run as standalone. When this option is
set, lit skips test discovery and the indirectly run check. It also adds
the missing documentation for --no-indirectly-run-check.
Reviewed By: jdenny
Differential Revision: https://reviews.llvm.org/D94766
GitOrigin-RevId: 1e2d50936a3bff2c8065a99d06f37b51185fbd81
diff --git a/docs/CommandGuide/lit.rst b/docs/CommandGuide/lit.rst
index f4d9d9d..3820cc4 100644
--- a/docs/CommandGuide/lit.rst
+++ b/docs/CommandGuide/lit.rst
@@ -154,6 +154,11 @@
suite take the most time to execute. Note that this option is most useful
with ``-j 1``.
+.. option:: --no-indirectly-run-check
+
+ Do not error if a test would not be run if the user had specified the
+ containing directory instead of naming the test directly.
+
.. _selection-options:
SELECTION OPTIONS
@@ -372,6 +377,11 @@
**environment** A dictionary representing the environment to use when executing
tests in the suite.
+ **standalone_tests** When true, mark a directory with tests expected to be run
+ standalone. Test discovery is disabled for that directory and
+ *--no-indirectly-run-check* is in effect. *lit.suffixes* and *lit.excludes*
+ must be empty when this variable is true.
+
**suffixes** For **lit** test formats which scan directories for tests, this
variable is a list of suffixes to identify test files. Used by: *ShTest*.
diff --git a/utils/lit/lit/TestingConfig.py b/utils/lit/lit/TestingConfig.py
index 38d0506..d40ee37 100644
--- a/utils/lit/lit/TestingConfig.py
+++ b/utils/lit/lit/TestingConfig.py
@@ -60,7 +60,8 @@
test_source_root = None,
excludes = [],
available_features = available_features,
- pipefail = True)
+ pipefail = True,
+ standalone_tests = False)
def load_from_path(self, path, litConfig):
"""
@@ -105,7 +106,8 @@
environment, substitutions, unsupported,
test_exec_root, test_source_root, excludes,
available_features, pipefail, limit_to_features = [],
- is_early = False, parallelism_group = None):
+ is_early = False, parallelism_group = None,
+ standalone_tests = False):
self.parent = parent
self.name = str(name)
self.suffixes = set(suffixes)
@@ -118,6 +120,7 @@
self.excludes = set(excludes)
self.available_features = set(available_features)
self.pipefail = pipefail
+ self.standalone_tests = standalone_tests
# This list is used by TestRunner.py to restrict running only tests that
# require one of the features in this list if this list is non-empty.
# Configurations can set this list to restrict the set of tests to run.
diff --git a/utils/lit/lit/discovery.py b/utils/lit/lit/discovery.py
index 2f027a5..a185ae6 100644
--- a/utils/lit/lit/discovery.py
+++ b/utils/lit/lit/discovery.py
@@ -160,8 +160,13 @@
# tests which are not executed. The check adds some performance
# overhead which might be important if a large number of tests
# are being run directly.
- # --no-indirectly-run-check: skips this check.
- if indirectlyRunCheck and lc.test_format is not None:
+ # This check can be disabled by using --no-indirectly-run-check or
+ # setting the standalone_tests variable in the suite's configuration.
+ if (
+ indirectlyRunCheck
+ and lc.test_format is not None
+ and not lc.standalone_tests
+ ):
found = False
for res in lc.test_format.getTestsInDirectory(ts, test_dir_in_suite,
litConfig, lc):
@@ -171,6 +176,7 @@
if not found:
litConfig.error(
'%r would not be run indirectly: change name or LIT config'
+ '(e.g. suffixes or standalone_tests variables)'
% test.getFullName())
yield test
@@ -180,6 +186,15 @@
# local configuration.
lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
+ # Directory contains tests to be run standalone. Do not try to discover.
+ if lc.standalone_tests:
+ if lc.suffixes or lc.excludes:
+ litConfig.warning(
+ 'standalone_tests set in LIT config but suffixes or excludes'
+ ' are also set'
+ )
+ return
+
# Search for tests.
if lc.test_format is not None:
for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
diff --git a/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg b/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg
new file mode 100644
index 0000000..37d96dd
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.excludes = ['.test']
+config.standalone_tests = True
diff --git a/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt b/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt
new file mode 100644
index 0000000..b80b60b
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg b/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg
new file mode 100644
index 0000000..41434c9
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg
@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.suffixes = ['.txt']
+config.standalone_tests = True
diff --git a/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt b/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt
new file mode 100644
index 0000000..b80b60b
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/utils/lit/tests/Inputs/standalone-tests/lit.cfg b/utils/lit/tests/Inputs/standalone-tests/lit.cfg
new file mode 100644
index 0000000..12ed7b0
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests/lit.cfg
@@ -0,0 +1,4 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.standalone_tests = True
diff --git a/utils/lit/tests/Inputs/standalone-tests/true.txt b/utils/lit/tests/Inputs/standalone-tests/true.txt
new file mode 100644
index 0000000..b80b60b
--- /dev/null
+++ b/utils/lit/tests/Inputs/standalone-tests/true.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/utils/lit/tests/discovery.py b/utils/lit/tests/discovery.py
index cb04eda..39743bf 100644
--- a/utils/lit/tests/discovery.py
+++ b/utils/lit/tests/discovery.py
@@ -148,6 +148,36 @@
# RUN: %{lit} \
# RUN: %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check
+# Check that a standalone test with no suffixes set is run without any errors.
+#
+# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
+#
+# CHECK-STANDALONE: PASS: Standalone tests :: true.txt
+
+# Check that an error is produced if suffixes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
+
+# Check that an error is produced if excludes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
+
+# Check that no discovery is done for testsuite with standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
+
# Check that we don't recurse infinitely when loading an site specific test
# suite located inside the test source root.
#
