libc/utils/libctest/format.py

# ===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===----------------------------------------------------------------------===##

"""
Lit test format for LLVM libc tests.

This format discovers pre-built test executables in the build directory
and runs them. It extends lit's ExecutableTest format.

The lit config sets test_source_root == test_exec_root (both to the build
directory), following the pattern used by llvm/test/Unit/lit.cfg.py.

Test executables are discovered by _isTestExecutable() and run by execute().

Integration tests that require command-line arguments or environment variables
have a sidecar <executable>.params file generated by CMake.  The format is
one arg per line, a "---" separator, then one KEY=VALUE env entry per line.
"""

import os
import shlex
import sys

import lit.formats
import lit.Test
import lit.util


kIsWindows = sys.platform in ["win32", "cygwin"]


class LibcTest(lit.formats.ExecutableTest):
    """
    Test format for libc unit tests.

    Extends ExecutableTest to discover pre-built test executables in the
    build directory rather than the source directory.
    """

    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
        """
        Discover test executables in the build directory.

        Since test_source_root == test_exec_root (both point to build dir),
        we use getSourcePath() to find test executables.
        """
        source_path = testSuite.getSourcePath(path_in_suite)

        # Look for test executables in the build directory
        if not os.path.isdir(source_path):
            return

        # Sort for deterministic test discovery/output ordering.
        for filename in sorted(os.listdir(source_path)):
            filepath = os.path.join(source_path, filename)

            # Match our test executable pattern
            if self._isTestExecutable(filename, filepath):
                # Create a test with the executable name
                yield lit.Test.Test(testSuite, path_in_suite + (filename,), localConfig)

    def _isTestExecutable(self, filename, filepath):
        """
        Check if a file is a libc test executable we should run.

        Recognized patterns (all must end with .__build__, optionally followed
        by .exe on Windows):
          libc.test.src.<category>.<test_name>.__build__
          libc.test.src.<category>.<test_name>.__unit__[.<opts>...].__build__
          libc.test.src.<category>.<test_name>.__hermetic__[.<opts>...].__build__
          libc.test.include.<test_name>.__unit__[.<opts>...].__build__
          libc.test.include.<test_name>.__hermetic__[.<opts>...].__build__
          libc.test.integration.<category>.<test_name>.__build__
        """
        test_name = filename
        if kIsWindows and filename.endswith(".exe"):
            test_name = filename[: -len(".exe")]

        if not test_name.endswith(".__build__"):
            return False
        if test_name.startswith("libc.test.src."):
            pass  # Accept all src tests ending in .__build__
        elif test_name.startswith("libc.test.include."):
            if ".__unit__." not in test_name and ".__hermetic__." not in test_name:
                return False
        elif test_name.startswith("libc.test.integration."):
            pass  # Accept all integration tests ending in .__build__
        else:
            return False
        if not os.path.isfile(filepath):
            return False
        if not kIsWindows and not os.access(filepath, os.X_OK):
            return False
        return True

    def _getParamsPath(self, test_path):
        params_path = test_path + ".params"
        if os.path.isfile(params_path):
            return params_path

        root, ext = os.path.splitext(test_path)
        if ext.lower() == ".exe":
            params_path = root + ".params"
            if os.path.isfile(params_path):
                return params_path

        return None

    def execute(self, test, litConfig):
        """
        Execute a test by running the test executable.

        Runs from the executable's directory so relative paths (like
        testdata/test.txt) work correctly.

        If a sidecar <executable>.params file exists, it supplies the
        command-line arguments and environment variables for the test.
        """

        test_path = test.getSourcePath()
        exec_dir = os.path.dirname(test_path)

        # Read optional sidecar .params file generated by CMake for tests that
        # need specific args/env (e.g. integration tests with ARGS/ENV).
        # Format: one arg per line, "---" separator, then KEY=VALUE env lines.
        extra_args = []
        extra_env = {}
        params_path = self._getParamsPath(test_path)
        if params_path:
            with open(params_path) as f:
                content = f.read()
            args_section, _, env_section = content.partition("---\n")
            extra_args = [l for l in args_section.splitlines() if l]
            for line in env_section.splitlines():
                if "=" in line:
                    k, _, v = line.partition("=")
                    extra_env[k] = v

        # Build the environment: inherit the current process environment, then
        # set PWD to exec_dir so getenv("PWD") matches getcwd(), then overlay
        # any test-specific variables from the .params file.
        env = dict(os.environ)
        env["PWD"] = exec_dir
        env.update(extra_env)

        test_cmd_template = getattr(test.config, "libc_test_cmd", "")
        if test_cmd_template:
            test_cmd = test_cmd_template.replace("@BINARY@", test_path)
            cmd_args = shlex.split(test_cmd)
            if not cmd_args:
                cmd_args = [test_path]
            cmd_args = cmd_args + extra_args
            out, err, exit_code = lit.util.executeCommand(
                cmd_args, cwd=exec_dir, env=env
            )
        else:
            out, err, exit_code = lit.util.executeCommand(
                [test_path] + extra_args, cwd=exec_dir, env=env
            )

        if not exit_code:
            return lit.Test.PASS, ""

        return lit.Test.FAIL, out + err