| """LLVM test-suite compile and execution tests""" |
| from __future__ import absolute_import |
| from __future__ import print_function |
| from future import standard_library |
| standard_library.install_aliases() |
| import csv |
| import os |
| import platform |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import glob |
| import time |
| import traceback |
| import urllib.error |
| import shlex |
| import pipes |
| import resource |
| |
| import click |
| |
| import lnt.testing |
| import lnt.testing.util.compilers |
| import lnt.util.ImportData as ImportData |
| |
| from lnt.testing.util.commands import fatal |
| from lnt.testing.util.commands import capture, mkdir_p, which |
| from lnt.testing.util.commands import resolve_command_path |
| |
| from lnt.testing.util.rcs import get_source_version |
| |
| from lnt.testing.util.misc import timestamp |
| |
| from lnt.server.reporting.analysis import UNCHANGED_PASS, UNCHANGED_FAIL |
| from lnt.server.reporting.analysis import REGRESSED, IMPROVED |
| from lnt.util import logger |
| from lnt.lnttool.common import submit_options |
| from . import builtintest |
| |
| |
| class TestModule(object): |
| """ |
| Base class for extension test modules. |
| """ |
| |
| def __init__(self): |
| self._log = None |
| |
| def main(self): |
| raise NotImplementedError |
| |
| def execute_test(self, options, make_variables, config): |
| raise RuntimeError("Abstract Method.") |
| |
| def _execute_test(self, test_log, options, make_variables, config): |
| self._log = test_log |
| self._user_time = config.test_time_stat == 'user' |
| try: |
| return self.execute_test(options, make_variables, config) |
| finally: |
| self._log = None |
| |
| def call(self, args, **kwargs): |
| if kwargs.get('shell', False): |
| cmdstr = args |
| else: |
| cmdstr = ' '.join(args) |
| |
| if 'cwd' in kwargs: |
| print("# In working dir: " + kwargs['cwd'], file=self._log) |
| print(cmdstr, file=self.log) |
| |
| self._log.flush() |
| p = subprocess.Popen(args, stdout=self._log, stderr=self._log, |
| **kwargs) |
| return p.wait() |
| |
| def get_time(self): |
| if self._user_time: |
| return resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime |
| else: |
| return time.time() |
| |
| @property |
| def log(self): |
| """Get the test log output stream.""" |
| if self._log is None: |
| raise ValueError("log() unavailable outside test execution") |
| return self._log |
| |
| |
| class TestConfiguration(object): |
| """Store and calculate important paths and options for this test based |
| on the command line arguments. This object is stateless and only |
| based on the command line arguments! Options which take a long |
| time to calculate are cached, since we are stateless this is okay. |
| |
| """ |
| |
| def __init__(self, opts, start_time): |
| """Prepare the configuration: |
| opts -- the command line options object |
| start_time -- the time the program was invoked as a string |
| """ |
| assert isinstance(opts, dict), "Options must be a dict." |
| self.opts = opts |
| self.__dict__.update(opts) |
| self.start_time = start_time |
| |
| # Report directory cache. |
| self._report_dir = None |
| # Compiler interrogation is a lot of work, this will cache it. |
| self._cc_info = None |
| # Getting compiler version spawns subprocesses, cache it. |
| self._get_source_version = None |
| self.rerun_test = None |
| |
| @property |
| def report_dir(self): |
| """Get the (possibly cached) path to the directory where test suite |
| will be placed. Report dir is a directory within the sandbox which |
| is either "build" or a timestamped directory based on """ |
| if self._report_dir is not None: |
| return self._report_dir |
| |
| if self.timestamp_build: |
| ts = self.start_time.replace(' ', '_').replace(':', '-') |
| build_dir_name = "test-%s" % ts |
| else: |
| build_dir_name = "build" |
| basedir = os.path.join(self.sandbox_path, build_dir_name) |
| # Canonicalize paths, in case we are using e.g. an NFS remote mount. |
| # |
| # FIXME: This should be eliminated, along with the realpath call below. |
| basedir = os.path.realpath(basedir) |
| self._report_dir = basedir |
| return basedir |
| |
| def report_path(self, iteration): |
| """Path to a single run's JSON results file.""" |
| return os.path.join(self.build_dir(iteration), 'report.json') |
| |
| def build_dir(self, iteration): |
| """Path of the build dir within the report dir. iteration -- the |
| iteration number if multisample otherwise None. |
| When multisample is off report_dir == build_dir. |
| """ |
| # Do nothing in single-sample build, because report_dir and the |
| # build_dir is the same directory. |
| if iteration is None: |
| return self.report_dir |
| |
| # Create the directory for individual iteration. |
| return os.path.join(self.report_dir, "sample-%d" % iteration) |
| |
| @property |
| def target_flags(self): |
| """Computed target flags list.""" |
| # Compute TARGET_FLAGS. |
| target_flags = [] |
| |
| # FIXME: Eliminate this blanket option. |
| target_flags.extend(self.cflags) |
| |
| if self.cflag_string: |
| # FIXME: This isn't generally OK on Windows :/ |
| target_flags.extend(_unix_quote_args(self.cflag_string)) |
| |
| # Pass flags to backend. |
| for f in self.mllvm: |
| target_flags.extend(['-mllvm', f]) |
| |
| if self.arch is not None: |
| target_flags.append('-arch') |
| target_flags.append(self.arch) |
| if self.isysroot is not None: |
| target_flags.append('-isysroot') |
| target_flags.append(self.isysroot) |
| return target_flags |
| |
| @property |
| def cc_info(self): |
| """Discovered compiler information from the cc under test. Cached |
| because discovery is slow. |
| |
| """ |
| if self._cc_info is None: |
| self._cc_info = lnt.testing.util.compilers.get_cc_info( |
| self.cc_under_test, |
| self.target_flags) |
| return self._cc_info |
| |
| @property |
| def target(self): |
| """Discovered compiler's target information.""" |
| # Get compiler info. |
| cc_target = self.cc_info.get('cc_target') |
| return cc_target |
| |
| @property |
| def llvm_source_version(self): |
| """The version of llvm from llvm_src_root.""" |
| if self.llvm_src_root: |
| if self._get_source_version is None: |
| self._get_source_version = get_source_version( |
| self.llvm_src_root) |
| return self._get_source_version |
| else: |
| return None |
| |
| @property |
| def qemu_user_mode_command(self): |
| """ The command used for qemu user mode """ |
| assert self.qemu_user_mode |
| qemu_cmd_line = [self.qemu_user_mode] + list(self.qemu_flags) |
| if self.qemu_string: |
| qemu_cmd_line += _unix_quote_args(self.qemu_string) |
| return ' '.join(qemu_cmd_line) |
| |
| @property |
| def generate_report_script(self): |
| """ The path to the report generation script. """ |
| return os.path.join(self.test_suite_root, "GenerateReport.pl") |
| |
| def build_report_path(self, iteration): |
| """The path of the results.csv file which each run of the test suite |
| will produce. |
| iteration -- the multisample iteration number otherwise None.""" |
| report_path = os.path.join(self.build_dir(iteration)) |
| if self.only_test is not None: |
| report_path = os.path.join(report_path, self.only_test) |
| report_path = os.path.join(report_path, 'report.%s.csv' % |
| self.test_style) |
| return report_path |
| |
| def test_log_path(self, iteration): |
| """The path of the log file for the build. |
| iteration -- the multisample iteration number otherwise None.""" |
| return os.path.join(self.build_dir(iteration), 'test.log') |
| |
| def compute_run_make_variables(self): |
| """Compute make variables from command line arguments and compiler. |
| Returns a dict of make_variables as well as a public version |
| with the remote options removed. |
| |
| """ |
| cc_info = self.cc_info |
| # Set the make variables to use. |
| make_variables = { |
| 'TARGET_CC': self.cc_reference, |
| 'TARGET_CXX': self.cxx_reference, |
| 'TARGET_LLVMGCC': self.cc_under_test, |
| 'TARGET_LLVMGXX': self.cxx_under_test, |
| 'TARGET_FLAGS': ' '.join(self.target_flags), |
| } |
| |
| # Compute TARGET_LLCFLAGS, for TEST=nightly runs. |
| if self.test_style == "nightly": |
| # Compute TARGET_LLCFLAGS. |
| target_llcflags = [] |
| if self.mcpu is not None: |
| target_llcflags.append('-mcpu') |
| target_llcflags.append(self.mcpu) |
| if self.relocation_model is not None: |
| target_llcflags.append('-relocation-model') |
| target_llcflags.append(self.relocation_model) |
| if self.disable_fp_elim: |
| target_llcflags.append('-disable-fp-elim') |
| make_variables['TARGET_LLCFLAGS'] = ' '.join(target_llcflags) |
| |
| # Set up environment overrides if requested, to effectively |
| # run under the specified Darwin iOS simulator. |
| # |
| # See /D/P/../Developer/Tools/RunPlatformUnitTests. |
| if self.ios_simulator_sdk is not None: |
| make_variables['EXECUTION_ENVIRONMENT_OVERRIDES'] = ' '.join( |
| ['DYLD_FRAMEWORK_PATH="%s"' % self.ios_simulator_sdk, |
| 'DYLD_LIBRARY_PATH=""', |
| 'DYLD_ROOT_PATH="%s"' % self.ios_simulator_sdk, |
| 'DYLD_NEW_LOCAL_SHARED_REGIONS=YES', |
| 'DYLD_NO_FIX_PREBINDING=YES', |
| 'IPHONE_SIMULATOR_ROOT="%s"' % self.ios_simulator_sdk, |
| 'CFFIXED_USER_HOME="%s"' % os.path.expanduser( |
| "~/Library/Application Support/iPhone Simulator/User")]) |
| |
| # Pick apart the build mode. |
| build_mode = self.build_mode |
| if build_mode.startswith("Debug"): |
| build_mode = build_mode[len("Debug"):] |
| make_variables['ENABLE_OPTIMIZED'] = '0' |
| elif build_mode.startswith("Unoptimized"): |
| build_mode = build_mode[len("Unoptimized"):] |
| make_variables['ENABLE_OPTIMIZED'] = '0' |
| elif build_mode.startswith("Release"): |
| build_mode = build_mode[len("Release"):] |
| make_variables['ENABLE_OPTIMIZED'] = '1' |
| else: |
| fatal('invalid build mode: %r' % self.build_mode) |
| |
| while build_mode: |
| for (name, key) in (('+Asserts', 'ENABLE_ASSERTIONS'), |
| ('+Checks', 'ENABLE_EXPENSIVE_CHECKS'), |
| ('+Coverage', 'ENABLE_COVERAGE'), |
| ('+Debug', 'DEBUG_SYMBOLS'), |
| ('+Profile', 'ENABLE_PROFILING')): |
| if build_mode.startswith(name): |
| build_mode = build_mode[len(name):] |
| make_variables[key] = '1' |
| break |
| else: |
| fatal('invalid build mode: %r' % self.build_mode) |
| |
| # Assertions are disabled by default. |
| if 'ENABLE_ASSERTIONS' in make_variables: |
| del make_variables['ENABLE_ASSERTIONS'] |
| else: |
| make_variables['DISABLE_ASSERTIONS'] = '1' |
| |
| # Set the optimization level options. |
| make_variables['OPTFLAGS'] = self.optimize_option |
| if self.optimize_option == '-Os': |
| make_variables['LLI_OPTFLAGS'] = '-O2' |
| make_variables['LLC_OPTFLAGS'] = '-O2' |
| else: |
| make_variables['LLI_OPTFLAGS'] = self.optimize_option |
| make_variables['LLC_OPTFLAGS'] = self.optimize_option |
| |
| # Set test selection variables. |
| if not self.test_cxx: |
| make_variables['DISABLE_CXX'] = '1' |
| if not self.test_jit: |
| make_variables['DISABLE_JIT'] = '1' |
| if not self.test_llc: |
| make_variables['DISABLE_LLC'] = '1' |
| if not self.test_lto: |
| make_variables['DISABLE_LTO'] = '1' |
| if self.test_llcbeta: |
| make_variables['ENABLE_LLCBETA'] = '1' |
| if self.test_small: |
| make_variables['SMALL_PROBLEM_SIZE'] = '1' |
| if self.test_large: |
| if self.test_small: |
| fatal('the --small and --large options are mutually exclusive') |
| make_variables['LARGE_PROBLEM_SIZE'] = '1' |
| if self.test_benchmarking_only: |
| make_variables['BENCHMARKING_ONLY'] = '1' |
| if self.test_integrated_as: |
| make_variables['TEST_INTEGRATED_AS'] = '1' |
| if self.liblto_path: |
| make_variables['LD_ENV_OVERRIDES'] = ( |
| 'env DYLD_LIBRARY_PATH=%s' % os.path.dirname( |
| self.liblto_path)) |
| # If ref input is requested for SPEC, we wil run SPEC through its new |
| # test module so skip SPEC as part of NT. |
| if self.test_spec_ref: |
| make_variables['USE_SPEC_TEST_MODULE'] = '1' |
| |
| if self.threads > 1 or self.build_threads > 1: |
| make_variables['ENABLE_PARALLEL_REPORT'] = '1' |
| |
| # Select the test style to use. |
| if self.test_style == "simple": |
| # We always use reference outputs with TEST=simple. |
| make_variables['ENABLE_HASHED_PROGRAM_OUTPUT'] = '1' |
| make_variables['USE_REFERENCE_OUTPUT'] = '1' |
| make_variables['TEST'] = self.test_style |
| |
| # Set CC_UNDER_TEST_IS_CLANG when appropriate. |
| if cc_info.get('cc_name') in ('apple_clang', 'clang'): |
| make_variables['CC_UNDER_TEST_IS_CLANG'] = '1' |
| elif cc_info.get('cc_name') in ('llvm-gcc',): |
| make_variables['CC_UNDER_TEST_IS_LLVM_GCC'] = '1' |
| elif cc_info.get('cc_name') in ('gcc',): |
| make_variables['CC_UNDER_TEST_IS_GCC'] = '1' |
| |
| # Convert the target arch into a make variable, to allow more |
| # target based specialization (e.g., |
| # CC_UNDER_TEST_TARGET_IS_ARMV7). |
| if '-' in cc_info.get('cc_target', ''): |
| arch_name = cc_info.get('cc_target').split('-', 1)[0] |
| CC_VAR = 'CC_UNDER_TEST_TARGET_IS_' + arch_name.upper() |
| make_variables[CC_VAR] = '1' |
| |
| # Set LLVM_RELEASE_IS_PLUS_ASSERTS when appropriate, to allow |
| # testing older LLVM source trees. |
| llvm_source_version = self.llvm_source_version |
| if llvm_source_version and llvm_source_version.isdigit() and \ |
| int(llvm_source_version) < 107758: |
| make_variables['LLVM_RELEASE_IS_PLUS_ASSERTS'] = 1 |
| |
| # Set ARCH appropriately, based on the inferred target. |
| # |
| # FIXME: We should probably be more strict about this. |
| cc_target = cc_info.get('cc_target') |
| llvm_arch = self.llvm_arch |
| if cc_target and llvm_arch is None: |
| # cc_target is expected to be a (GCC style) target |
| # triple. Pick out the arch component, and then try to |
| # convert it to an LLVM nightly test style architecture |
| # name, which is of course totally different from all of |
| # GCC names, triple names, LLVM target names, and LLVM |
| # triple names. Stupid world. |
| # |
| # FIXME: Clean this up once everyone is on 'lnt runtest |
| # nt' style nightly testing. |
| arch = cc_target.split('-', 1)[0].lower() |
| if len(arch) == 4 and arch[0] == 'i' and arch.endswith('86') and \ |
| arch[1] in '3456789': # i[3-9]86 |
| llvm_arch = 'x86' |
| elif arch in ('x86_64', 'amd64'): |
| llvm_arch = 'x86_64' |
| elif arch in ('powerpc', 'powerpc64', 'powerpc64le', 'ppu'): |
| llvm_arch = 'PowerPC' |
| elif (arch == 'arm' or arch.startswith('armv') or |
| arch == 'thumb' or arch.startswith('thumbv') or |
| arch == 'xscale'): |
| llvm_arch = 'ARM' |
| elif arch in ('aarch64', 'arm64'): |
| llvm_arch = 'AArch64' |
| elif arch.startswith('alpha'): |
| llvm_arch = 'Alpha' |
| elif arch.startswith('sparc'): |
| llvm_arch = 'Sparc' |
| elif arch in ('mips', 'mipsel', 'mips64', 'mips64el'): |
| llvm_arch = 'Mips' |
| elif arch in ('ve'): |
| llvm_arch = 'VE' |
| |
| if llvm_arch is not None: |
| make_variables['ARCH'] = llvm_arch |
| else: |
| logger.warning("unable to infer ARCH, " + |
| "some tests may not run correctly!") |
| |
| # Add in any additional make flags passed in via --make-param. |
| for entry in self.make_parameters: |
| if '=' not in entry: |
| name, value = entry, '' |
| else: |
| name, value = entry.split('=', 1) |
| |
| make_variables[name] = value |
| |
| # Set remote execution variables, if used. |
| if self.remote: |
| # make a copy of args for report, without remote options. |
| public_vars = make_variables.copy() |
| make_variables['REMOTE_HOST'] = self.remote_host |
| make_variables['REMOTE_USER'] = self.remote_user |
| make_variables['REMOTE_PORT'] = str(self.remote_port) |
| make_variables['REMOTE_CLIENT'] = self.remote_client |
| else: |
| public_vars = make_variables |
| |
| # Set qemu user mode variables, if used. |
| if self.qemu_user_mode: |
| make_variables['USER_MODE_EMULATION'] = '1' |
| make_variables['RUNUNDER'] = self.qemu_user_mode_command |
| |
| # Set USE_PERF flag, if specified. |
| if self.use_perf: |
| make_variables['USE_PERF'] = '1' |
| |
| return make_variables, public_vars |
| |
| ### |
| |
| |
| def scan_for_test_modules(config): |
| base_modules_path = os.path.join(config.test_suite_root, 'LNTBased') |
| if config.only_test is None: |
| test_modules_path = base_modules_path |
| elif config.only_test.startswith('LNTBased'): |
| test_modules_path = os.path.join(config.test_suite_root, |
| config.only_test) |
| else: |
| return |
| |
| # We follow links here because we want to support the ability for having |
| # various "suites" of LNTBased tests in separate repositories, and allowing |
| # users to just checkout them out elsewhere and link them into their LLVM |
| # test-suite source tree. |
| for dirpath, dirnames, filenames in os.walk(test_modules_path, |
| followlinks=True): |
| # Ignore the example tests, unless requested. |
| if not config.include_test_examples and 'Examples' in dirnames: |
| dirnames.remove('Examples') |
| |
| # Only use the SPEC test module if the ref input was requested. |
| if not config.test_spec_ref and re.search('/spec', dirpath): |
| continue |
| |
| # Check if this directory defines a test module. |
| if 'TestModule' not in filenames: |
| continue |
| |
| # If so, don't traverse any lower. |
| del dirnames[:] |
| |
| # Add to the list of test modules. |
| assert dirpath.startswith(base_modules_path + '/') |
| yield dirpath[len(base_modules_path) + 1:] |
| |
| |
| def execute_command(test_log, basedir, args, report_dir): |
| logfile = test_log |
| |
| if report_dir is not None: |
| logfile = subprocess.PIPE |
| # Open a duplicated logfile at the global dir. |
| _, logname = os.path.split(test_log.name) |
| global_log_path = os.path.join(report_dir, logname) |
| global_log = open(global_log_path, 'a+') |
| |
| p = subprocess.Popen(args=args, stdin=None, stdout=logfile, |
| stderr=subprocess.STDOUT, cwd=basedir, |
| env=os.environ, universal_newlines=True) |
| |
| if report_dir is not None: |
| while p.poll() is None: |
| line = p.stdout.readline() |
| if len(line) > 0: |
| test_log.write(line) |
| global_log.write(line) |
| |
| global_log.close() |
| |
| return p.wait() |
| |
| |
| # FIXME: Support duplicate logfiles to global directory. |
| def execute_test_modules(test_log, test_modules, test_module_variables, |
| basedir, config, make_variables): |
| # For now, we don't execute these in parallel, but we do forward the |
| # parallel build options to the test. |
| test_modules.sort() |
| |
| print('%s: executing test modules' % (timestamp(), ), file=sys.stderr) |
| results = [] |
| for name in test_modules: |
| # First, load the test module file. |
| globals = {} |
| test_path = os.path.join(config.test_suite_root, 'LNTBased', name) |
| # This is where shared code between test modules should go. |
| sys.path.append(os.path.join(config.test_suite_root, 'LNTBased/lib')) |
| test_obj_path = os.path.join(basedir, 'LNTBased', name) |
| module_path = os.path.join(test_path, 'TestModule') |
| module_file = open(module_path) |
| try: |
| exec(module_file, globals) |
| except Exception: |
| info = traceback.format_exc() |
| fatal("unable to import test module: %r\n%s" % ( |
| module_path, info)) |
| |
| # Lookup and instantiate the test class. |
| test_class = globals.get('test_class') |
| if test_class is None: |
| fatal("no 'test_class' global in import test module: %r" % ( |
| module_path,)) |
| try: |
| test_instance = test_class() |
| except Exception: |
| info = traceback.format_exc() |
| fatal("unable to instantiate test class for: %r\n%s" % ( |
| module_path, info)) |
| |
| if not isinstance(test_instance, TestModule): |
| fatal("invalid test class (expected lnt.tests.nt.TestModule " |
| "subclass) for: %r" % module_path) |
| |
| # Create the per test variables, and ensure the output directory |
| # exists. |
| variables = test_module_variables.copy() |
| variables['MODULENAME'] = name |
| variables['SRCROOT'] = test_path |
| variables['OBJROOT'] = test_obj_path |
| mkdir_p(test_obj_path) |
| |
| # Execute the tests. |
| try: |
| test_samples = test_instance._execute_test(test_log, variables, |
| make_variables, config) |
| except Exception: |
| info = traceback.format_exc() |
| fatal("exception executing tests for: %r\n%s" % ( |
| module_path, info)) |
| |
| # Check that the test samples are in the expected format. |
| is_ok = True |
| try: |
| test_samples = list(test_samples) |
| for item in test_samples: |
| if not isinstance(item, lnt.testing.TestSamples): |
| is_ok = False |
| break |
| except Exception: |
| is_ok = False |
| if not is_ok: |
| fatal("test module did not return samples list: %r" % ( |
| module_path,)) |
| |
| results.append((name, test_samples)) |
| |
| return results |
| |
| |
| def compute_test_module_variables(make_variables, config): |
| # Set the test module options, which we try and restrict to a tighter |
| # subset than what we pass to the LNT makefiles. |
| test_module_variables = { |
| 'CC': make_variables['TARGET_LLVMGCC'], |
| 'CXX': make_variables['TARGET_LLVMGXX'], |
| 'CFLAGS': (make_variables['TARGET_FLAGS'] + ' ' + |
| make_variables['OPTFLAGS']), |
| 'CXXFLAGS': (make_variables['TARGET_FLAGS'] + ' ' + |
| make_variables['OPTFLAGS']), |
| } |
| |
| # Add the remote execution variables. |
| if config.remote: |
| test_module_variables['REMOTE_HOST'] = make_variables['REMOTE_HOST'] |
| test_module_variables['REMOTE_USER'] = make_variables['REMOTE_USER'] |
| test_module_variables['REMOTE_PORT'] = make_variables['REMOTE_PORT'] |
| test_module_variables['REMOTE_CLIENT'] = \ |
| make_variables['REMOTE_CLIENT'] |
| |
| # Add miscellaneous optional variables. |
| if 'LD_ENV_OVERRIDES' in make_variables: |
| value = make_variables['LD_ENV_OVERRIDES'] |
| assert value.startswith('env ') |
| test_module_variables['LINK_ENVIRONMENT_OVERRIDES'] = value[4:] |
| |
| # This isn't possible currently, just here to mark what the option variable |
| # would be called. |
| if 'COMPILE_ENVIRONMENT_OVERRIDES' in make_variables: |
| test_module_variables['COMPILE_ENVIRONMENT_OVERRIDES'] = \ |
| make_variables['COMPILE_ENVIRONMENT_OVERRIDES'] |
| |
| if 'EXECUTION_ENVIRONMENT_OVERRIDES' in make_variables: |
| test_module_variables['EXECUTION_ENVIRONMENT_OVERRIDES'] = \ |
| make_variables['EXECUTION_ENVIRONMENT_OVERRIDES'] |
| |
| # We pass the test execution values as variables too, this might be better |
| # passed as actual arguments. |
| test_module_variables['THREADS'] = config.threads |
| test_module_variables['BUILD_THREADS'] = \ |
| config.build_threads or config.threads |
| return test_module_variables |
| |
| |
| def execute_nt_tests(test_log, make_variables, basedir, config): |
| report_dir = config.report_dir |
| common_args = ['make', '-k'] |
| common_args.extend('%s=%s' % (k, v) for k, v in make_variables.items()) |
| if config.only_test is not None: |
| common_args.extend(['-C', config.only_test]) |
| |
| # If we are using isolation, run under sandbox-exec. |
| if config.use_isolation: |
| # Write out the sandbox profile. |
| sandbox_profile_path = os.path.join(basedir, "isolation.sb") |
| print("%s: creating sandbox profile %r" % ( |
| timestamp(), sandbox_profile_path), file=sys.stderr) |
| with open(sandbox_profile_path, 'w') as f: |
| print(""" |
| ;; Sandbox profile for isolation test access. |
| (version 1) |
| |
| ;; Allow everything by default, and log debug messages on deny. |
| (allow default) |
| (debug deny) |
| |
| ;; Deny all file writes by default. |
| (deny file-write*) |
| |
| ;; Deny all network access by default. |
| (deny network*) |
| |
| ;; Explicitly allow writes to temporary directories, /dev/, and the sandbox |
| ;; output directory. |
| (allow file-write* (regex #"^/private/var/tmp/") |
| (regex #"^/private/tmp/") |
| (regex #"^/private/var/folders/") |
| (regex #"^/dev/") |
| (regex #"^%s"))""" % (basedir, ), file=f) |
| common_args = ['sandbox-exec', '-f', sandbox_profile_path] +\ |
| common_args |
| |
| # Run a separate 'make build' step if --build-threads was given. |
| if config.build_threads > 0: |
| args = common_args + ['-j', str(config.build_threads), 'build'] |
| print('%s: running: %s' % (timestamp(), |
| ' '.join('"%s"' % a |
| for a in args)), file=test_log) |
| test_log.flush() |
| |
| print('%s: building "nightly tests" with -j%u...' % ( |
| timestamp(), config.build_threads), file=sys.stderr) |
| res = execute_command(test_log, basedir, args, report_dir) |
| if res != 0: |
| print("Failure while running make build! " \ |
| "See log: %s" % test_log.name, file=sys.stderr) |
| |
| # Then 'make report'. |
| args = common_args + ['-j', str(config.threads), |
| 'report', 'report.%s.csv' % config.test_style] |
| print('%s: running: %s' % (timestamp(), |
| ' '.join('"%s"' % a |
| for a in args)), file=test_log) |
| test_log.flush() |
| |
| # FIXME: We shouldn't need to set env=os.environ here, but if we don't |
| # somehow MACOSX_DEPLOYMENT_TARGET gets injected into the environment on OS |
| # X (which changes the driver behavior and causes generally weirdness). |
| print('%s: executing "nightly tests" with -j%u...' % ( |
| timestamp(), config.threads), file=sys.stderr) |
| |
| res = execute_command(test_log, basedir, args, report_dir) |
| |
| if res != 0: |
| print("Failure while running nightly tests! "\ |
| "See log: %s" % test_log.name, file=sys.stderr) |
| |
| |
| # Keep a mapping of mangled test names, to the original names in the |
| # test-suite. |
| TEST_TO_NAME = {} |
| KNOWN_SAMPLE_KEYS = ('compile', 'exec', 'hash', |
| 'gcc.compile', 'bc.compile', 'llc.compile', |
| 'llc-beta.compile', 'jit.compile', 'gcc.exec', 'llc.exec', |
| 'llc-beta.exec', 'jit.exec') |
| |
| |
| def load_nt_report_file(report_path, config): |
| # Compute the test samples to report. |
| sample_keys = [] |
| |
| def append_to_sample_keys(tup): |
| stat = tup[1] |
| assert stat in KNOWN_SAMPLE_KEYS |
| if stat not in config.exclude_stat_from_submission: |
| sample_keys.append(tup) |
| if config.test_style == "simple": |
| test_namespace = 'nts' |
| time_stat = '' |
| # for now, user time is the unqualified Time stat |
| if config.test_time_stat == "real": |
| time_stat = 'Real_' |
| append_to_sample_keys((True, 'compile', 'CC_' + time_stat + 'Time', |
| None, 'CC', float)) |
| append_to_sample_keys((False, 'hash', 'CC_Hash', None, 'CC', str)) |
| append_to_sample_keys((True, 'exec', 'Exec_' + time_stat + 'Time', |
| None, 'Exec', float)) |
| else: |
| test_namespace = 'nightlytest' |
| append_to_sample_keys((True, 'gcc.compile', 'GCCAS', 'time')) |
| append_to_sample_keys((True, 'bc.compile', 'Bytecode', 'size')) |
| if config.test_llc: |
| append_to_sample_keys((True, 'llc.compile', 'LLC compile', 'time')) |
| if config.test_llcbeta: |
| append_to_sample_keys((True, 'llc-beta.compile', |
| 'LLC-BETA compile', 'time')) |
| if config.test_jit: |
| append_to_sample_keys((True, 'jit.compile', 'JIT codegen', 'time')) |
| append_to_sample_keys((True, 'gcc.exec', 'GCC', 'time')) |
| if config.test_llc: |
| append_to_sample_keys((True, 'llc.exec', 'LLC', 'time')) |
| if config.test_llcbeta: |
| append_to_sample_keys((True, 'llc-beta.exec', 'LLC-BETA', 'time')) |
| if config.test_jit: |
| append_to_sample_keys((True, 'jit.exec', 'JIT', 'time')) |
| |
| # Load the report file. |
| if sys.version_info[0] < 3: |
| report_file = open(report_path, 'rb') |
| else: |
| report_file = open(report_path, 'r', newline='') |
| reader_it = iter(csv.reader(report_file)) |
| |
| # Get the header. |
| header = next(reader_it) |
| if header[0] != 'Program': |
| fatal('unexpected report file, missing header') |
| |
| # Verify we have the keys we expect. |
| if 'Program' not in header: |
| fatal('missing key %r in report header' % 'Program') |
| for item in sample_keys: |
| required = item[0] |
| header_name = item[2] |
| if required and header_name not in header: |
| fatal('missing key %r in report header' % header_name) |
| |
| # We don't use the test info, currently. |
| test_info = {} |
| test_samples = [] |
| no_errors = True |
| for row in reader_it: |
| record = dict(zip(header, row)) |
| |
| program = record['Program'] |
| |
| if config.only_test is not None: |
| program = os.path.join(config.only_test, program) |
| if config.rerun_test is not None: |
| program = os.path.join(config.rerun_test, program) |
| |
| program_real = program |
| program_mangled = program.replace('.', '_') |
| test_base_name = program_mangled |
| |
| # Check if this is a subtest result, in which case we ignore missing |
| # values. |
| if '_Subtest_' in test_base_name: |
| is_subtest = True |
| test_base_name = test_base_name.replace('_Subtest_', '.') |
| |
| else: |
| is_subtest = False |
| |
| test_base_name = '%s.%s' % (test_namespace, test_base_name) |
| |
| TEST_TO_NAME[test_base_name] = program_real |
| |
| for info in sample_keys: |
| if len(info) == 4: |
| required, name, key, tname = info |
| success_key = None |
| conv_f = float |
| else: |
| required, name, key, tname, success_key, conv_f = info |
| |
| test_name = '%s.%s' % (test_base_name, name) |
| if not required and key not in record: |
| continue |
| value = record[key] |
| if success_key is None: |
| success_value = value |
| else: |
| success_value = record[success_key] |
| |
| # FIXME: Move to simpler and more succinct format, using .failed. |
| if success_value == '*': |
| if is_subtest: |
| continue |
| status_value = lnt.testing.FAIL |
| no_errors = False |
| elif success_value == 'xfail': |
| status_value = lnt.testing.XFAIL |
| else: |
| status_value = lnt.testing.PASS |
| |
| if test_namespace == 'nightlytest': |
| test_samples.append(lnt.testing.TestSamples( |
| test_name + '.success', |
| [status_value != lnt.testing.FAIL], test_info)) |
| else: |
| if status_value != lnt.testing.PASS: |
| test_samples.append(lnt.testing.TestSamples( |
| test_name + '.status', [status_value], test_info)) |
| if value != '*': |
| sample_test_name = test_name |
| if tname is not None: |
| sample_test_name += '.' + tname |
| test_samples.append(lnt.testing.TestSamples( |
| sample_test_name, [conv_f(value)], test_info, |
| conv_f=conv_f)) |
| |
| report_file.close() |
| |
| return test_samples, no_errors |
| |
| |
| def prepare_report_dir(config): |
| # Set up the sandbox. |
| sandbox_path = config.sandbox_path |
| print(sandbox_path) |
| if not os.path.exists(sandbox_path): |
| print("%s: creating sandbox: %r" % ( |
| timestamp(), sandbox_path), file=sys.stderr) |
| os.mkdir(sandbox_path) |
| |
| # Create the per-test directory. |
| report_dir = config.report_dir |
| if os.path.exists(report_dir): |
| needs_clean = True |
| else: |
| needs_clean = False |
| os.mkdir(report_dir) |
| |
| # Unless not using timestamps, we require the report dir not to exist. |
| if needs_clean and config.timestamp_build: |
| fatal('refusing to reuse pre-existing build dir %r' % report_dir) |
| |
| |
| def prepare_build_dir(config, iteration): |
| # report_dir is supposed to be canonicalized, so we do not need to |
| # call os.path.realpath before mkdir. |
| build_dir = config.build_dir(iteration) |
| if iteration is None: |
| return build_dir |
| |
| if os.path.exists(build_dir): |
| needs_clean = True |
| else: |
| needs_clean = False |
| os.mkdir(build_dir) |
| |
| # Unless not using timestamps, we require the basedir not to exist. |
| if needs_clean and config.timestamp_build: |
| fatal('refusing to reuse pre-existing build dir %r' % build_dir) |
| return build_dir |
| |
| |
| def update_tools(make_variables, config, iteration): |
| """Update the test suite tools. """ |
| |
| print('%s: building test-suite tools' % (timestamp(), ), file=sys.stderr) |
| args = ['make', 'tools'] |
| args.extend('%s=%s' % (k, v) for k, v in make_variables.items()) |
| build_tools_log_path = os.path.join(config.build_dir(iteration), |
| 'build-tools.log') |
| build_tools_log = open(build_tools_log_path, 'w') |
| print('%s: running: %s' % (timestamp(), |
| ' '.join('"%s"' % a |
| for a in args)), file=build_tools_log) |
| build_tools_log.flush() |
| res = execute_command(build_tools_log, config.build_dir(iteration), |
| args, config.report_dir) |
| build_tools_log.close() |
| if res != 0: |
| fatal('Unable to build tools, aborting! See log: %s' % |
| build_tools_log_path) |
| |
| |
| def configure_test_suite(config, iteration): |
| """Run configure on the test suite.""" |
| |
| basedir = config.build_dir(iteration) |
| configure_log_path = os.path.join(basedir, 'configure.log') |
| configure_log = open(configure_log_path, 'w') |
| |
| args = [os.path.realpath(os.path.join(config.test_suite_root, |
| 'configure'))] |
| if config.without_llvm: |
| args.extend(['--without-llvmsrc', '--without-llvmobj']) |
| else: |
| args.extend(['--with-llvmsrc=%s' % config.llvm_src_root, |
| '--with-llvmobj=%s' % config.llvm_obj_root]) |
| |
| if config.test_suite_externals: |
| args.append('--with-externals=%s' % |
| os.path.realpath(config.test_suite_externals)) |
| |
| args.extend(['--target=%s' % config.target]) |
| |
| print('%s: running: %s' % (timestamp(), |
| ' '.join('"%s"' % a |
| for a in args)), file=configure_log) |
| configure_log.flush() |
| |
| print('%s: configuring...' % timestamp(), file=sys.stderr) |
| res = execute_command(configure_log, basedir, args, config.report_dir) |
| configure_log.close() |
| if res != 0: |
| fatal('Configure failed, log is here: %r' % configure_log_path) |
| |
| |
| def copy_missing_makefiles(config, basedir): |
| """When running with only_test something, makefiles will be missing, |
| so copy them into place. """ |
| suffix = '' |
| for component in config.only_test.split('/'): |
| suffix = os.path.join(suffix, component) |
| obj_path = os.path.join(basedir, suffix) |
| src_path = os.path.join(config.test_suite_root, suffix) |
| if not os.path.exists(obj_path): |
| print('%s: initializing test dir %s' % (timestamp(), suffix)) |
| os.mkdir(obj_path) |
| shutil.copyfile(os.path.join(src_path, 'Makefile'), |
| os.path.join(obj_path, 'Makefile')) |
| |
| |
| def run_test(nick_prefix, iteration, config): |
| print("%s: checking source versions" % ( |
| timestamp(), ), file=sys.stderr) |
| |
| test_suite_source_version = get_source_version(config.test_suite_root) |
| |
| # Compute the make variables. |
| make_variables, public_make_variables = config.compute_run_make_variables() |
| |
| # Compute the test module variables, which are a restricted subset of the |
| # make variables. |
| test_module_variables = compute_test_module_variables(make_variables, |
| config) |
| |
| # Scan for LNT-based test modules. |
| print("%s: scanning for LNT-based test modules" % ( |
| timestamp(), ), file=sys.stderr) |
| test_modules = list(scan_for_test_modules(config)) |
| print("%s: found %d LNT-based test modules" % ( |
| timestamp(), len(test_modules)), file=sys.stderr) |
| |
| nick = nick_prefix |
| if config.auto_name: |
| # Construct the nickname from a few key parameters. |
| cc_info = config.cc_info |
| cc_nick = '%s_%s' % (cc_info.get('cc_name'), cc_info.get('cc_build')) |
| nick += "__%s__%s" % (cc_nick, cc_info.get('cc_target').split('-')[0]) |
| print("%s: using nickname: %r" % (timestamp(), nick), file=sys.stderr) |
| |
| basedir = prepare_build_dir(config, iteration) |
| |
| # FIXME: Auto-remove old test directories in the source directory (which |
| # cause make horrible fits). |
| |
| start_time = timestamp() |
| print('%s: starting test in %r' % (start_time, basedir), file=sys.stderr) |
| |
| # Configure the test suite. |
| if config.run_configure or not os.path.exists(os.path.join( |
| basedir, 'Makefile.config')): |
| configure_test_suite(config, iteration) |
| |
| # If running with --only-test, creating any dirs which might be missing and |
| # copy Makefiles. |
| if config.only_test is not None and \ |
| not config.only_test.startswith("LNTBased"): |
| copy_missing_makefiles(config, basedir) |
| |
| # If running without LLVM, make sure tools are up to date. |
| if config.without_llvm: |
| update_tools(make_variables, config, iteration) |
| |
| # Always blow away any existing report. |
| build_report_path = config.build_report_path(iteration) |
| if os.path.exists(build_report_path): |
| os.remove(build_report_path) |
| |
| # Execute the tests. |
| test_log = open(config.test_log_path(iteration), 'w') |
| |
| # Run the make driven tests if needed. |
| run_nightly_test = (config.only_test is None or |
| not config.only_test.startswith("LNTBased")) |
| if run_nightly_test: |
| execute_nt_tests(test_log, make_variables, basedir, config) |
| |
| # Run the extension test modules, if needed. |
| test_module_results = execute_test_modules(test_log, test_modules, |
| test_module_variables, basedir, |
| config, make_variables) |
| test_log.close() |
| |
| end_time = timestamp() |
| |
| # Load the nightly test samples. |
| if config.test_style == "simple": |
| test_namespace = 'nts' |
| else: |
| test_namespace = 'nightlytest' |
| if run_nightly_test: |
| print('%s: loading nightly test data...' % timestamp(), file=sys.stderr) |
| # If nightly test went screwy, it won't have produced a report. |
| print(build_report_path) |
| if not os.path.exists(build_report_path): |
| fatal('nightly test failed, no report generated') |
| |
| test_samples, no_errors = \ |
| load_nt_report_file(build_report_path, config) |
| else: |
| test_samples = [] |
| no_errors = True |
| |
| # Merge in the test samples from all of the test modules. |
| existing_tests = set(s.name for s in test_samples) |
| for module, results in test_module_results: |
| for s in results: |
| if s.name in existing_tests: |
| fatal("test module %r added duplicate test: %r" % ( |
| module, s.name)) |
| existing_tests.add(s.name) |
| test_samples.extend(results) |
| |
| print('%s: capturing machine information' % (timestamp(), ), file=sys.stderr) |
| # Collect the machine and run info. |
| # |
| # FIXME: Import full range of data that the Clang tests are using? |
| machine_info = {} |
| machine_info['hardware'] = capture(["uname", "-m"], |
| include_stderr=True).strip() |
| machine_info['os'] = capture(["uname", "-sr"], include_stderr=True).strip() |
| if config.cc_reference is not None: |
| machine_info['gcc_version'] = capture( |
| [config.cc_reference, '--version'], |
| include_stderr=True).split('\n')[0] |
| |
| # FIXME: We aren't getting the LLCBETA options. |
| run_info = {} |
| run_info['tag'] = test_namespace |
| run_info['no_errors'] = no_errors |
| run_info.update(config.cc_info) |
| |
| # Capture sw_vers if this looks like Darwin. |
| if 'Darwin' in machine_info['os']: |
| run_info['sw_vers'] = capture(['sw_vers'], include_stderr=True).strip() |
| |
| # Query remote properties if in use. |
| if config.remote: |
| remote_args = [config.remote_client, |
| "-l", config.remote_user, |
| "-p", str(config.remote_port), |
| config.remote_host] |
| run_info['remote_uname'] = capture(remote_args + ["uname", "-a"], |
| include_stderr=True).strip() |
| |
| # Capture sw_vers if this looks like Darwin. |
| if 'Darwin' in run_info['remote_uname']: |
| run_info['remote_sw_vers'] = capture(remote_args + ["sw_vers"], |
| include_stderr=True).strip() |
| |
| # Query qemu user mode properties if in use. |
| if config.qemu_user_mode: |
| run_info['qemu_user_mode'] = config.qemu_user_mode_command |
| |
| # Add machine dependent info. |
| if config.use_machdep_info: |
| machdep_info = machine_info |
| else: |
| machdep_info = run_info |
| |
| machdep_info['uname'] = capture(["uname", "-a"], |
| include_stderr=True).strip() |
| machdep_info['name'] = capture(["uname", "-n"], |
| include_stderr=True).strip() |
| |
| # FIXME: Hack, use better method of getting versions. Ideally, from |
| # binaries so we are more likely to be accurate. |
| if config.llvm_source_version is not None: |
| run_info['llvm_revision'] = config.llvm_source_version |
| run_info['test_suite_revision'] = test_suite_source_version |
| run_info.update(public_make_variables) |
| |
| # Set the run order from the user, if given. |
| if config.run_order is not None: |
| run_info['run_order'] = config.run_order |
| |
| else: |
| # Otherwise, use the inferred run order from the compiler. |
| run_info['run_order'] = config.cc_info['inferred_run_order'] |
| |
| # Add any user specified parameters. |
| for target, params in ((machine_info, config.machine_parameters), |
| (run_info, config.run_parameters)): |
| for entry in params: |
| if '=' not in entry: |
| name, value = entry, '' |
| else: |
| name, value = entry.split('=', 1) |
| if name in target: |
| logger.warning("parameter %r overwrote existing value: %r" % |
| (name, target.get(name))) |
| print(target, name, value) |
| target[name] = value |
| |
| # Generate the test report. |
| lnt_report_path = config.report_path(iteration) |
| print('%s: generating report: %r' % (timestamp(), |
| lnt_report_path), file=sys.stderr) |
| machine = lnt.testing.Machine(nick, machine_info) |
| run = lnt.testing.Run(start_time, end_time, info=run_info) |
| |
| report = lnt.testing.Report(machine, run, test_samples) |
| lnt_report_file = open(lnt_report_path, 'w') |
| print(report.render(), file=lnt_report_file) |
| lnt_report_file.close() |
| |
| return report |
| |
| ### |
| |
| |
| def _construct_report_path(basedir, only_test, test_style, file_type="csv"): |
| """Get the full path to report files in the sandbox. |
| """ |
| report_path = os.path.join(basedir) |
| if only_test is not None: |
| report_path = os.path.join(report_path, only_test) |
| report_path = os.path.join(report_path, |
| ('report.%s.' % test_style) + file_type) |
| return report_path |
| |
| |
| def rerun_test(config, name, num_times): |
| """Take the test at name, and rerun it num_times with the previous settings |
| stored in config. |
| |
| """ |
| # Extend the old log file. |
| logfile = open(config.test_log_path(None), 'a') |
| |
| # Grab the real test name instead of the LNT benchmark URL. |
| real_name = TEST_TO_NAME["nts." + name] |
| |
| relative_test_path = os.path.dirname(real_name) |
| test_name = os.path.basename(real_name) |
| |
| test_full_path = os.path.join( |
| config.report_dir, relative_test_path) |
| |
| assert os.path.exists(test_full_path), \ |
| "Previous test directory not there?" + test_full_path |
| |
| results = [] |
| no_errors = True |
| for _ in range(0, num_times): |
| test_results, t_no_errors = _execute_test_again( |
| config, test_name, test_full_path, relative_test_path, logfile) |
| no_errors &= t_no_errors |
| results.extend(test_results) |
| |
| # Check we got an exec and status from each run. |
| assert len(results) >= num_times, \ |
| "Did not get all the runs?" + str(results) |
| |
| logfile.close() |
| return results, no_errors |
| |
| |
| def _prepare_testsuite_for_rerun(test_name, test_full_path, config): |
| """Rerun step 1: wipe out old files to get ready for rerun. |
| |
| """ |
| output = os.path.join(test_full_path, "Output/") |
| test_path_prefix = output + test_name + "." |
| os.remove(test_path_prefix + "out-" + config.test_style) |
| |
| # Remove all the test-suite accounting files for this benchmark |
| to_go = glob.glob(test_path_prefix + "*.time") |
| to_go.extend(glob.glob(test_path_prefix + "*.txt")) |
| to_go.extend(glob.glob(test_path_prefix + "*.csv")) |
| |
| assert len(to_go) >= 1, "Missing at least one accounting file." |
| for path in to_go: |
| print("Removing:", path) |
| os.remove(path) |
| |
| |
| def _execute_test_again(config, test_name, test_path, test_relative_path, |
| logfile): |
| """(Re)Execute the benchmark of interest. """ |
| |
| _prepare_testsuite_for_rerun(test_name, test_path, config) |
| |
| # Grab old make invocation. |
| mk_vars, _ = config.compute_run_make_variables() |
| to_exec = ['make', '-k'] |
| to_exec.extend('%s=%s' % (k, v) for k, v in mk_vars.items()) |
| |
| # We need to run the benchmark's makefile, not the global one. |
| if config.only_test is not None: |
| to_exec.extend(['-C', config.only_test]) |
| else: |
| if test_relative_path: |
| to_exec.extend(['-C', test_relative_path]) |
| config.rerun_test = test_relative_path |
| # The target for the specific benchmark. |
| # Make target. |
| benchmark_report_target = "Output/" + test_name + \ |
| "." + config.test_style + ".report.txt" |
| # Actual file system location of the target. |
| benchmark_report_path = os.path.join(config.build_dir(None), |
| test_path, |
| benchmark_report_target) |
| to_exec.append(benchmark_report_target) |
| |
| returncode = execute_command(logfile, |
| config.build_dir(None), to_exec, |
| config.report_dir) |
| assert returncode == 0, "Remake command failed." |
| assert os.path.exists(benchmark_report_path), "Missing " \ |
| "generated report: " + benchmark_report_path |
| |
| # Now we need to pull out the results into the CSV format LNT can read. |
| schema = os.path.join(config.test_suite_root, |
| "TEST." + config.test_style + ".report") |
| result_path = os.path.join(config.build_dir(None), |
| test_path, "Output", |
| test_name + "." + config.test_style + |
| ".report.csv") |
| |
| gen_report_template = "{gen} -csv {schema} < {input} > {output}" |
| gen_cmd = gen_report_template.format(gen=config.generate_report_script, |
| schema=schema, |
| input=benchmark_report_path, |
| output=result_path) |
| bash_gen_cmd = ["/bin/bash", "-c", gen_cmd] |
| |
| assert not os.path.exists(result_path), "Results should not exist yet." + \ |
| result_path |
| returncode = execute_command(logfile, config.build_dir(None), bash_gen_cmd, |
| config.report_dir) |
| assert returncode == 0, "command failed" |
| assert os.path.exists(result_path), "Missing results file." |
| |
| results, no_errors = load_nt_report_file(result_path, config) |
| assert len(results) > 0 |
| return results, no_errors |
| |
| |
| def _unix_quote_args(s): |
| return list(map(pipes.quote, shlex.split(s))) |
| |
| |
| # When set to true, all benchmarks will be rerun. |
| # TODO: remove me when rerun patch is done. |
| NUMBER_OF_RERUNS = 4 |
| |
| SERVER_FAIL = u'FAIL' |
| SERVER_PASS = u'PASS' |
| |
| # Local test results names have these suffixes |
| # Test will have the perf suffix if it passed, or |
| # if it failed it will have a status suffix. |
| LOCAL_COMPILE_PERF = "compile" |
| LOCAL_COMPILE_STATUS = "compile.status" |
| LOCAL_HASH = "hash" |
| LOCAL_HASH_STATUS = "hash.status" |
| LOCAL_EXEC_PERF = "exec" |
| LOCAL_EXEC_STATUS = "exec.status" |
| |
| # Server results have both status and performance in each entry |
| SERVER_COMPILE_RESULT = "compile_time" |
| SERVER_EXEC_RESULT = "execution_time" |
| SERVER_SCORE_RESULT = "score" |
| SERVER_MEM_RESULT = "mem" |
| SERVER_HASH_RESULT = "hash" |
| SERVER_CODESIZE_RESULT = "code_size" |
| |
| |
| class PastRunData(object): |
| """To decide if we need to rerun, we must know |
| what happened on each test in the first runs. |
| Because the server returns data in a different format than |
| the local results, this class is comprised of a per-test |
| per-run aggregate of the two reports.""" |
| def __init__(self, name): |
| self.name = name |
| self.compile_status = None |
| self.compile_time = None |
| self.execution_status = None |
| self.execution_time = None |
| self.valid = False |
| |
| def check(self): |
| """Make sure this run data is complete.""" |
| assert self.name is not None |
| msg = "Malformed test: %s" % (repr(self)) |
| assert self.compile_status is not None, msg |
| assert self.execution_status is not None, msg |
| assert self.compile_time is not None, msg |
| |
| self.valid = True |
| |
| def is_rerunable(self): |
| """Decide if we should rerun this test.""" |
| assert self.valid |
| # Don't rerun if compile failed. |
| if self.compile_status == SERVER_FAIL: |
| return False |
| |
| # Don't rerun on correctness failure or test pass. |
| if self.execution_status == UNCHANGED_FAIL or \ |
| self.execution_status == UNCHANGED_PASS or \ |
| self.execution_status == SERVER_FAIL: |
| return False |
| |
| # Do rerun on regression or improvement. |
| if self.execution_status == REGRESSED or \ |
| self.execution_status == IMPROVED: |
| return True |
| |
| assert False, "Malformed run data: " \ |
| "you should not get here. " + str(self) |
| |
| def __repr__(self): |
| template = "<{}: CS {}, CT {}, ES {}, ET {}>" |
| return template.format(self.name, |
| repr(self.compile_status), |
| repr(self.compile_time), |
| repr(self.execution_status), |
| repr(self.execution_time)) |
| |
| |
| def _process_reruns(config, server_reply, local_results): |
| """Rerun each benchmark which the server reported "changed", N more |
| times. |
| """ |
| try: |
| server_results = server_reply['test_results'][0]['results'] |
| except KeyError: |
| # Server might send us back an error. |
| if server_reply.get('error', None): |
| logger.warning("Server returned an error:" + server_reply['error']) |
| fatal("No Server results. Cannot do reruns.") |
| # Holds the combined local and server results. |
| collated_results = dict() |
| |
| for b in local_results.tests: |
| # format: suite.test/path/and/name.type<.type> |
| fields = b.name.split('.') |
| |
| test_type_size = -1 |
| if fields[-1] == "status": |
| test_type_size = -2 |
| |
| test_type = '.'.join(fields[test_type_size:]) |
| |
| test_name = '.'.join(fields[1:test_type_size]) |
| |
| updating_entry = collated_results.get(test_name, |
| PastRunData(test_name)) |
| |
| # Filter out "LNTBased" benchmarks for rerun, they |
| # won't work. LNTbased look like nts.module.test |
| # So just filter out anything with . |
| if '.' in test_name: |
| continue |
| |
| if test_type == LOCAL_COMPILE_PERF: |
| updating_entry.compile_time = b.data |
| elif test_type == LOCAL_COMPILE_STATUS: |
| updating_entry.compile_status = SERVER_FAIL |
| elif test_type == LOCAL_EXEC_PERF: |
| updating_entry.execution_time = b.data |
| elif test_type == LOCAL_EXEC_STATUS: |
| updating_entry.execution_status = SERVER_FAIL |
| elif test_type == LOCAL_HASH: |
| # do not trigger reruns just because the binary has changed. |
| pass |
| elif test_type == LOCAL_HASH_STATUS: |
| updating_entry.hash_status = SERVER_FAIL |
| else: |
| assert False, "Unexpected local test type." |
| |
| collated_results[test_name] = updating_entry |
| |
| # Now add on top the server results to any entry we already have. |
| for full_name, results_status, perf_status in server_results: |
| fields = full_name.split(".") |
| test_name = '.'.join(fields[:-1]) |
| test_type = fields[-1] |
| |
| new_entry = collated_results.get(test_name, None) |
| # Some tests will come from the server, which we did not run locally. |
| # Drop them. |
| if new_entry is None: |
| continue |
| # Set these, if they were not set with fails above. |
| if SERVER_COMPILE_RESULT in test_type: |
| if new_entry.compile_status is None: |
| new_entry.compile_status = results_status |
| elif (SERVER_EXEC_RESULT in test_type or |
| SERVER_SCORE_RESULT in test_type or |
| SERVER_MEM_RESULT in test_type): |
| if new_entry.execution_status is None: |
| # If the server has not seen the test before, it will return |
| # None for the performance results analysis. In this case we |
| # will assume no rerun is needed, so assign unchanged. |
| if perf_status is None: |
| derived_perf_status = UNCHANGED_PASS |
| else: |
| derived_perf_status = perf_status |
| new_entry.execution_status = derived_perf_status |
| elif SERVER_CODESIZE_RESULT in test_type: |
| # NT suite can't do code size, so don't rerun. |
| pass |
| else: |
| assert False, "Unexpected server result type." + test_type |
| collated_results[test_name] = new_entry |
| |
| # Double check that all values are there for all tests. |
| for test in collated_results.values(): |
| test.check() |
| |
| rerunable_benches = list(filter(lambda bench: bench.is_rerunable(), |
| collated_results.values())) |
| rerunable_benches.sort(key=lambda x: x.name) |
| # Now lets do the reruns. |
| rerun_results = [] |
| summary = "Rerunning {} of {} benchmarks." |
| logger.info(summary.format(len(rerunable_benches), |
| len(collated_results))) |
| |
| for i, bench in enumerate(rerunable_benches): |
| logger.info("Rerunning: {} [{}/{}]".format(bench.name, |
| i + 1, |
| len(rerunable_benches))) |
| |
| fresh_samples, t_no_errors = rerun_test(config, |
| bench.name, NUMBER_OF_RERUNS) |
| rerun_results.extend(fresh_samples) |
| |
| return rerun_results |
| |
| |
| usage_info = """ |
| |
| Script for running the tests in LLVM's test-suite repository. |
| |
| This script expects to run against a particular LLVM source tree, build, and |
| compiler. It is only responsible for running the tests in the test-suite |
| repository, and formatting the results for submission to an LNT server. |
| |
| Basic usage: |
| |
| \b |
| lnt runtest nt \\ |
| --sandbox FOO \\ |
| --cc ~/llvm.obj.64/Release/bin/clang \\ |
| --test-suite ~/llvm-test-suite |
| |
| where --sandbox is the directory to build and store results in, --cc and --cxx |
| are the full paths to the compilers to test, and --test-suite is the path to |
| the test-suite source. |
| |
| To do a quick test, you can add something like: |
| |
| -j 16 --only-test SingleSource |
| |
| which will run with 16 threads and only run the tests inside SingleSource. |
| |
| To do a really quick test, you can further add |
| |
| --no-timestamp --no-configure |
| |
| which will cause the same build directory to be used, and the configure step |
| will be skipped if it appears to already have been configured. This is |
| effectively an incremental retest. It is useful for testing the scripts or |
| nightly test, but it should not be used for submissions.""" |
| |
| |
| class NTTest(builtintest.BuiltinTest): |
| def run_test(self, opts): |
| |
| opts.cflag_string = ' '.join(opts.cflag_string) |
| |
| # The --without--llvm option is the default if no LLVM paths are given. |
| if opts.llvm_src_root is None and opts.llvm_obj_root is None: |
| opts.without_llvm = True |
| |
| # Deprecate --simple |
| if opts.test_simple: |
| logger.warning("--simple is deprecated, it is the default.") |
| del opts.test_simple |
| |
| if opts.test_style == "simple": |
| # TEST=simple doesn't use a reference compiler. |
| if opts.cc_reference is not None: |
| self._fatal('--cc-reference is unused with --simple') |
| if opts.cxx_reference is not None: |
| self._fatal('--cxx-reference is unused with --simple') |
| # TEST=simple doesn't use a llc options. |
| if opts.mcpu is not None: |
| self._fatal('--mcpu is unused with --simple (use --cflag)') |
| if opts.relocation_model is not None: |
| self._fatal('--relocation-model is unused with --simple ' |
| '(use --cflag)') |
| if opts.disable_fp_elim: |
| self._fatal('--disable-fp-elim is unused with --simple ' |
| '(use --cflag)') |
| else: |
| if opts.without_llvm: |
| self._fatal('--simple is required with --without-llvm') |
| |
| # Attempt to infer cc_reference and cxx_reference if not given. |
| if opts.cc_reference is None: |
| opts.cc_reference = which('gcc') or which('cc') |
| if opts.cc_reference is None: |
| self._fatal('unable to infer --cc-reference (required)') |
| if opts.cxx_reference is None: |
| opts.cxx_reference = which('g++') or which('c++') |
| if opts.cxx_reference is None: |
| self._fatal('unable to infer --cxx-reference (required)') |
| |
| if opts.cc_under_test is None: |
| self._fatal('--cc is required') |
| |
| # Resolve the cc_under_test path. |
| opts.cc_under_test = resolve_command_path(opts.cc_under_test) |
| |
| if not lnt.testing.util.compilers.is_valid(opts.cc_under_test): |
| self._fatal('--cc does not point to a valid executable.') |
| |
| # If there was no --cxx given, attempt to infer it from the --cc. |
| if opts.cxx_under_test is None: |
| opts.cxx_under_test = \ |
| lnt.testing.util.compilers.infer_cxx_compiler( |
| opts.cc_under_test) |
| if opts.cxx_under_test is not None: |
| logger.info("inferred C++ compiler under test as: %r" % |
| (opts.cxx_under_test,)) |
| |
| # The cxx_under_test option is required if we are testing C++. |
| if opts.test_cxx and opts.cxx_under_test is None: |
| self._fatal('--cxx is required') |
| |
| if opts.cxx_under_test is not None: |
| opts.cxx_under_test = resolve_command_path(opts.cxx_under_test) |
| |
| # Always set cxx_under_test, since it may be used as the linker even |
| # when not testing C++ code. |
| if opts.cxx_under_test is None: |
| opts.cxx_under_test = opts.cc_under_test |
| |
| # Validate that the compilers under test exist. |
| if not os.path.exists(opts.cc_under_test): |
| self._fatal("invalid --cc argument %r, does not exist" % ( |
| opts.cc_under_test)) |
| if not os.path.exists(opts.cxx_under_test): |
| self._fatal("invalid --cxx argument %r, does not exist" % ( |
| opts.cxx_under_test)) |
| |
| # FIXME: As a hack to allow sampling old Clang revisions, if we are |
| # given a C++ compiler that doesn't exist, reset it to just use the |
| # given C compiler. |
| if not os.path.exists(opts.cxx_under_test): |
| logger.warning("invalid cxx_under_test, " + |
| "falling back to cc_under_test") |
| opts.cxx_under_test = opts.cc_under_test |
| |
| if opts.without_llvm: |
| if opts.llvm_src_root is not None: |
| self._fatal('--llvm-src is not allowed with --without-llvm') |
| if opts.llvm_obj_root is not None: |
| self._fatal('--llvm-obj is not allowed with --without-llvm') |
| else: |
| if opts.llvm_src_root is None: |
| self._fatal('--llvm-src is required') |
| if opts.llvm_obj_root is None: |
| self._fatal('--llvm-obj is required') |
| |
| # Make LLVM source and object paths absolute, this is required. |
| opts.llvm_src_root = os.path.abspath(opts.llvm_src_root) |
| opts.llvm_obj_root = os.path.abspath(opts.llvm_obj_root) |
| if not os.path.exists(opts.llvm_src_root): |
| self._fatal('--llvm-src argument does not exist') |
| if not os.path.exists(opts.llvm_obj_root): |
| self._fatal('--llvm-obj argument does not exist') |
| |
| if opts.test_suite_root is None: |
| self._fatal('--test-suite is required') |
| elif not os.path.exists(opts.test_suite_root): |
| self._fatal("invalid --test-suite argument, does not exist: %r" % ( |
| opts.test_suite_root)) |
| |
| if opts.remote: |
| if opts.remote_port is None: |
| self._fatal('--remote-port is required with --remote') |
| if opts.remote_user is None: |
| self._fatal('--remote-user is required with --remote') |
| else: |
| if opts.remote_port is not None: |
| self._fatal('--remote is required with --remote-port') |
| if opts.remote_user is not None: |
| self._fatal('--remote is required with --remote-user') |
| |
| if opts.spec_with_pgo and not opts.test_spec_ref: |
| self._fatal('--spec-with-pgo is only supported with ' |
| '--spec-with-ref') |
| |
| # libLTO should exist, if given. |
| if opts.liblto_path: |
| if not os.path.exists(opts.liblto_path): |
| self._fatal('invalid --liblto-path argument %r' % ( |
| opts.liblto_path,)) |
| |
| # Support disabling test suite externals separately from providing |
| # path. |
| if not opts.test_externals: |
| opts.test_suite_externals = '/dev/null' |
| else: |
| if not os.path.exists(opts.test_suite_externals): |
| self._fatal( |
| "invalid --test-externals argument, does not exist: %r" % ( |
| opts.test_suite_externals)) |
| |
| # Set up iOS simulator options. |
| if opts.ios_simulator_sdk: |
| # Warn if the user asked to run under an iOS simulator SDK, but |
| # didn't set an isysroot for compilation. |
| if opts.isysroot is None: |
| logger.warning('expected --isysroot when executing with ' |
| '--ios-simulator-sdk') |
| |
| config = TestConfiguration(vars(opts), timestamp()) |
| # FIXME: We need to validate that there is no configured output in the |
| # test-suite directory, that borks things. <rdar://problem/7876418> |
| prepare_report_dir(config) |
| |
| # These notes are used by the regression tests to check if we've |
| # handled flags correctly. |
| logger.info('TARGET_FLAGS: {}'.format(' '.join(config.target_flags))) |
| if config.qemu_user_mode: |
| logger.info('QEMU_USER_MODE_COMMAND: {}' |
| .format(config.qemu_user_mode_command)) |
| |
| # Multisample, if requested. |
| merge_run = None |
| if opts.multisample is not None: |
| # Collect the sample reports. |
| reports = [] |
| |
| for i in range(opts.multisample): |
| print("%s: (multisample) running iteration %d" %\ |
| (timestamp(), i), file=sys.stderr) |
| report = run_test(opts.label, i, config) |
| reports.append(report) |
| |
| # Create the merged report. |
| # |
| # FIXME: Do a more robust job of merging the reports? |
| print("%s: (multisample) creating merged report" % ( |
| timestamp(), ), file=sys.stderr) |
| machine = reports[0].machine |
| run = reports[0].run |
| run.end_time = reports[-1].run.end_time |
| test_samples = sum([r.tests |
| for r in reports], []) |
| |
| # Write out the merged report. |
| lnt_report_path = config.report_path(None) |
| report = lnt.testing.Report(machine, run, test_samples) |
| lnt_report_file = open(lnt_report_path, 'w') |
| print(report.render(), file=lnt_report_file) |
| lnt_report_file.close() |
| |
| else: |
| test_results = run_test(opts.label, None, config) |
| if opts.rerun: |
| self.log("Performing any needed reruns.") |
| server_report = self.submit_helper(config) |
| new_samples = _process_reruns(config, server_report, |
| test_results) |
| test_results.update_report(new_samples) |
| |
| # persist report with new samples. |
| lnt_report_path = config.report_path(None) |
| |
| lnt_report_file = open(lnt_report_path, 'w') |
| print(test_results.render(), file=lnt_report_file) |
| lnt_report_file.close() |
| merge_run = 'replace' |
| |
| if config.output is not None: |
| self.print_report(test_results, config.output) |
| |
| server_report = self.submit_helper(config, merge_run) |
| |
| ImportData.print_report_result(server_report, |
| sys.stdout, |
| sys.stderr, |
| config.verbose) |
| return server_report |
| |
| def submit_helper(self, config, merge_run=None): |
| """Submit the report to the server. If no server |
| was specified, use a local mock server. |
| """ |
| report_path = config.report_path(None) |
| assert os.path.exists(report_path), "Passed an invalid report file. " \ |
| "Should have never gotten here!" |
| |
| result = None |
| if config.submit_url: |
| from lnt.util import ServerUtil |
| for server in config.submit_url: |
| self.log("submitting result to %r" % (server,)) |
| try: |
| result = ServerUtil.submitFile(server, report_path, False, |
| merge_run=merge_run) |
| except (urllib.error.HTTPError, urllib.error.URLError) as e: |
| logger.warning("submitting to {} failed with {}" |
| .format(server, e)) |
| else: |
| # Simulate a submission to retrieve the results report. |
| # Construct a temporary database and import the result. |
| self.log("submitting result to dummy instance") |
| |
| import lnt.server.db.v4db |
| import lnt.server.config |
| db = lnt.server.db.v4db.V4DB("sqlite:///:memory:", |
| lnt.server.config.Config |
| .dummy_instance()) |
| session = db.make_session() |
| result = lnt.util.ImportData.import_and_report( |
| None, None, db, session, report_path, 'json', 'nts') |
| |
| if result is None: |
| fatal("Results were not obtained from submission.") |
| |
| return result |
| |
| |
| def _tools_check(): |
| """ |
| Check that the required software is installed in the system. |
| |
| This check is used to make sure the tests won't fail because of a missing |
| tool (like yacc or tclsh). |
| |
| As new tools end up required by the tests, add them here. |
| """ |
| from subprocess import call |
| |
| # Let's try only on Linux, for now |
| if platform.system() != 'Linux': |
| return |
| |
| FNULL = open(os.devnull, 'w') |
| |
| status = call(["which", "yacc"], stdout=FNULL, stderr=FNULL) |
| if status > 0: |
| raise SystemExit("""error: yacc not available on your system.""") |
| |
| status = call(["which", "awk"], stdout=FNULL, stderr=FNULL) |
| if status > 0: |
| raise SystemExit("""error: awk not available on your system.""") |
| |
| status = call(["which", "groff"], stdout=FNULL, stderr=FNULL) |
| if status > 0: |
| raise SystemExit("""error: groff not available on your system.""") |
| |
| status = call(["which", "tclsh"], stdout=FNULL, stderr=FNULL) |
| if status > 0: |
| raise SystemExit("""error: tclsh not available on your system.""") |
| |
| |
| # FIXME: an equivalent to argparse's add_argument_group is not implemented |
| # on click. Need to review it when such functionality is available. |
| # https://github.com/pallets/click/issues/373 |
| @click.command("nt", help=usage_info, short_help=__doc__) |
| @click.argument("label", default=platform.uname()[1], required=False, |
| type=click.UNPROCESSED) |
| # Sandbox |
| @click.option("-s", "--sandbox", "sandbox_path", required=True, |
| help="parent directory to build and run tests in", |
| type=click.UNPROCESSED) |
| @click.option("--no-timestamp", "timestamp_build", flag_value=False, |
| default=True, show_default=True, |
| help="don't timestamp build directory (for testing)") |
| @click.option("--no-configure", "run_configure", flag_value=False, |
| default=True, show_default=True, |
| help="don't run configure if Makefile.config is " |
| "present (only useful with --no-timestamp)") |
| # Inputs |
| @click.option("--without-llvm", is_flag=True, show_default=True, |
| help="don't use any LLVM source or build products") |
| @click.option("--llvm-src", "llvm_src_root", |
| help="path to the LLVM source tree", |
| type=click.UNPROCESSED) |
| @click.option("--llvm-obj", "llvm_obj_root", metavar="PATH", |
| help="path to the LLVM source tree", |
| type=click.UNPROCESSED) |
| @click.option("--test-suite", "test_suite_root", metavar="PATH", |
| help="path to the LLVM test-suite sources", |
| type=click.UNPROCESSED) |
| @click.option("--test-externals", "test_suite_externals", |
| show_default=True, |
| help="path to the LLVM test-suite externals", |
| default='/dev/null', metavar="PATH", |
| type=click.UNPROCESSED) |
| # Test Compiler |
| @click.option("--cc", "cc_under_test", metavar="CC", |
| help="path to the C compiler to test", |
| type=click.UNPROCESSED) |
| @click.option("--cxx", "cxx_under_test", metavar="CXX", |
| help="path to the C++ compiler to test", |
| type=click.UNPROCESSED) |
| @click.option("--cc-reference", |
| help="path to the reference C compiler", |
| type=click.UNPROCESSED) |
| @click.option("--cxx-reference", |
| help="path to the reference C++ compiler", |
| type=click.UNPROCESSED) |
| # Test Options |
| @click.option("--arch", help="Set -arch in TARGET_FLAGS") |
| @click.option("--llvm-arch", |
| help="Set the ARCH value used in the makefiles to", |
| type=click.UNPROCESSED) |
| @click.option("--make-param", "make_parameters", multiple=True, |
| help="Add 'NAME' = 'VAL' to the makefile parameters", |
| type=click.UNPROCESSED) |
| @click.option("--isysroot", "isysroot", metavar="PATH", |
| help="Set -isysroot in TARGET_FLAGS", |
| type=click.UNPROCESSED) |
| @click.option("--liblto-path", metavar="PATH", |
| help="Specify the path to the libLTO library", |
| type=click.UNPROCESSED) |
| @click.option("--mcpu", metavar="CPU", |
| help="Set -mcpu in TARGET_LLCFLAGS", |
| type=click.UNPROCESSED) |
| @click.option("--relocation-model", metavar="MODEL", |
| help="Set -relocation-model in TARGET_LLCFLAGS", |
| type=click.UNPROCESSED) |
| @click.option("--disable-fp-elim", is_flag=True, |
| help="Set -disable-fp-elim in TARGET_LLCFLAGS") |
| @click.option("--optimize-option", show_default=True, |
| help="Set optimization level for {LLC_,LLI_,}OPTFLAGS", |
| type=click.Choice(['-O0', '-O1', '-O2', |
| '-O3', '-Os', '-Oz']), |
| default='-O3') |
| @click.option("--cflag", "cflags", multiple=True, |
| help="Additional flags to set in TARGET_FLAGS", |
| type=click.UNPROCESSED, default=[], metavar="FLAG") |
| @click.option("--cflags", "cflag_string", multiple=True, |
| help="Additional flags to set in TARGET_FLAGS, space " |
| "separated string. These flags are appended after " |
| "*all* the individual --cflag arguments.", |
| type=click.UNPROCESSED, default='', metavar="FLAG") |
| @click.option("--mllvm", multiple=True, |
| help="Add -mllvm FLAG to TARGET_FLAGS", |
| type=click.UNPROCESSED, default=[], metavar="FLAG") |
| @click.option("--spec-with-pgo", is_flag=True, help="Use PGO with SPEC") |
| @click.option("--build-mode", metavar="NAME", show_default=True, |
| default='Release+Asserts', |
| help="Select the LLVM build mode to use", |
| type=click.UNPROCESSED) |
| @click.option("--simple", "test_simple", is_flag=True, |
| help="Use TEST=simple instead of TEST=nightly") |
| @click.option("--test-style", type=click.Choice(['nightly', 'simple']), |
| default='simple', help="Set the test style to run") |
| @click.option("--test-time-stat", type=click.Choice(['user', 'real']), |
| default='user', show_default=True, |
| help="Set the test timing statistic to gather") |
| @click.option("--disable-cxx", "test_cxx", flag_value=False, default=True, |
| show_default=True, help="Disable C++ tests") |
| @click.option("--disable-externals", "test_externals", flag_value=False, |
| default=True, show_default=True, |
| help="Disable test suite externals (if configured)") |
| @click.option("--enable-integrated-as", "test_integrated_as", is_flag=True, |
| help="Enable TEST_INTEGRATED_AS tests") |
| @click.option("--enable-jit", "test_jit", is_flag=True, |
| help="Enable JIT tests") |
| @click.option("--disable-llc", "test_llc", |
| help="Disable LLC tests", |
| flag_value=False, show_default=True, default=True) |
| @click.option("--enable-llcbeta", "test_llcbeta", |
| help="Enable LLCBETA tests", is_flag=True) |
| @click.option("--disable-lto", "test_lto", |
| help="Disable use of link-time optimization", |
| flag_value=False, show_default=True, default=True) |
| @click.option("--small", "test_small", |
| help="Use smaller test inputs and disable large tests", |
| is_flag=True) |
| @click.option("--large", "test_large", |
| help="Use larger test inputs", is_flag=True) |
| @click.option("--spec-with-ref", "test_spec_ref", |
| help="Use reference test inputs for SPEC. " |
| "This is currently experimental", is_flag=True) |
| @click.option("--benchmarking-only", "test_benchmarking_only", |
| help="Benchmarking-only mode", is_flag=True) |
| @click.option("--only-test", "only_test", metavar="PATH", |
| help="Only run tests under PATH", |
| type=click.UNPROCESSED, default=None) |
| @click.option("--include-test-examples", |
| help="Include test module examples", |
| is_flag=True) |
| # Test Execution |
| @click.option("-j", "--threads", "threads", |
| help="Number of testing threads", |
| type=int, default=1, metavar="N") |
| @click.option("--build-threads", "build_threads", |
| help="Number of compilation threads", |
| type=int, default=0, metavar="N") |
| @click.option("--use-perf", type=click.UNPROCESSED, default=None, |
| help="Use perf to obtain high accuracy timing") |
| @click.option("--rerun", help="Rerun tests that have regressed.", |
| is_flag=True) |
| @click.option("--remote", is_flag=True, |
| help="Execute remotely, see " |
| "--remote-{host,port,user,client}") |
| @click.option("--remote-host", default="localhost", metavar="HOST", |
| help="Set remote execution host", type=click.UNPROCESSED) |
| @click.option("--remote-port", type=int, default=None, |
| metavar="PORT", help="Set remote execution port") |
| @click.option("--remote-user", help="Set remote execution user", |
| type=click.UNPROCESSED, default=None, metavar="USER",) |
| @click.option("--remote-client", type=click.UNPROCESSED, default="ssh", |
| help="Set remote execution client", metavar="RSH") |
| @click.option("--use-ios-simulator", "ios_simulator_sdk", |
| help=("Execute using an iOS simulator SDK (using " |
| "environment overrides)"), |
| type=click.UNPROCESSED, default=None, metavar="SDKPATH") |
| @click.option("--use-isolation", "use_isolation", |
| help=("Execute using a sandboxing profile to limit " |
| "OS access (e.g., to the network or " |
| "non-test directories)"), |
| is_flag=True) |
| @click.option("--qemu-user-mode", "qemu_user_mode", |
| help=("Enable qemu user mode emulation using this " |
| "qemu executable"), |
| type=click.UNPROCESSED, default=None) |
| @click.option("--qemu-flag", "qemu_flags", |
| help="Additional flags to pass to qemu", multiple=True, |
| type=click.UNPROCESSED, metavar="FLAG") |
| @click.option("--qemu-flags", "qemu_string", |
| help="Additional flags to pass to qemu, space " |
| "separated string. These flags are appended after " |
| "*all* the individual --qemu-flag arguments.", |
| type=click.UNPROCESSED, default='', metavar="FLAG") |
| @click.option("--multisample", "multisample", |
| help="Accumulate test data from multiple runs", |
| type=int, default=None, metavar="N") |
| # Output Options |
| @click.option("--no-auto-name", "auto_name", |
| help="Don't automatically derive submission name", |
| flag_value=False, show_default=True, default=True) |
| @click.option("--no-machdep-info", "use_machdep_info", |
| help=("Don't put machine (instance) dependent " |
| "variables with machine info"), |
| flag_value=False, show_default=True, default=True) |
| @click.option("--run-order", "run_order", metavar="STR", |
| help="String to use to identify and order this run", |
| type=click.UNPROCESSED, default=None) |
| @click.option("--machine-param", "machine_parameters", |
| metavar="NAME=VAL", |
| help="Add 'NAME' = 'VAL' to the machine parameters", |
| type=click.UNPROCESSED, multiple=True, default=[]) |
| @click.option("--run-param", "run_parameters", |
| metavar="NAME=VAL", |
| help="Add 'NAME' = 'VAL' to the run parameters", |
| type=click.UNPROCESSED, multiple=True, default=[]) |
| @click.option("--submit", "submit_url", metavar="URLORPATH", |
| help=("autosubmit the test result to the given server" |
| " (or local instance)"), |
| type=click.UNPROCESSED, default=[], multiple=True) |
| @click.option("--output", "output", metavar="PATH", |
| help="write raw report data to PATH (or stdout if '-')", |
| default=None) |
| @click.option("-v", "--verbose", "verbose", |
| help="show verbose test results", is_flag=True) |
| @click.option("--exclude-stat-from-submission", |
| "exclude_stat_from_submission", |
| help="Do not submit the stat of this type ", |
| multiple=True, type=click.Choice(KNOWN_SAMPLE_KEYS), |
| default=['hash']) |
| @submit_options |
| def cli_action(*args, **kwargs): |
| _tools_check() |
| nt = NTTest() |
| |
| for key, value in kwargs.items(): |
| setattr(nt.opts, key, value) |
| |
| results = nt.run_test(nt.opts) |
| nt.show_results_url(results) |
