| #!/usr/bin/env python |
| # -*- Python -*- |
| from __future__ import print_function |
| |
| import os |
| import shutil |
| import stat |
| |
| # We import test functionity from LNT. |
| from lnt.tests import nt |
| from lnt.testing import TestSamples, PASS, FAIL, XFAIL |
| from lnt.testing.util import commands |
| |
| # Implement cp -rf such that src can be both a file or a directory and if |
| # the dst exists create the file/dir under dst rather than rename on the |
| # fly. |
| def cp_rf(src, dst): |
| if os.path.isfile(src): |
| shutil.copy(src, dst) |
| else: |
| newdst = dst |
| if os.path.exists(dst): |
| newdst = os.path.join(dst, os.path.basename(src)) |
| shutil.copytree(src, newdst) |
| |
| |
| # Test module classes are currently required to subclass 'nt.TestModule'. |
| class TestModule(nt.TestModule): |
| |
| timeout = 5000 |
| |
| def __init__(self): |
| # The subdirectory name under externals E.g. CINT2006 |
| self.suite = "C" + self.category.upper() + str(self.year) |
| |
| # Name to identify this LNT test, e.g. nts.LNT/CINT2006_ref/456.hmmer |
| self.testname = "nts.LNT/%s_ref/%s" % (self.suite, self.benchmark) |
| |
| # Executable name, e.g. 456.hmmer.simple |
| self.exe = self.benchmark + ".simple" |
| |
| def configure_test_suite(self, working_dir): |
| args = [ |
| os.path.realpath(os.path.join(self.config.test_suite_root, "configure")) |
| ] |
| if self.config.without_llvm: |
| args.extend(["--without-llvmsrc", "--without-llvmobj"]) |
| else: |
| args.extend( |
| [ |
| "--with-llvmsrc=%s" % self.config.llvm_src_root, |
| "--with-llvmobj=%s" % self.config.llvm_obj_root, |
| ] |
| ) |
| |
| if self.config.test_suite_externals: |
| args.append( |
| "--with-externals=%s" |
| % os.path.realpath(self.config.test_suite_externals) |
| ) |
| |
| return self.call(args, cwd=working_dir) |
| |
| def fail(self): |
| return [TestSamples(self.testname + ".compile.status", [FAIL])] |
| |
| def copy_input_set(self, dest_dir, size): |
| if size == "train": |
| input_set = self.train_input_set |
| elif size == "ref": |
| input_set = self.ref_input_set |
| else: |
| fatal("only train and ref are supported for now") |
| |
| for f in input_set: |
| cp_rf(os.path.join(self.datadir, f), dest_dir) |
| |
| def copy_output_set(self, dest_dir, size): |
| if size == "train": |
| output_set = self.train_output_set |
| else: |
| output_set = self.ref_output_set |
| |
| for f in output_set: |
| cp_rf(os.path.join(self.datadir, f), dest_dir) |
| |
| def run_safely(self, args, **kwargs): |
| if kwargs.get("shell", False): |
| argstr = args |
| else: |
| argstr = " ".join(args) |
| kwargs["shell"] = True |
| |
| if "file_index" in kwargs: |
| file_index = kwargs["file_index"] |
| # Remove so that it can be forwarded to self.call. |
| del kwargs["file_index"] |
| else: |
| file_index = 0 |
| |
| # Generate a unique file for each training run (if requested) |
| timeit = [ |
| "LLVM_PROFILE_FILE=run_%d.profraw" % file_index, |
| os.path.join(self.OBJROOT, "tools", "timeit-target"), |
| ] |
| |
| cwd = kwargs.get("cwd", os.getcwd()) |
| summary_file = os.path.join(cwd, "summary_%d.time" % file_index) |
| timeit.extend( |
| [ |
| "--limit-core", |
| "0", |
| "--limit-cpu", |
| str(self.timeout), |
| "--timeout", |
| str(self.timeout), |
| "--limit-file-size", |
| "104857600", |
| "--limit-rss-size", |
| "838860800", |
| "--summary", |
| summary_file, |
| ] |
| ) |
| timeit.append(argstr) |
| cmdstr = " ".join(timeit) |
| |
| if self.config.remote: |
| command_file = os.path.join(cwd, "command_%d" % file_index) |
| with open(command_file, "w") as f: |
| # Chdir here so that the redirects are put into CWD as well. |
| remote_command = "cd %s\n%s\n" % (cwd, cmdstr) |
| print("command:", remote_command, file=self.log, end=" ") |
| f.write(remote_command) |
| |
| st = os.stat(command_file) |
| os.chmod(command_file, st.st_mode | stat.S_IEXEC) |
| |
| res = self.call( |
| [ |
| self.config.remote_client, |
| "-l", |
| self.config.remote_user, |
| self.config.remote_host, |
| "-p", |
| str(self.config.remote_port), |
| command_file, |
| ] |
| ) |
| |
| else: |
| kwargs["shell"] = True |
| res = self.call(cmdstr, **kwargs) |
| |
| if res != 0: |
| return (res, 0) |
| |
| summary = open(summary_file, "r").readlines() |
| status = [line.split()[1] for line in summary if line.startswith("exit")] |
| assert len(status) == 1, "incorrect exit status" |
| time = [ |
| line.split()[1] |
| for line in summary |
| if line.startswith(self.config.test_time_stat) |
| ] |
| assert len(time) == 1, "incorrect ellapsed time" |
| |
| return (int(status[0]), float(time[0])) |
| |
| def execute_test(self, options, make_variables, config): |
| MODULENAME = options["MODULENAME"] |
| self.SRCROOT = options["SRCROOT"] |
| self.config = config |
| self.OBJROOT = options["OBJROOT"] |
| CC = options["CC"] |
| CFLAGS = options["CFLAGS"] |
| TEST_SUITE_EXTERNALS = config.test_suite_externals |
| |
| # test-suite-externals//speccpu2006/benchspec/CPU2006/400.perlbench vs |
| # test-suite-externals//speccpu2000/benchspec/CINT2000/164.gzip |
| suitedir = os.path.join( |
| TEST_SUITE_EXTERNALS, "speccpu" + str(self.year), "benchspec" |
| ) |
| if self.year == "2000": |
| suitedir = os.path.join(suitedir, "C%s2000" % self.category.upper()) |
| else: |
| suitedir = os.path.join(suitedir, "CPU" + self.year) |
| |
| self.datadir = os.path.join(suitedir, self.benchmark, "data") |
| print( |
| "%s\n options: %s\n\nmake variables: %s\n\n" |
| % (self.testname, str(options), str(make_variables)), |
| file=self.log, |
| ) |
| |
| if not os.path.exists(self.datadir): |
| print("skipping, no source under externals", file=self.log) |
| return [] |
| |
| res = self.configure_test_suite(self.OBJROOT) |
| if res != 0: |
| return self.fail() |
| |
| make_cmd = ["make", "-k"] |
| make_cmd.extend("%s=%s" % (k, v) for k, v in make_variables.items()) |
| |
| # RunSafely.sh requires timeit, build it. |
| res = self.call(make_cmd + ["tools"], cwd=self.OBJROOT) |
| if res != 0: |
| return self.fail() |
| |
| make_cmd.append("USE_SPEC_TEST_MODULE=1") |
| |
| # Run make clean to create the benchmark directories. |
| external_obj = os.path.join( |
| self.OBJROOT, "External", "SPEC", self.suite, self.benchmark |
| ) |
| clean_cmd = make_cmd |
| external_spec_obj = os.path.split(external_obj)[0] |
| result = self.call(make_cmd + ["-C", external_spec_obj, "clean"]) |
| if result != 0: |
| return self.fail() |
| |
| # From this on, we are operating in the 456.hmmer directory. |
| make_cmd.extend(["-C", external_obj]) |
| exe_file = os.path.join("Output", self.exe) |
| |
| if config.spec_with_pgo: |
| pgo_dir = os.path.join(self.OBJROOT, "pgo") |
| os.mkdir(pgo_dir) |
| |
| target_flags = make_variables["TARGET_FLAGS"] + " -fprofile-instr-generate" |
| pgo_make_cmd = make_cmd[:] |
| pgo_make_cmd.append("TARGET_FLAGS=%s" % target_flags) |
| result = self.call(pgo_make_cmd + [exe_file]) |
| |
| exe = os.path.join(external_obj, exe_file) |
| if result != 0 or not os.path.exists(exe): |
| return self.fail() |
| shutil.copy(exe, pgo_dir) |
| |
| self.copy_input_set(pgo_dir, "train") |
| |
| for (i, args) in enumerate(self.train_args): |
| pgo_cmd = "./%s %s" % (self.exe, args) |
| (result, time) = self.run_safely( |
| pgo_cmd, cwd=pgo_dir, shell=True, env=os.environ, file_index=i |
| ) |
| if result != 0: |
| return self.fail() |
| if not os.path.exists(os.path.join(pgo_dir, "run_%d.profraw" % i)): |
| print("Failed to create PGO output", file=self.log) |
| return self.fail() |
| |
| llvm_profdata = os.path.join(os.path.dirname(CC), "llvm-profdata") |
| result = self.call( |
| llvm_profdata + " merge -output=code.profdata pgo/run_*.profraw", |
| shell=True, |
| cwd=self.OBJROOT, |
| ) |
| if result != 0: |
| return self.fail() |
| |
| target_flags = ( |
| make_variables["TARGET_FLAGS"] |
| + " -fprofile-instr-use=" |
| + os.path.join(self.OBJROOT, "code.profdata") |
| ) |
| make_cmd.append("TARGET_FLAGS=%s" % target_flags) |
| |
| result = self.call(make_cmd + ["-C", external_spec_obj, "clean"]) |
| if result != 0: |
| return self.fail() |
| |
| start_time = self.get_time() |
| result = self.call(make_cmd + [exe_file]) |
| compile_time = self.get_time() - start_time |
| |
| exe = os.path.join(external_obj, exe_file) |
| if result != 0 or not os.path.exists(exe): |
| return self.fail() |
| |
| shutil.copy(exe, self.OBJROOT) |
| |
| self.copy_input_set(self.OBJROOT, "ref") |
| self.copy_output_set(self.OBJROOT, "ref") |
| |
| run_cmds = ["./%s %s" % (self.exe, args) for args in self.ref_args] |
| status = PASS |
| |
| exec_time = 0 |
| for (file_index, cmd) in enumerate(run_cmds): |
| (result, time) = self.run_safely( |
| cmd, cwd=self.OBJROOT, shell=True, file_index=file_index |
| ) |
| if result != 0: |
| status = FAIL |
| exec_time += time |
| |
| os.environ["PATH"] += ":" + os.path.join(self.OBJROOT, "tools") |
| for cmd in self.ref_cmp_cmds: |
| if self.call(cmd, cwd=self.OBJROOT, shell=True, env=os.environ) != 0: |
| status = FAIL |
| |
| result = [ |
| TestSamples(self.testname + ".compile", [compile_time]), |
| TestSamples(self.testname + ".exec", [exec_time]), |
| ] |
| if status == FAIL: |
| result.append(TestSamples(self.testname + ".exec.status", [status])) |
| |
| return result |
