| import buildbot |
| import buildbot.process.factory |
| import copy |
| import os |
| from datetime import datetime |
| |
| from buildbot.process.properties import WithProperties, Property |
| from buildbot.steps.shell import Configure, ShellCommand, SetProperty |
| from buildbot.steps.shell import WarningCountingShellCommand |
| from buildbot.steps.source import SVN |
| from buildbot.steps.transfer import FileDownload |
| |
| import zorg.buildbot.util.artifacts as artifacts |
| import zorg.buildbot.builders.Util as builders_util |
| import zorg.buildbot.util.phasedbuilderutils as phasedbuilderutils |
| import zorg.buildbot.commands as commands |
| import zorg.buildbot.commands.BatchFileDownload as batch_file_download |
| import zorg.buildbot.commands.LitTestCommand as lit_test_command |
| from zorg.buildbot.conditions.FileConditions import FileDoesNotExist |
| from zorg.buildbot.commands.CmakeCommand import CmakeCommand |
| from zorg.buildbot.process.factory import LLVMBuildFactory |
| |
| # FIXME: This method is deprecated and will be removed. Please use getClangCMakeBuildFactory instead. |
| def getClangBuildFactory( |
| triple=None, |
| clean=True, |
| test=True, |
| package_dst=None, |
| run_cxx_tests=False, |
| examples=False, |
| valgrind=False, |
| valgrindLeakCheck=False, |
| useTwoStage=False, |
| completely_clean=False, |
| make='make', |
| jobs="%(jobs)s", |
| stage1_config='Debug+Asserts', |
| stage2_config='Release+Asserts', |
| env={}, # Environmental variables for all steps. |
| extra_configure_args=[], |
| stage2_extra_configure_args=[], |
| use_pty_in_tests=False, |
| trunk_revision=None, |
| force_checkout=False, |
| extra_clean_step=None, |
| checkout_compiler_rt=False, |
| checkout_lld=False, |
| run_gdb=False, |
| run_modern_gdb=False, |
| run_gcc=False): |
| # Prepare environmental variables. Set here all env we want everywhere. |
| merged_env = { |
| 'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences. |
| } |
| if env is not None: |
| # Overwrite pre-set items with the given ones, so user can set anything. |
| merged_env.update(env) |
| |
| llvm_srcdir = "llvm.src" |
| llvm_1_objdir = "llvm.obj" |
| llvm_1_installdir = "llvm.install.1" |
| llvm_2_objdir = "llvm.obj.2" |
| llvm_2_installdir = "llvm.install" |
| |
| f = buildbot.process.factory.BuildFactory() |
| |
| # Determine the build directory. |
| f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir", |
| command=["pwd"], |
| property="builddir", |
| description="set build dir", |
| workdir=".", |
| env=merged_env)) |
| |
| # Blow away completely, if requested. |
| if completely_clean: |
| f.addStep(ShellCommand(name="rm-llvm.src", |
| command=["rm", "-rf", llvm_srcdir], |
| haltOnFailure=True, |
| description=["rm src dir", "llvm"], |
| workdir=".", |
| env=merged_env)) |
| |
| # Checkout sources. |
| if trunk_revision: |
| # The SVN build step provides no mechanism to check out a specific revision |
| # based on a property, so just run the commands directly here. |
| svn_co = ['svn', 'checkout'] |
| if force_checkout: |
| svn_co += ['--force'] |
| svn_co += ['--revision', WithProperties(trunk_revision)] |
| |
| svn_co_llvm = svn_co + \ |
| [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' % |
| trunk_revision), |
| llvm_srcdir] |
| svn_co_clang = svn_co + \ |
| [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' % |
| trunk_revision), |
| '%s/tools/clang' % llvm_srcdir] |
| svn_co_clang_tools_extra = svn_co + \ |
| [WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' % |
| trunk_revision), |
| '%s/tools/clang/tools/extra' % llvm_srcdir] |
| |
| f.addStep(ShellCommand(name='svn-llvm', |
| command=svn_co_llvm, |
| haltOnFailure=True, |
| workdir='.')) |
| f.addStep(ShellCommand(name='svn-clang', |
| command=svn_co_clang, |
| haltOnFailure=True, |
| workdir='.')) |
| f.addStep(ShellCommand(name='svn-clang-tools-extra', |
| command=svn_co_clang_tools_extra, |
| haltOnFailure=True, |
| workdir='.')) |
| else: |
| f.addStep(SVN(name='svn-llvm', |
| mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/llvm/', |
| defaultBranch='trunk', |
| workdir=llvm_srcdir)) |
| f.addStep(SVN(name='svn-clang', |
| mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/cfe/', |
| defaultBranch='trunk', |
| workdir='%s/tools/clang' % llvm_srcdir)) |
| f.addStep(SVN(name='svn-clang-tools-extra', |
| mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/', |
| defaultBranch='trunk', |
| workdir='%s/tools/clang/tools/extra' % llvm_srcdir)) |
| if checkout_compiler_rt: |
| f.addStep(SVN(name='svn-compiler-rt', |
| mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/compiler-rt/', |
| defaultBranch='trunk', |
| workdir='%s/projects/compiler-rt' % llvm_srcdir)) |
| |
| # Clean up llvm (stage 1); unless in-dir. |
| if clean and llvm_srcdir != llvm_1_objdir: |
| f.addStep(ShellCommand(name="rm-llvm.obj.stage1", |
| command=["rm", "-rf", llvm_1_objdir], |
| haltOnFailure=True, |
| description=["rm build dir", "llvm"], |
| workdir=".", |
| env=merged_env)) |
| |
| if not clean: |
| expected_makefile = 'Makefile' |
| f.addStep(SetProperty(name="Makefile_isready", |
| workdir=llvm_1_objdir, |
| command=["sh", "-c", |
| "test -e %s && echo OK || echo Missing" % expected_makefile], |
| flunkOnFailure=False, |
| property="exists_Makefile")) |
| |
| cmake_triple_arg = [] |
| if triple: |
| cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple] |
| f.addStep(ShellCommand(name='cmake', |
| command=['cmake', |
| '-DLLVM_BUILD_TESTS=ON', |
| '-DCMAKE_BUILD_TYPE=%s' % stage1_config] + |
| cmake_triple_arg + |
| extra_configure_args + |
| ["../" + llvm_srcdir], |
| description='cmake stage1', |
| workdir=llvm_1_objdir, |
| env=merged_env, |
| doStepIf=lambda step: step.build.getProperty("exists_Makefile") != "OK")) |
| |
| # Make clean if using in-dir builds. |
| if clean and llvm_srcdir == llvm_1_objdir: |
| f.addStep(WarningCountingShellCommand(name="clean-llvm", |
| command=[make, "clean"], |
| haltOnFailure=True, |
| description="cleaning llvm", |
| descriptionDone="clean llvm", |
| workdir=llvm_1_objdir, |
| doStepIf=clean, |
| env=merged_env)) |
| |
| if extra_clean_step: |
| f.addStep(extra_clean_step) |
| |
| f.addStep(WarningCountingShellCommand(name="compile", |
| command=['nice', '-n', '10', |
| make, WithProperties("-j%s" % jobs)], |
| haltOnFailure=True, |
| flunkOnFailure=not run_gdb, |
| description=["compiling", stage1_config], |
| descriptionDone=["compile", stage1_config], |
| workdir=llvm_1_objdir, |
| env=merged_env)) |
| |
| if examples: |
| f.addStep(WarningCountingShellCommand(name="compile.examples", |
| command=['nice', '-n', '10', |
| make, WithProperties("-j%s" % jobs), |
| "BUILD_EXAMPLES=1"], |
| haltOnFailure=True, |
| description=["compiling", stage1_config, "examples"], |
| descriptionDone=["compile", stage1_config, "examples"], |
| workdir=llvm_1_objdir, |
| env=merged_env)) |
| |
| clangTestArgs = '-v -j %s' % jobs |
| if valgrind: |
| clangTestArgs += ' --vg' |
| if valgrindLeakCheck: |
| clangTestArgs += ' --vg-leak' |
| clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp' |
| extraTestDirs = '' |
| if run_cxx_tests: |
| extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests' |
| if test: |
| f.addStep(lit_test_command.LitTestCommand(name='check-all', |
| command=[make, "check-all", "VERBOSE=1", |
| WithProperties("LIT_ARGS=%s" % clangTestArgs), |
| WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)], |
| flunkOnFailure=not run_gdb, |
| description=["checking"], |
| descriptionDone=["checked"], |
| workdir=llvm_1_objdir, |
| usePTY=use_pty_in_tests, |
| env=merged_env)) |
| |
| # TODO: Install llvm and clang for stage1. |
| |
| if run_gdb or run_gcc or run_modern_gdb: |
| ignores = getClangTestsIgnoresFromPath(os.path.expanduser('~/public/clang-tests'), 'clang-x86_64-darwin10') |
| install_prefix = "%%(builddir)s/%s" % llvm_1_installdir |
| if run_gdb: |
| addClangGDBTests(f, ignores, install_prefix) |
| if run_modern_gdb: |
| addModernClangGDBTests(f, jobs, install_prefix) |
| if run_gcc: |
| addClangGCCTests(f, ignores, install_prefix) |
| |
| if not useTwoStage: |
| if package_dst: |
| name = WithProperties( |
| "%(builddir)s/" + llvm_1_objdir + |
| "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") |
| f.addStep(ShellCommand(name='pkg.tar', |
| description="tar root", |
| command=["tar", "zcvf", name, "./"], |
| workdir=llvm_1_installdir, |
| warnOnFailure=True, |
| flunkOnFailure=False, |
| haltOnFailure=False, |
| env=merged_env)) |
| f.addStep(ShellCommand(name='pkg.upload', |
| description="upload root", |
| command=["scp", name, |
| WithProperties( |
| package_dst + "/%(buildername)s")], |
| workdir=".", |
| warnOnFailure=True, |
| flunkOnFailure=False, |
| haltOnFailure=False, |
| env=merged_env)) |
| |
| return f |
| |
| # Clean up llvm (stage 2). |
| # |
| # We always cleanly build the stage 2. If the compiler has been |
| # changed on the stage 1, we cannot trust any of the intermediate file |
| # from the old compiler. And if the stage 1 compiler is the same, we should |
| # not build in the first place. |
| f.addStep(ShellCommand(name="rm-llvm.obj.stage2", |
| command=["rm", "-rf", llvm_2_objdir], |
| haltOnFailure=True, |
| description=["rm build dir", "llvm", "(stage 2)"], |
| workdir=".", |
| env=merged_env)) |
| |
| # Configure llvm (stage 2). |
| f.addStep(ShellCommand(name='cmake', |
| command=['cmake'] + stage2_extra_configure_args + [ |
| '-DLLVM_BUILD_TESTS=ON', |
| WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang' % llvm_1_objdir), # FIXME use installdir |
| WithProperties('-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' % llvm_1_objdir), |
| '-DCMAKE_BUILD_TYPE=%s' % stage2_config, |
| "../" + llvm_srcdir], |
| description='cmake stage2', |
| workdir=llvm_2_objdir, |
| env=merged_env)) |
| |
| # Build llvm (stage 2). |
| f.addStep(WarningCountingShellCommand(name="compile.llvm.stage2", |
| command=['nice', '-n', '10', |
| make, WithProperties("-j%s" % jobs)], |
| haltOnFailure=True, |
| description=["compiling", "(stage 2)", |
| stage2_config], |
| descriptionDone=["compile", "(stage 2)", |
| stage2_config], |
| workdir=llvm_2_objdir, |
| env=merged_env)) |
| |
| if test: |
| f.addStep(lit_test_command.LitTestCommand(name='check-all', |
| command=[make, "check-all", "VERBOSE=1", |
| WithProperties("LIT_ARGS=%s" % clangTestArgs), |
| WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)], |
| description=["checking"], |
| descriptionDone=["checked"], |
| workdir=llvm_2_objdir, |
| usePTY=use_pty_in_tests, |
| env=merged_env)) |
| |
| # TODO: Install llvm and clang for stage2. |
| |
| if package_dst: |
| name = WithProperties( |
| "%(builddir)s/" + llvm_2_objdir + |
| "/clang-r%(got_revision)s-b%(buildnumber)s.tgz") |
| f.addStep(ShellCommand(name='pkg.tar', |
| description="tar root", |
| command=["tar", "zcvf", name, "./"], |
| workdir=llvm_2_installdir, |
| warnOnFailure=True, |
| flunkOnFailure=False, |
| haltOnFailure=False, |
| env=merged_env)) |
| f.addStep(ShellCommand(name='pkg.upload', |
| description="upload root", |
| command=["scp", name, |
| WithProperties( |
| package_dst + "/%(buildername)s")], |
| workdir=".", |
| warnOnFailure=True, |
| flunkOnFailure=False, |
| haltOnFailure=False, |
| env=merged_env)) |
| |
| return f |
| |
| def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env, |
| gcs_url_property=None, use_pixz_compression=False, |
| xz_compression_factor=6): |
| """ |
| Add steps to upload to the Google Cloud Storage bucket. |
| |
| f - The BuildFactory to modify. |
| package_name - The name of this package for the descriptions (e.g. |
| 'stage 1') |
| install_prefix - The directory the build has been installed to. |
| gcs_directory - The subdirectory of the bucket root to upload to. This |
| should match the builder name. |
| env - The environment to use. Set BOTO_CONFIG to use a configuration file |
| in a non-standard location, and BUCKET to use a different GCS bucket. |
| gcs_url_property - Property to assign the GCS url to. |
| """ |
| |
| gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/' |
| 'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz') |
| time_fmt = '%Y-%m-%d_%H-%M-%S' |
| output_file_name = '../install.tar.xz' |
| |
| gcs_url = \ |
| WithProperties( |
| gcs_url_fmt, |
| gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'), |
| gcs_directory=lambda _: gcs_directory, |
| now=lambda _: datetime.utcnow().strftime(time_fmt)) |
| |
| if gcs_url_property: |
| f.addStep(SetProperty( |
| name="record GCS url for " + package_name, |
| command=['echo', gcs_url], |
| property=gcs_url_property)) |
| |
| if use_pixz_compression: |
| # tweak the xz compression level to generate packages faster |
| tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.'] |
| else: |
| xz_command = 'xz -{0}'.format(xz_compression_factor) |
| tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.'] |
| |
| f.addStep(ShellCommand(name='package ' + package_name, |
| command=tar_command, |
| description='packaging ' + package_name + '...', |
| workdir=install_prefix, |
| env=env)) |
| |
| f.addStep(ShellCommand( |
| name='upload ' + package_name + ' to storage bucket', |
| command=['gsutil', 'cp', '../install.tar.xz', gcs_url], |
| description=('uploading ' + package_name + |
| 'to storage bucket ...'), |
| workdir=install_prefix, |
| env=env)) |
| |
| def getClangCMakeGCSBuildFactory( |
| clean=True, |
| test=True, |
| cmake='cmake', |
| jobs=None, |
| |
| # VS tools environment variable if using MSVC. For example, |
| # %VS120COMNTOOLS% selects the 2013 toolchain. |
| vs=None, |
| vs_target_arch='x86', |
| |
| # Multi-stage compilation |
| useTwoStage=False, |
| testStage1=True, |
| stage1_config='Release', |
| stage2_config='Release', |
| |
| # Test-suite |
| runTestSuite=False, |
| nt_flags=None, |
| testsuite_flags=None, |
| submitURL=None, |
| testerName=None, |
| |
| # Environmental variables for all steps. |
| env=None, |
| extra_cmake_args=None, |
| |
| # Extra repositories |
| checkout_clang_tools_extra=True, |
| checkout_compiler_rt=True, |
| checkout_lld=True, |
| checkout_libcxx=False, |
| |
| # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) |
| stage1_upload_directory=None, |
| |
| # Use a lower compression level to generate the build-cache package faster. |
| # defuault is 6 according to xz documentation |
| xz_compression_factor=6, |
| use_pixz_compression=False, |
| |
| # Triggers |
| trigger_after_stage1=None): |
| return _getClangCMakeBuildFactory( |
| clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, |
| vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, |
| testStage1=testStage1, stage1_config=stage1_config, |
| stage2_config=stage2_config, runTestSuite=runTestSuite, |
| nt_flags=nt_flags, testsuite_flags=testsuite_flags, |
| submitURL=submitURL, testerName=testerName, |
| env=env, extra_cmake_args=extra_cmake_args, |
| checkout_clang_tools_extra=checkout_clang_tools_extra, |
| checkout_compiler_rt=checkout_compiler_rt, |
| checkout_lld=checkout_lld, |
| checkout_libcxx=checkout_libcxx, |
| stage1_upload_directory=stage1_upload_directory, |
| xz_compression_factor=xz_compression_factor, |
| use_pixz_compression=use_pixz_compression, |
| trigger_after_stage1=trigger_after_stage1) |
| |
| def getClangCMakeBuildFactory( |
| is_legacy_mode=False, |
| clean=True, |
| test=True, |
| cmake='cmake', |
| jobs=None, |
| |
| # VS tools environment variable if using MSVC. For example, |
| # %VS120COMNTOOLS% selects the 2013 toolchain. |
| vs=None, |
| vs_target_arch='x86', |
| |
| # Multi-stage compilation |
| useTwoStage=False, |
| testStage1=True, |
| stage1_config='Release', |
| stage2_config='Release', |
| |
| # Test-suite |
| runTestSuite=False, |
| nt_flags=None, |
| testsuite_flags=None, |
| submitURL=None, |
| testerName=None, |
| |
| # Environmental variables for all steps. |
| env=None, |
| extra_cmake_args=None, |
| |
| # Extra repositories |
| checkout_clang_tools_extra=True, |
| checkout_compiler_rt=True, |
| checkout_lld=True, |
| checkout_libcxx=False, |
| checkout_test_suite=False): |
| return _getClangCMakeBuildFactory( |
| is_legacy_mode=is_legacy_mode, |
| clean=clean, test=test, cmake=cmake, jobs=jobs, vs=vs, |
| vs_target_arch=vs_target_arch, useTwoStage=useTwoStage, |
| testStage1=testStage1, stage1_config=stage1_config, |
| stage2_config=stage2_config, runTestSuite=runTestSuite, |
| nt_flags=nt_flags, testsuite_flags=testsuite_flags, |
| submitURL=submitURL, testerName=testerName, |
| env=env, extra_cmake_args=extra_cmake_args, |
| checkout_clang_tools_extra=checkout_clang_tools_extra, |
| checkout_lld=checkout_lld, |
| checkout_compiler_rt=checkout_compiler_rt, |
| checkout_libcxx=checkout_libcxx, |
| checkout_test_suite=checkout_test_suite) |
| |
| def _getClangCMakeBuildFactory( |
| is_legacy_mode=False, |
| clean=True, |
| test=True, |
| cmake='cmake', |
| jobs=None, |
| |
| # VS tools environment variable if using MSVC. For example, |
| # %VS120COMNTOOLS% selects the 2013 toolchain. |
| vs=None, |
| vs_target_arch='x86', |
| |
| # Multi-stage compilation |
| useTwoStage=False, |
| testStage1=True, |
| stage1_config='Release', |
| stage2_config='Release', |
| |
| # Test-suite |
| runTestSuite=False, |
| nt_flags=None, |
| testsuite_flags=None, |
| submitURL=None, |
| testerName=None, |
| |
| # Environmental variables for all steps. |
| env=None, |
| extra_cmake_args=None, |
| |
| # Extra repositories |
| checkout_clang_tools_extra=True, |
| checkout_compiler_rt=True, |
| checkout_lld=True, |
| checkout_libcxx=False, |
| checkout_test_suite=False, |
| |
| # Upload artifacts to Google Cloud Storage (for the llvmbisect tool) |
| stage1_upload_directory=None, |
| |
| # Use a lower compression level to generate the build-cache package faster |
| # default is 6 according to documentation |
| xz_compression_factor=6, |
| use_pixz_compression=False, |
| |
| # Triggers |
| trigger_after_stage1=None): |
| |
| ############# PREPARING |
| if nt_flags is None: |
| nt_flags = [] |
| if testsuite_flags is None: |
| testsuite_flags = [] |
| if env is None: |
| env = {} |
| if extra_cmake_args is None: |
| extra_cmake_args = [] |
| if trigger_after_stage1 is None: |
| trigger_after_stage1 = [] |
| |
| clean_build_requested = lambda step: \ |
| step.build.getProperty( \ |
| "clean", \ |
| default=step.build.getProperty("clean_obj") \ |
| ) or clean |
| |
| # We *must* checkout at least Clang+LLVM |
| depends_on_projects = ['llvm', 'clang'] |
| if checkout_clang_tools_extra: |
| depends_on_projects.append('clang-tools-extra') |
| if checkout_compiler_rt: |
| depends_on_projects.append('compiler-rt') |
| if checkout_lld: |
| depends_on_projects.append('lld') |
| if checkout_libcxx: |
| depends_on_projects.append('libcxx') |
| depends_on_projects.append('libcxxabi') |
| depends_on_projects.append('libunwind') |
| |
| # Some projects are not a part of the monorepo. |
| # So, depending on the legacy mode, we |
| # would have to checkout them differently. |
| if is_legacy_mode and (runTestSuite or checkout_test_suite): |
| depends_on_projects.append('lnt') |
| depends_on_projects.append('test-suite') |
| |
| f = LLVMBuildFactory( |
| is_legacy_mode=is_legacy_mode, |
| depends_on_projects=depends_on_projects, |
| llvm_srcdir='llvm') |
| |
| # If we get the source code form the monorepo, |
| # we need to checkout the latest code for LNT |
| # and the test-suite separately. Le's do this first, |
| # so we wouldn't poison got_revision property. |
| if not is_legacy_mode and (runTestSuite or checkout_test_suite): |
| f.addGetSourcecodeForProject( |
| project='lnt', |
| src_dir='test/lnt', |
| alwaysUseLatest=True) |
| f.addGetSourcecodeForProject( |
| project='test-suite', |
| src_dir='test/test-suite', |
| alwaysUseLatest=True) |
| |
| # Then get the LLVM source code revision this particular build is for. |
| f.addGetSourcecodeSteps() |
| |
| # If jobs not defined, Ninja will choose a suitable value |
| jobs_cmd = [] |
| lit_args = "'-v" |
| if jobs is not None: |
| jobs_cmd = ["-j"+str(jobs)] |
| lit_args += " -j"+str(jobs)+"'" |
| else: |
| lit_args += "'" |
| ninja_cmd = ['ninja'] + jobs_cmd |
| ninja_install_cmd = ['ninja', 'install'] + jobs_cmd |
| ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd |
| |
| # Global configurations |
| stage1_build = 'stage1' |
| stage1_install = 'stage1.install' |
| stage2_build = 'stage2' |
| stage2_install = 'stage2.install' |
| |
| # Set up VS environment, if appropriate. |
| if vs: |
| f.addStep(SetProperty( |
| command=builders_util.getVisualStudioEnvironment(vs, vs_target_arch), |
| extract_fn=builders_util.extractSlaveEnvironment)) |
| assert not env, "Can't have custom builder env vars with VS" |
| env = Property('slave_env') |
| |
| |
| ############# CLEANING |
| f.addStep(ShellCommand(name='clean stage 1', |
| command=['rm','-rf',stage1_build], |
| warnOnFailure=True, |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| description='cleaning stage 1', |
| descriptionDone='clean', |
| workdir='.', |
| doStepIf=clean_build_requested)) |
| |
| |
| ############# STAGE 1 |
| if not f.is_legacy_mode: |
| CmakeCommand.applyRequiredOptions(extra_cmake_args, [ |
| ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)), |
| ]) |
| rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, stage1_build) |
| |
| f.addStep(ShellCommand(name='cmake stage 1', |
| command=[cmake, "-G", "Ninja", rel_src_dir, |
| "-DCMAKE_BUILD_TYPE="+stage1_config, |
| "-DLLVM_ENABLE_ASSERTIONS=True", |
| "-DLLVM_LIT_ARGS="+lit_args, |
| "-DCMAKE_INSTALL_PREFIX=../"+stage1_install] |
| + extra_cmake_args, |
| haltOnFailure=True, |
| description='cmake stage 1', |
| workdir=stage1_build, |
| doStepIf=FileDoesNotExist("build.ninja"), |
| env=env)) |
| |
| f.addStep(WarningCountingShellCommand(name='build stage 1', |
| command=ninja_cmd, |
| haltOnFailure=True, |
| description='ninja all', |
| workdir=stage1_build, |
| env=env)) |
| |
| if test and testStage1: |
| haltOnStage1Check = not useTwoStage and not runTestSuite |
| f.addStep(lit_test_command.LitTestCommand(name='ninja check 1', |
| command=ninja_check_cmd, |
| haltOnFailure=haltOnStage1Check, |
| description=["checking stage 1"], |
| descriptionDone=["stage 1 checked"], |
| workdir=stage1_build, |
| env=env)) |
| |
| if useTwoStage or runTestSuite or stage1_upload_directory: |
| f.addStep(ShellCommand(name='clean stage 1 install', |
| command=['rm','-rf',stage1_install], |
| warnOnFailure=True, |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| description='cleaning stage 1 install', |
| descriptionDone='clean', |
| workdir='.')) |
| f.addStep(ShellCommand(name='install stage 1', |
| command=ninja_install_cmd, |
| description='ninja install', |
| workdir=stage1_build, |
| env=env)) |
| |
| if stage1_upload_directory: |
| addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory, |
| env, gcs_url_property='stage1_package_gcs_url', |
| use_pixz_compression=use_pixz_compression, |
| xz_compression_factor=xz_compression_factor) |
| |
| # Compute the cmake define flag to set the C and C++ compiler to clang. Use |
| # clang-cl if we used MSVC for stage1. |
| if not vs: |
| cc = 'clang' |
| cxx = 'clang++' |
| else: |
| cc = 'clang-cl.exe' |
| cxx = 'clang-cl.exe' |
| |
| |
| ############# STAGE 2 |
| if useTwoStage: |
| # We always cleanly build the stage 2. If the compiler has been |
| # changed on the stage 1, we cannot trust any of the intermediate file |
| # from the old compiler. And if the stage 1 compiler is the same, we |
| # should not build in the first place. |
| f.addStep(ShellCommand(name='clean stage 2', |
| command=['rm','-rf',stage2_build], |
| warnOnFailure=True, |
| description='cleaning stage 2', |
| descriptionDone='clean', |
| workdir='.')) |
| |
| # Set the compiler using the CC and CXX environment variables to work around |
| # backslash string escaping bugs somewhere between buildbot and cmake. The |
| # env.exe helper is required to run the tests, so hopefully it's already on |
| # PATH. |
| rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, stage2_build) |
| cmake_cmd2 = ['env', |
| WithProperties('CC=%(workdir)s/'+stage1_install+'/bin/'+cc), |
| WithProperties('CXX=%(workdir)s/'+stage1_install+'/bin/'+cxx), |
| cmake, "-G", "Ninja", rel_src_dir, |
| "-DCMAKE_BUILD_TYPE="+stage2_config, |
| "-DLLVM_ENABLE_ASSERTIONS=True", |
| "-DLLVM_LIT_ARGS="+lit_args, |
| "-DCMAKE_INSTALL_PREFIX=../"+stage2_install] + extra_cmake_args |
| |
| f.addStep(ShellCommand(name='cmake stage 2', |
| command=cmake_cmd2, |
| haltOnFailure=True, |
| description='cmake stage 2', |
| workdir=stage2_build, |
| env=env)) |
| |
| f.addStep(WarningCountingShellCommand(name='build stage 2', |
| command=ninja_cmd, |
| haltOnFailure=True, |
| description='ninja all', |
| workdir=stage2_build, |
| env=env)) |
| |
| if test: |
| f.addStep(lit_test_command.LitTestCommand(name='ninja check 2', |
| command=ninja_check_cmd, |
| haltOnFailure=not runTestSuite, |
| description=["checking stage 2"], |
| descriptionDone=["stage 2 checked"], |
| workdir=stage2_build, |
| env=env)) |
| |
| ############# TEST SUITE |
| ## Test-Suite (stage 2 if built, stage 1 otherwise) |
| if runTestSuite: |
| compiler_path = stage1_install |
| if useTwoStage: |
| compiler_path=stage2_install |
| f.addStep(ShellCommand(name='clean stage 2 install', |
| command=['rm','-rf',stage2_install], |
| warnOnFailure=True, |
| description='cleaning stage 2 install', |
| descriptionDone='clean', |
| workdir='.')) |
| f.addStep(ShellCommand(name='install stage 2', |
| command=ninja_install_cmd, |
| description='ninja install 2', |
| workdir=stage2_build, |
| env=env)) |
| |
| # Get generated python, lnt |
| python = WithProperties('%(workdir)s/test/sandbox/bin/python') |
| lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt') |
| lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py') |
| |
| # Paths |
| sandbox = WithProperties('%(workdir)s/test/sandbox') |
| test_suite_dir = WithProperties('%(workdir)s/test/test-suite') |
| |
| # Get latest built Clang (stage1 or stage2) |
| cc = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cc) |
| cxx = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cxx) |
| |
| # LNT Command line (don't pass -jN. Users need to pass both --threads |
| # and --build-threads in nt_flags/test_suite_flags to get the same effect) |
| use_runtest_testsuite = len(nt_flags) == 0 |
| if not use_runtest_testsuite: |
| test_suite_cmd = [python, lnt, 'runtest', 'nt', |
| '--no-timestamp', |
| '--sandbox', sandbox, |
| '--test-suite', test_suite_dir, |
| '--cc', cc, |
| '--cxx', cxx] |
| # Append any option provided by the user |
| test_suite_cmd.extend(nt_flags) |
| else: |
| lit = WithProperties('%(workdir)s/'+stage1_build+'/bin/llvm-lit') |
| test_suite_cmd = [python, lnt, 'runtest', 'test-suite', |
| '--no-timestamp', |
| '--sandbox', sandbox, |
| '--test-suite', test_suite_dir, |
| '--cc', cc, |
| '--cxx', cxx, |
| '--use-lit', lit] |
| # Append any option provided by the user |
| test_suite_cmd.extend(testsuite_flags) |
| |
| # Only submit if a URL has been specified |
| if submitURL is not None: |
| if not isinstance(submitURL, list): |
| submitURL = [submitURL] |
| for url in submitURL: |
| test_suite_cmd.extend(['--submit', url]) |
| # lnt runtest test-suite doesn't understand --no-machdep-info: |
| if testerName and not use_runtest_testsuite: |
| test_suite_cmd.extend(['--no-machdep-info', testerName]) |
| # CC and CXX are needed as env for build-tools |
| test_suite_env = copy.deepcopy(env) |
| test_suite_env['CC'] = cc |
| test_suite_env['CXX'] = cxx |
| |
| # Steps to prepare, build and run LNT |
| f.addStep(ShellCommand(name='clean sandbox', |
| command=['rm', '-rf', 'sandbox'], |
| haltOnFailure=True, |
| description='removing sandbox directory', |
| workdir='test', |
| env=env)) |
| f.addStep(ShellCommand(name='recreate sandbox', |
| command=['virtualenv', 'sandbox'], |
| haltOnFailure=True, |
| description='recreating sandbox', |
| workdir='test', |
| env=env)) |
| f.addStep(ShellCommand(name='setup lit', |
| command=[python, lnt_setup, 'develop'], |
| haltOnFailure=True, |
| description='setting up LNT in sandbox', |
| workdir='test/sandbox', |
| env=env)) |
| f.addStep(commands.LitTestCommand.LitTestCommand( |
| name='test-suite', |
| command=test_suite_cmd, |
| haltOnFailure=True, |
| description=['running the test suite'], |
| workdir='test/sandbox', |
| logfiles={'configure.log' : 'build/configure.log', |
| 'build-tools.log' : 'build/build-tools.log', |
| 'test.log' : 'build/test.log', |
| 'report.json' : 'build/report.json'}, |
| env=test_suite_env)) |
| |
| return f |
| |
| # FIXME: Deprecated. |
| def addClangGCCTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install", |
| languages = ('gcc', 'g++', 'objc', 'obj-c++')): |
| make_vars = [WithProperties( |
| 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), |
| WithProperties( |
| 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] |
| f.addStep(SVN(name='svn-clang-gcc-tests', mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/clang-tests/', |
| defaultBranch='trunk', workdir='clang-tests')) |
| gcc_dg_ignores = ignores.get('gcc-4_2-testsuite', {}) |
| for lang in languages: |
| f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( |
| name='test-gcc-4_2-testsuite-%s' % lang, |
| command=["make", "-k", "check-%s" % lang] + make_vars, |
| description="gcc-4_2-testsuite (%s)" % lang, |
| workdir='clang-tests/gcc-4_2-testsuite', |
| logfiles={ 'dg.sum' : 'obj/%s/%s.sum' % (lang, lang), |
| '%s.log' % lang : 'obj/%s/%s.log' % (lang, lang)}, |
| ignore=gcc_dg_ignores.get(lang, []))) |
| |
| # FIXME: Deprecated. |
| def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install"): |
| make_vars = [WithProperties( |
| 'CC_UNDER_TEST=%s/bin/clang' % install_prefix), |
| WithProperties( |
| 'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)] |
| f.addStep(SVN(name='svn-clang-gdb-tests', mode='update', |
| baseURL='http://llvm.org/svn/llvm-project/clang-tests/', |
| defaultBranch='trunk', workdir='clang-tests')) |
| f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand( |
| name='test-gdb-1472-testsuite', |
| command=["make", "-k", "check"] + make_vars, |
| description="gdb-1472-testsuite", |
| workdir='clang-tests/gdb-1472-testsuite', |
| logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum', |
| 'gdb.log' : 'obj/gdb.log' })) |
| |
| # FIXME: Deprecated. |
| def addModernClangGDBTests(f, jobs, install_prefix): |
| make_vars = [WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' ' |
| 'CXX_FOR_TARGET=\'{0}/bin/clang++\' ' |
| 'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\'' |
| .format(install_prefix))] |
| f.addStep(SVN(name='svn-clang-modern-gdb-tests', mode='update', |
| svnurl='http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5', |
| workdir='clang-tests/src')) |
| f.addStep(Configure(command='../src/configure', |
| workdir='clang-tests/build/')) |
| f.addStep(WarningCountingShellCommand(name='gdb-75-build', |
| command=['make', WithProperties('-j%s' % jobs)], |
| haltOnFailure=True, |
| workdir='clang-tests/build')) |
| f.addStep(commands.DejaGNUCommand.DejaGNUCommand( |
| name='gdb-75-check', |
| command=['make', '-k', WithProperties('-j%s' % jobs), 'check'] + make_vars, |
| workdir='clang-tests/build', |
| logfiles={'dg.sum':'gdb/testsuite/gdb.sum', |
| 'gdb.log':'gdb/testsuite/gdb.log'})) |
| |
| |
| |
| # FIXME: Deprecated. |
| addClangTests = addClangGCCTests |
| |
| def getClangTestsIgnoresFromPath(path, key): |
| def readList(path): |
| if not os.path.exists(path): |
| return [] |
| |
| f = open(path) |
| lines = [ln.strip() for ln in f] |
| f.close() |
| return lines |
| |
| ignores = {} |
| |
| gcc_dg_ignores = {} |
| for lang in ('gcc', 'g++', 'objc', 'obj-c++'): |
| lang_path = os.path.join(path, 'gcc-4_2-testsuite', 'expected_results', |
| key, lang) |
| gcc_dg_ignores[lang] = ( |
| readList(os.path.join(lang_path, 'FAIL.txt')) + |
| readList(os.path.join(lang_path, 'UNRESOLVED.txt')) + |
| readList(os.path.join(lang_path, 'XPASS.txt'))) |
| ignores['gcc-4_2-testsuite' ] = gcc_dg_ignores |
| |
| ignores_path = os.path.join(path, 'gdb-1472-testsuite', 'expected_results', |
| key) |
| gdb_dg_ignores = ( |
| readList(os.path.join(ignores_path, 'FAIL.txt')) + |
| readList(os.path.join(ignores_path, 'UNRESOLVED.txt')) + |
| readList(os.path.join(ignores_path, 'XPASS.txt'))) |
| ignores['gdb-1472-testsuite' ] = gdb_dg_ignores |
| |
| return ignores |
