| from buildbot.steps.shell import ShellCommand |
| from buildbot.process.results import FAILURE |
| from buildbot.plugins import steps, util |
| |
| from zorg.buildbot.commands.CmakeCommand import CmakeCommand |
| from zorg.buildbot.commands.NinjaCommand import NinjaCommand |
| from zorg.buildbot.conditions.FileConditions import FileDoesNotExist |
| from zorg.buildbot.process.factory import LLVMBuildFactory |
| from zorg.buildbot.builders import UnifiedTreeBuilder |
| |
| def _addSteps4SystemCompiler( |
| f, |
| stage_idx = 0, |
| clean = True, |
| jobs = None, |
| extra_configure_args = None, |
| env = None): |
| |
| # Index is zero-based, so we want to use a human friendly number instead. |
| stage_num = stage_idx + 1 |
| |
| # Directories to use on this stage. |
| obj_dir = f.stage_objdirs[stage_idx] |
| src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) |
| install_dir = LLVMBuildFactory.pathRelativeTo(f.stage_installdirs[stage_idx], obj_dir) |
| |
| # This stage could use incremental build. |
| # Clean stage1, only if requested. |
| f.addStep(steps.RemoveDirectory(name=f'clean-{obj_dir}-dir', |
| dir=obj_dir, |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| doStepIf=clean |
| )) |
| f.addStep(steps.RemoveDirectory(name=f'clean-{f.stage_installdirs[stage_idx]}-dir', |
| dir=f.stage_installdirs[stage_idx], |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| doStepIf=clean |
| )) |
| |
| # Reconcile the cmake options for this stage. |
| |
| # Make a local copy of the configure args, as we are going to modify that. |
| if extra_configure_args: |
| cmake_args = extra_configure_args[:] |
| else: |
| cmake_args = list() |
| |
| # Set proper defaults. |
| CmakeCommand.applyDefaultOptions(cmake_args, [ |
| ('-DCMAKE_BUILD_TYPE=', 'Release'), |
| ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), |
| ('-DLLVM_BUILD_TESTS=', 'ON'), |
| ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), |
| ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), |
| # Do not expect warning free build by the system toolchain. |
| ('-DLLVM_ENABLE_WERROR=', 'OFF'), |
| ]) |
| |
| # Some options are required for this stage no matter what. |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-G', 'Ninja'), |
| ('-DCMAKE_INSTALL_PREFIX=', install_dir), |
| ]) |
| |
| if f.enable_projects: |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), |
| ]) |
| if f.enable_runtimes: |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), |
| ]) |
| |
| # Note: On this stage we do not care of warnings, as we build with |
| # a system toolchain and cannot control the environment. |
| # Warnings are likely, and we ignore them. |
| |
| # Create configuration files with cmake |
| f.addStep(CmakeCommand(name=f"cmake-configure-stage{stage_num}", |
| description=[f"stage{stage_num} cmake configure"], |
| haltOnFailure=True, |
| flunkOnWarnings=False, |
| options=cmake_args, |
| path=src_dir, |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Build clang by the system compiler |
| f.addStep(NinjaCommand(name=f"build-stage{stage_num}-compiler", |
| jobs=jobs, |
| haltOnFailure=True, |
| flunkOnWarnings=False, |
| description=[f"build stage{stage_num} compiler"], |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Test stage1 compiler |
| f.addStep(NinjaCommand(name=f"test-stage{stage_num}-compiler", |
| targets=["check-all"], # or "check-llvm", "check-clang" |
| jobs=jobs, |
| haltOnFailure=True, |
| flunkOnWarnings=False, |
| description=[f"test stage{stage_num} compiler"], |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Install stage1 compiler |
| f.addStep(NinjaCommand(name=f"install-stage{stage_num}-compiler", |
| targets=["install"], |
| jobs=jobs, |
| haltOnFailure=True, |
| description=[f"install stage{stage_num} compiler"], |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| |
| def _addSteps4StagedCompiler( |
| f, |
| stage_idx = 1, |
| use_stage_idx = -1, |
| jobs = None, |
| extra_configure_args = None, |
| env = None): |
| |
| if use_stage_idx < 0: |
| use_stage_idx = stage_idx - 1 |
| |
| # Index is zero-based, so we want to use a human friendly number instead. |
| stage_num = stage_idx + 1 |
| |
| # Directories to use on this stage. |
| obj_dir = f.stage_objdirs[stage_idx] |
| src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir) |
| install_dir = LLVMBuildFactory.pathRelativeTo(f.stage_installdirs[stage_idx], obj_dir) |
| staged_install = f.stage_installdirs[use_stage_idx] |
| |
| # Always do a clean build for the staged compiler. |
| f.addStep(steps.RemoveDirectory(name=f'clean-{obj_dir}-dir', |
| dir=obj_dir, |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| )) |
| |
| f.addStep(steps.RemoveDirectory(name=f'clean-{f.stage_installdirs[stage_idx]}-dir', |
| dir=f.stage_installdirs[stage_idx], |
| haltOnFailure=False, |
| flunkOnFailure=False, |
| )) |
| |
| # Reconcile the cmake options for this stage. |
| |
| # Make a local copy of the configure args, as we are going to modify that. |
| if extra_configure_args: |
| cmake_args = extra_configure_args[:] |
| else: |
| cmake_args = list() |
| |
| # Set proper defaults. |
| CmakeCommand.applyDefaultOptions(cmake_args, [ |
| ('-DCMAKE_BUILD_TYPE=', 'Release'), |
| ('-DCLANG_BUILD_EXAMPLES=', 'OFF'), |
| ('-DLLVM_BUILD_TESTS=', 'ON'), |
| ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'), |
| ('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'), |
| ]) |
| |
| # Some options are required for this stage no matter what. |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-G', 'Ninja'), |
| ('-DCMAKE_INSTALL_PREFIX=', install_dir), |
| ]) |
| |
| cmake_args.append( |
| util.Interpolate( |
| f"-DCMAKE_CXX_COMPILER=%(prop:builddir)s/{staged_install}/bin/clang++" |
| )) |
| cmake_args.append( |
| util.Interpolate( |
| f"-DCMAKE_C_COMPILER=%(prop:builddir)s/{staged_install}/bin/clang" |
| )) |
| |
| if f.enable_projects: |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.enable_projects)), |
| ]) |
| if f.enable_runtimes: |
| CmakeCommand.applyRequiredOptions(cmake_args, [ |
| ('-DLLVM_ENABLE_RUNTIMES=', ";".join(f.enable_runtimes)), |
| ]) |
| |
| # Create configuration files with cmake |
| f.addStep(CmakeCommand(name=f"cmake-configure-stage{stage_num}", |
| description=[f"stage{stage_num} cmake configure"], |
| haltOnFailure=True, |
| options=cmake_args, |
| path=src_dir, |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Build clang by the staged compiler |
| f.addStep(NinjaCommand(name=f"build-stage{stage_num}-compiler", |
| jobs=jobs, |
| haltOnFailure=True, |
| description=[f"build stage{stage_num} compiler"], |
| timeout=10800, # LTO could take time. |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Test just built compiler |
| f.addStep(NinjaCommand(name=f"test-stage{stage_num}-compiler", |
| targets=["check-all"], |
| jobs=jobs, |
| haltOnFailure=True, |
| description=[f"test stage{stage_num} compiler"], |
| timeout=10800, # LTO could take time. |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| # Install just built compiler |
| f.addStep(NinjaCommand(name=f"install-stage{stage_num}-compiler", |
| targets=["install"], |
| jobs=jobs, |
| haltOnFailure=True, |
| description=[f"install stage{stage_num} compiler"], |
| timeout=10800, # LTO could take time. |
| env=env, |
| workdir=obj_dir, |
| )) |
| |
| |
| def getClangWithLTOBuildFactory( |
| depends_on_projects = None, |
| clean = False, |
| jobs = None, |
| extra_configure_args = None, |
| extra_configure_args_lto_stage = None, |
| compare_last_2_stages = True, |
| lto = None, # The string gets passed to -flto flag as is. Like -flto=thin. |
| env = None, |
| **kwargs): |
| |
| # Set defaults |
| if depends_on_projects: |
| depends_on_projects = list(depends_on_projects) |
| else: |
| # By default we link with LLD. |
| depends_on_projects = ['llvm', 'clang', 'lld'] |
| |
| if lto is None: |
| lto = 'ON' |
| |
| if extra_configure_args is None: |
| extra_configure_args = [] |
| else: |
| extra_configure_args = list(extra_configure_args) |
| |
| if extra_configure_args_lto_stage is None: |
| extra_configure_args_lto_stage = [] |
| else: |
| extra_configure_args_lto_stage = list(extra_configure_args_lto_stage) |
| |
| # Make sure CMAKE_INSTALL_PREFIX and -G are not specified |
| # in the extra_configure_args. We set them internally as needed. |
| # TODO: assert extra_configure_args. |
| install_prefix_specified = (any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args) or |
| any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args_lto_stage)) |
| assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build." |
| |
| # Prepare environmental variables. Set here all env we want everywhere. |
| merged_env = { |
| 'TERM' : 'dumb' # Be cautious and disable color output from all tools. |
| } |
| if env is not None: |
| # Overwrite pre-set items with the given ones, so user can set anything. |
| merged_env.update(env) |
| |
| f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps( |
| depends_on_projects=depends_on_projects, |
| stage_objdirs=[ |
| "build/stage1", |
| "build/stage2", |
| "build/stage3", |
| "build/stage4", |
| ], |
| stage_installdirs=[ |
| "install/stage1", |
| "install/stage2", |
| "install/stage3", |
| "install/stage4", |
| ], |
| staged_compiler_idx = 1, |
| **kwargs) |
| |
| cleanBuildRequested = lambda step: clean or step.build.getProperty("clean") or step.build.getProperty("clean_obj") |
| |
| # Get the source code. |
| f.addGetSourcecodeSteps() |
| |
| # Build with the system compiler first |
| _addSteps4SystemCompiler(f, |
| stage_idx=0, |
| clean=cleanBuildRequested, |
| jobs=jobs, |
| extra_configure_args=extra_configure_args, |
| env=merged_env) |
| |
| # Then build the compiler we would use for the bootstrap. |
| _addSteps4StagedCompiler(f, |
| stage_idx=1, |
| jobs=jobs, |
| extra_configure_args=extra_configure_args, |
| env=merged_env) |
| |
| # Build all the remaining stages with exactly the same configuration. |
| |
| CmakeCommand.applyRequiredOptions(extra_configure_args, [ |
| ('-DLLVM_ENABLE_LTO=', lto), |
| # NOTE: Diagnostic messages could contain __FILE__, thus final executable would depend on build path. |
| # We do not want that since we expect executables built from the same source code in different |
| # build directories be exactly the same. |
| ('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'), |
| ]) |
| |
| # If we build LLD, we would link with LLD. |
| # Otherwise we link with the system linker. |
| if 'lld' in depends_on_projects: |
| CmakeCommand.applyRequiredOptions(extra_configure_args, [ |
| ('-DLLVM_ENABLE_LLD=', 'ON'), |
| ]) |
| |
| # The rest are test stages, which depend on the staged compiler we are ultimately after. |
| s = f.staged_compiler_idx + 1 |
| staged_install = f.stage_installdirs[f.staged_compiler_idx] |
| for i in range(s, len(f.stage_objdirs[s:]) + s): |
| configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:] |
| |
| configure_args.append( |
| util.Interpolate( |
| f"-DCMAKE_AR=%(prop:builddir)s/{staged_install}/bin/llvm-ar" |
| )) |
| configure_args.append( |
| util.Interpolate( |
| f"-DCMAKE_RANLIB=%(prop:builddir)s/{staged_install}/bin/llvm-ranlib" |
| )) |
| |
| _addSteps4StagedCompiler(f, |
| stage_idx=i, |
| use_stage_idx=f.staged_compiler_idx, |
| jobs=jobs, |
| extra_configure_args=configure_args, |
| env=merged_env) |
| |
| if compare_last_2_stages: |
| # Compare the compilers built on the last 2 stages if requested. |
| diff_command = [ |
| "diff", |
| "-q", |
| f.stage_installdirs[-2] + "/bin/clang", |
| f.stage_installdirs[-1] + "/bin/clang", |
| ] |
| f.addStep( |
| ShellCommand( |
| name="compare-compilers", |
| description=[ |
| "compare", |
| f"stage{(len(f.stage_installdirs)-1)}", |
| "and", |
| f"stage{len(f.stage_installdirs)}", |
| "compilers", |
| ], |
| haltOnFailure=False, |
| command=util.Interpolate(" ".join(diff_command)), |
| workdir=".", |
| env=merged_env |
| ) |
| ) |
| |
| # Only if the compare-compilers step has failed. |
| def _prevStepFailed(step): |
| steps = step.build.executedSteps |
| prev_step = steps[-2] |
| return (prev_step.results == FAILURE) |
| |
| dir1 = f.stage_objdirs[-2] |
| dir2 = f.stage_objdirs[-1] |
| inc_pattern = "-type f -not -name *.inc -printf '%f\n'" |
| find_cmd = f"find {dir1} {dir2}" |
| diff_cmd = f"diff -ru {dir1} {dir2} -x '*.tmp*' -X -" |
| |
| # Note: Use a string here as we want the command executed by a shell. |
| diff_tablegen_inc_files_command = f"{find_cmd} {inc_pattern} | {diff_cmd}" |
| |
| f.addStep( |
| ShellCommand( |
| name="compare-tablegen-inc-files", |
| description=[ |
| "compare", |
| f"stage{(len(f.stage_installdirs)-1)}", |
| "and", |
| f"stage{len(f.stage_installdirs)}", |
| "Tablegen inc files", |
| ], |
| command=diff_tablegen_inc_files_command, |
| workdir=".", |
| env=merged_env, |
| doStepIf=_prevStepFailed, |
| ) |
| ) |
| |
| return f |