zorg/buildbot/builders/XToolchainBuilder.py

from buildbot.steps.shell import ShellCommand, SetPropertyFromCommand
from buildbot.plugins import util

from zorg.buildbot.commands.CmakeCommand   import CmakeCommand
from zorg.buildbot.commands.NinjaCommand   import NinjaCommand
from zorg.buildbot.commands.LitTestCommand import LitTestCommand
from zorg.buildbot.process.factory         import LLVMBuildFactory

import zorg.buildbot.builders.Util as builders_util

def getCmakeWithMSVCBuildFactory(
        clean = True,                # False for incremental builds.
        depends_on_projects  = None, # List of projects to listen.
        cmake_cache = None,          # Path to a cmake cache file.
        extra_configure_args = None, # Extra CMake args if any.
        llvm_srcdir = None,          # Source code root directory.
        obj_dir = None,              # Build tree root directory.
        install_dir = None,          # Directory to install the results to.
        checks = None,               # List of checks to test the build.
        checks_on_target = None,     # [(<name>,[<command tokens>])] array of
                                     # check name and command to run on target.
        jobs = None,                 # Restrict a degree of parallelism.
        env  = None,                 # Environmental variables for all steps.
        # VS tools environment variable if using MSVC.
        # For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
        # the VS 2015 toolchain, or empty string if environment is already set.
        vs=None,
        **kwargs):

    if vs is None:
        # We autodetect Visual Studio, unless otherwise is requested.
        vs = "autodetect"

    if install_dir is None:
        install_dir = 'install'

    # Prepare environmental variables. Set here all env we want for all steps.
    merged_env = {
        'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
        }
    if env is not None:
        # Overwrite pre-set items with the given ones, so user can set anything.
        merged_env.update(env)

    # Make a local copy of the configure args, as we are going to modify that.
    if extra_configure_args:
        cmake_args = extra_configure_args[:]
    else:
        cmake_args = list()

    if depends_on_projects is None:
        depends_on_projects = [
            'llvm',
            'compiler-rt',
            'clang',
            'clang-tools-extra',
            'libunwind',
            'libcxx',
            'libcxxabi',
            'lld',
        ]

    if checks is None:
        # Check only host-side tools. Target-side tests should run on a target.
        checks=[
            "check-llvm",
            "check-clang",
            "check-lld",
        ]

    source_remove_requested = lambda step: step.build.getProperty("clean")
    clean_build_requested = lambda step: \
        clean or \
        step.build.getProperty("clean") or \
        step.build.getProperty("clean_obj")

    f = LLVMBuildFactory(
            depends_on_projects=depends_on_projects,
            llvm_srcdir=llvm_srcdir,
            obj_dir=obj_dir,
            install_dir=install_dir,
            cleanBuildRequested=clean_build_requested,
            **kwargs) # Pass through all the extra arguments.

    # Remove the source code tree if requested.
    # NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
    # as somthing keeps a lock. We use rm command instead realying on a shell
    # to support that.
    f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
                           command=['rm','-rf', f.monorepo_dir],
                           warnOnFailure=True,
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           description='Remove the source code',
                           workdir='.',
                           env=merged_env,
                           doStepIf=source_remove_requested))

    # Get the source code.
    f.addGetSourcecodeSteps(**kwargs)

    # Remove the build directory if requested.
    f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir,
                           command=['rm','-rf', f.obj_dir],
                           warnOnFailure=True,
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           description='Remove build directory',
                           workdir='.',
                           env=merged_env,
                           doStepIf=clean_build_requested))

    if vs:
        # Configure MSVC environment if requested.
        f.addStep(SetPropertyFromCommand(
            command=builders_util.getVisualStudioEnvironment(vs, None),
            extract_fn=builders_util.extractVSEnvironment,
            env=merged_env))
        merged_env = util.Property('vs_env')

    CmakeCommand.applyDefaultOptions(cmake_args, [
        ('-G', 'Ninja'),
        ('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
        ('-DCMAKE_BUILD_TYPE=', 'Release'),
        ('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
        ('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
        ('-DLLVM_LIT_ARGS=', '-v -vv'),
        ])

    if install_dir:
        install_dir_rel = LLVMBuildFactory.pathRelativeTo(
                              install_dir,
                              f.obj_dir)
        CmakeCommand.applyRequiredOptions(cmake_args, [
            ('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
            ])

    # Remove the build directory if requested.
    f.addStep(ShellCommand(name='clean-%s-dir' % install_dir,
                           command=['rm','-rf', install_dir],
                           warnOnFailure=True,
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           description='Remove install directory',
                           workdir='.',
                           env=merged_env,
                           doStepIf=clean_build_requested))

    src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)

    # Add given cmake cache at the very end.
    if cmake_cache:
        cmake_args.append('-C%s' % cmake_cache)

    f.addStep(CmakeCommand(name="cmake-configure",
                           haltOnFailure=True,
                           description=["Cmake", "configure"],
                           options=cmake_args,
                           path=src_dir_rel,
                           workdir=f.obj_dir,
                           env=merged_env,
                           **kwargs # Pass through all the extra arguments.
                           ))

    f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir,
                           haltOnFailure=True,
                           description=["Build", f.monorepo_dir],
                           workdir=f.obj_dir,
                           env=merged_env,
                           **kwargs # Pass through all the extra arguments.
                           ))

    # Test the components if requested one check at a time.
    for check in checks:
        f.addStep(
            LitTestCommand(
                haltOnFailure=False, # We want to test as much as we could.
                name=f'test-{check}',
                command=["ninja", util.Interpolate(check)],
                description=[
                    "Testing", "just", "built", "components", "for", check],
                descriptionDone=[
                    "Test", "just", "built", "components", "for", check,
                     "completed"],
                env=merged_env,
                **kwargs # Pass through all the extra arguments.
                ))

    # Run commands on a target if requested.
    if checks_on_target:
        for check, cmd in checks_on_target:
            f.addStep(
                LitTestCommand(
                    haltOnFailure=False, # We want to test as much as we could.
                    name=f'test-{check}',
                    command=cmd,
                    description=[
                        "Testing", "just", "built", "components", "for", check],
                    descriptionDone=[
                        "Test", "just", "built", "components", "for", check,
                        "completed"],
                    env=merged_env,
                    **kwargs # Pass through all the extra arguments.
                    ))

    # Install just built components
    if install_dir:
        f.addStep(NinjaCommand(name="install-all",
                               haltOnFailure=True,
                               targets=["install"],
                               description=[
                                   "Install", "just", "built", "components"],
                               workdir=f.obj_dir,
                               env=merged_env,
                               **kwargs # Pass through all the extra arguments.
                               ))

    return f