| #!/usr/bin/env python |
| |
| import argparse |
| import sys |
| import os |
| |
| from subprocess import call |
| |
| SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) |
| PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") |
| DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR, |
| os.path.pardir, |
| os.path.pardir, |
| os.path.pardir)) |
| |
| |
| def add(parser, args): |
| import SATestAdd |
| from ProjectMap import ProjectInfo |
| |
| if args.source == "git" and (args.origin == "" or args.commit == ""): |
| parser.error( |
| "Please provide both --origin and --commit if source is 'git'") |
| |
| if args.source != "git" and (args.origin != "" or args.commit != ""): |
| parser.error("Options --origin and --commit don't make sense when " |
| "source is not 'git'") |
| |
| project = ProjectInfo(args.name[0], args.mode, args.source, args.origin, |
| args.commit) |
| |
| SATestAdd.add_new_project(project) |
| |
| |
| def build(parser, args): |
| import SATestBuild |
| |
| SATestBuild.VERBOSE = args.verbose |
| |
| projects = get_projects(parser, args) |
| tester = SATestBuild.RegressionTester(args.jobs, |
| projects, |
| args.override_compiler, |
| args.extra_analyzer_config, |
| args.extra_checkers, |
| args.regenerate, |
| args.strictness) |
| tests_passed = tester.test_all() |
| |
| if not tests_passed: |
| sys.stderr.write("ERROR: Tests failed.\n") |
| sys.exit(42) |
| |
| |
| def compare(parser, args): |
| import CmpRuns |
| |
| choices = [CmpRuns.HistogramType.RELATIVE.value, |
| CmpRuns.HistogramType.LOG_RELATIVE.value, |
| CmpRuns.HistogramType.ABSOLUTE.value] |
| |
| if args.histogram is not None and args.histogram not in choices: |
| parser.error("Incorrect histogram type, available choices are {}" |
| .format(choices)) |
| |
| dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) |
| dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) |
| |
| CmpRuns.dump_scan_build_results_diff(dir_old, dir_new, |
| show_stats=args.show_stats, |
| stats_only=args.stats_only, |
| histogram=args.histogram, |
| verbose_log=args.verbose_log) |
| |
| |
| def update(parser, args): |
| import SATestUpdateDiffs |
| from ProjectMap import ProjectMap |
| |
| project_map = ProjectMap() |
| for project in project_map.projects: |
| SATestUpdateDiffs.update_reference_results(project, args.git) |
| |
| |
| def benchmark(parser, args): |
| from SATestBenchmark import Benchmark |
| |
| projects = get_projects(parser, args) |
| benchmark = Benchmark(projects, args.iterations, args.output) |
| benchmark.run() |
| |
| |
| def benchmark_compare(parser, args): |
| import SATestBenchmark |
| SATestBenchmark.compare(args.old, args.new, args.output) |
| |
| |
| def get_projects(parser, args): |
| from ProjectMap import ProjectMap, Size |
| |
| project_map = ProjectMap() |
| projects = project_map.projects |
| |
| def filter_projects(projects, predicate, force=False): |
| return [project.with_fields(enabled=(force or project.enabled) and |
| predicate(project)) |
| for project in projects] |
| |
| if args.projects: |
| projects_arg = args.projects.split(",") |
| available_projects = [project.name |
| for project in projects] |
| |
| # validate that given projects are present in the project map file |
| for manual_project in projects_arg: |
| if manual_project not in available_projects: |
| parser.error("Project '{project}' is not found in " |
| "the project map file. Available projects are " |
| "{all}.".format(project=manual_project, |
| all=available_projects)) |
| |
| projects = filter_projects(projects, lambda project: |
| project.name in projects_arg, |
| force=True) |
| |
| try: |
| max_size = Size.from_str(args.max_size) |
| except ValueError as e: |
| parser.error("{}".format(e)) |
| |
| projects = filter_projects(projects, lambda project: |
| project.size <= max_size) |
| |
| return projects |
| |
| |
| def docker(parser, args): |
| if len(args.rest) > 0: |
| if args.rest[0] != "--": |
| parser.error("REST arguments should start with '--'") |
| args.rest = args.rest[1:] |
| |
| if args.build_image: |
| docker_build_image() |
| elif args.shell: |
| docker_shell(args) |
| else: |
| sys.exit(docker_run(args, ' '.join(args.rest))) |
| |
| |
| def docker_build_image(): |
| sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), |
| shell=True)) |
| |
| |
| def docker_shell(args): |
| try: |
| # First we need to start the docker container in a waiting mode, |
| # so it doesn't do anything, but most importantly keeps working |
| # while the shell session is in progress. |
| docker_run(args, "--wait", "--detach") |
| # Since the docker container is running, we can actually connect to it |
| call("docker exec -it satest bash", shell=True) |
| |
| except KeyboardInterrupt: |
| pass |
| |
| finally: |
| docker_cleanup() |
| |
| |
| def docker_run(args, command, docker_args=""): |
| try: |
| return call("docker run --rm --name satest " |
| "-v {llvm}:/llvm-project " |
| "-v {build}:/build " |
| "-v {clang}:/analyzer " |
| "-v {scripts}:/scripts " |
| "-v {projects}:/projects " |
| "{docker_args} " |
| "satest-image:latest {command}" |
| .format(llvm=args.llvm_project_dir, |
| build=args.build_dir, |
| clang=args.clang_dir, |
| scripts=SCRIPTS_DIR, |
| projects=PROJECTS_DIR, |
| docker_args=docker_args, |
| command=command), |
| shell=True) |
| |
| except KeyboardInterrupt: |
| docker_cleanup() |
| |
| |
| def docker_cleanup(): |
| print("Please wait for docker to clean up") |
| call("docker stop satest", shell=True) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| subparsers = parser.add_subparsers() |
| |
| # add subcommand |
| add_parser = subparsers.add_parser( |
| "add", |
| help="Add a new project for the analyzer testing.") |
| # TODO: Add an option not to build. |
| # TODO: Set the path to the Repository directory. |
| add_parser.add_argument("name", nargs=1, help="Name of the new project") |
| add_parser.add_argument("--mode", action="store", default=1, type=int, |
| choices=[0, 1, 2], |
| help="Build mode: 0 for single file project, " |
| "1 for scan_build, " |
| "2 for single file c++11 project") |
| add_parser.add_argument("--source", action="store", default="script", |
| choices=["script", "git", "zip"], |
| help="Source type of the new project: " |
| "'git' for getting from git " |
| "(please provide --origin and --commit), " |
| "'zip' for unpacking source from a zip file, " |
| "'script' for downloading source by running " |
| "a custom script") |
| add_parser.add_argument("--origin", action="store", default="", |
| help="Origin link for a git repository") |
| add_parser.add_argument("--commit", action="store", default="", |
| help="Git hash for a commit to checkout") |
| add_parser.set_defaults(func=add) |
| |
| # build subcommand |
| build_parser = subparsers.add_parser( |
| "build", |
| help="Build projects from the project map and compare results with " |
| "the reference.") |
| build_parser.add_argument("--strictness", dest="strictness", |
| type=int, default=0, |
| help="0 to fail on runtime errors, 1 to fail " |
| "when the number of found bugs are different " |
| "from the reference, 2 to fail on any " |
| "difference from the reference. Default is 0.") |
| build_parser.add_argument("-r", dest="regenerate", action="store_true", |
| default=False, |
| help="Regenerate reference output.") |
| build_parser.add_argument("--override-compiler", action="store_true", |
| default=False, help="Call scan-build with " |
| "--override-compiler option.") |
| build_parser.add_argument("-j", "--jobs", dest="jobs", |
| type=int, default=0, |
| help="Number of projects to test concurrently") |
| build_parser.add_argument("--extra-analyzer-config", |
| dest="extra_analyzer_config", type=str, |
| default="", |
| help="Arguments passed to to -analyzer-config") |
| build_parser.add_argument("--extra-checkers", |
| dest="extra_checkers", type=str, |
| default="", |
| help="Extra checkers to enable") |
| build_parser.add_argument("--projects", action="store", default="", |
| help="Comma-separated list of projects to test") |
| build_parser.add_argument("--max-size", action="store", default=None, |
| help="Maximum size for the projects to test") |
| build_parser.add_argument("-v", "--verbose", action="count", default=0) |
| build_parser.set_defaults(func=build) |
| |
| # compare subcommand |
| cmp_parser = subparsers.add_parser( |
| "compare", |
| help="Comparing two static analyzer runs in terms of " |
| "reported warnings and execution time statistics.") |
| cmp_parser.add_argument("--root-old", dest="root_old", |
| help="Prefix to ignore on source files for " |
| "OLD directory", |
| action="store", type=str, default="") |
| cmp_parser.add_argument("--root-new", dest="root_new", |
| help="Prefix to ignore on source files for " |
| "NEW directory", |
| action="store", type=str, default="") |
| cmp_parser.add_argument("--verbose-log", dest="verbose_log", |
| help="Write additional information to LOG " |
| "[default=None]", |
| action="store", type=str, default=None, |
| metavar="LOG") |
| cmp_parser.add_argument("--stats-only", action="store_true", |
| dest="stats_only", default=False, |
| help="Only show statistics on reports") |
| cmp_parser.add_argument("--show-stats", action="store_true", |
| dest="show_stats", default=False, |
| help="Show change in statistics") |
| cmp_parser.add_argument("--histogram", action="store", default=None, |
| help="Show histogram of paths differences. " |
| "Requires matplotlib") |
| cmp_parser.add_argument("old", nargs=1, help="Directory with old results") |
| cmp_parser.add_argument("new", nargs=1, help="Directory with new results") |
| cmp_parser.set_defaults(func=compare) |
| |
| # update subcommand |
| upd_parser = subparsers.add_parser( |
| "update", |
| help="Update static analyzer reference results based on the previous " |
| "run of SATest build. Assumes that SATest build was just run.") |
| upd_parser.add_argument("--git", action="store_true", |
| help="Stage updated results using git.") |
| upd_parser.set_defaults(func=update) |
| |
| # docker subcommand |
| dock_parser = subparsers.add_parser( |
| "docker", |
| help="Run regression system in the docker.") |
| |
| dock_parser.add_argument("--build-image", action="store_true", |
| help="Build docker image for running tests.") |
| dock_parser.add_argument("--shell", action="store_true", |
| help="Start a shell on docker.") |
| dock_parser.add_argument("--llvm-project-dir", action="store", |
| default=DEFAULT_LLVM_DIR, |
| help="Path to LLVM source code. Defaults " |
| "to the repo where this script is located. ") |
| dock_parser.add_argument("--build-dir", action="store", default="", |
| help="Path to a directory where docker should " |
| "build LLVM code.") |
| dock_parser.add_argument("--clang-dir", action="store", default="", |
| help="Path to find/install LLVM installation.") |
| dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[], |
| help="Additionall args that will be forwarded " |
| "to the docker's entrypoint.") |
| dock_parser.set_defaults(func=docker) |
| |
| # benchmark subcommand |
| bench_parser = subparsers.add_parser( |
| "benchmark", |
| help="Run benchmarks by building a set of projects multiple times.") |
| |
| bench_parser.add_argument("-i", "--iterations", action="store", |
| type=int, default=20, |
| help="Number of iterations for building each " |
| "project.") |
| bench_parser.add_argument("-o", "--output", action="store", |
| default="benchmark.csv", |
| help="Output csv file for the benchmark results") |
| bench_parser.add_argument("--projects", action="store", default="", |
| help="Comma-separated list of projects to test") |
| bench_parser.add_argument("--max-size", action="store", default=None, |
| help="Maximum size for the projects to test") |
| bench_parser.set_defaults(func=benchmark) |
| |
| bench_subparsers = bench_parser.add_subparsers() |
| bench_compare_parser = bench_subparsers.add_parser( |
| "compare", |
| help="Compare benchmark runs.") |
| bench_compare_parser.add_argument("--old", action="store", required=True, |
| help="Benchmark reference results to " |
| "compare agains.") |
| bench_compare_parser.add_argument("--new", action="store", required=True, |
| help="New benchmark results to check.") |
| bench_compare_parser.add_argument("-o", "--output", |
| action="store", required=True, |
| help="Output file for plots.") |
| bench_compare_parser.set_defaults(func=benchmark_compare) |
| |
| args = parser.parse_args() |
| args.func(parser, args) |
| |
| |
| if __name__ == "__main__": |
| main() |
