| #!/usr/bin/env python3 | |
| # ===----------------------------------------------------------------------===## | |
| # | |
| # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| # | |
| # ===----------------------------------------------------------------------===## | |
| """Script to bisect over files in an rsp file. | |
| This is mostly used for detecting which file contains a miscompile between two | |
| compiler revisions. It does this by bisecting over an rsp file. Between two | |
| build directories, this script will make the rsp file reference the current | |
| build directory's version of some set of the rsp's object files/libraries, and | |
| reference the other build directory's version of the same files for the | |
| remaining set of object files/libraries. | |
| Build the target in two separate directories with the two compiler revisions, | |
| keeping the rsp file around since ninja by default deletes the rsp file after | |
| building. | |
| $ ninja -d keeprsp mytarget | |
| Create a script to build the target and run an interesting test. Get the | |
| command to build the target via | |
| $ ninja -t commands | grep mytarget | |
| The command to build the target should reference the rsp file. | |
| This script doesn't care if the test script returns 0 or 1 for specifically the | |
| successful or failing test, just that the test script returns a different | |
| return code for success vs failure. | |
| Since the command that `ninja -t commands` is run from the build directory, | |
| usually the test script cd's to the build directory. | |
| $ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp | |
| --other_rel_path=../Other | |
| where --other_rel_path is the relative path from the first build directory to | |
| the other build directory. This is prepended to files in the rsp. | |
| For a full example, if the foo target is suspected to contain a miscompile in | |
| some file, have two different build directories, buildgood/ and buildbad/ and | |
| run | |
| $ ninja -d keeprsp foo | |
| in both so we have two versions of all relevant object files that may contain a | |
| miscompile, one built by a good compiler and one by a bad compiler. | |
| In buildgood/, run | |
| $ ninja -t commands | grep '-o .*foo' | |
| to get the command to link the files together. It may look something like | |
| clang -o foo @foo.rsp | |
| Now create a test script that runs the link step and whatever test reproduces a | |
| miscompile and returns a non-zero exit code when there is a miscompile. For | |
| example | |
| ``` | |
| #!/bin/bash | |
| # immediately bail out of script if any command returns a non-zero return code | |
| set -e | |
| clang -o foo @foo.rsp | |
| ./foo | |
| ``` | |
| With buildgood/ as the working directory, run | |
| $ path/to/llvm-project/llvm/utils/rsp_bisect.py \ | |
| --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/ | |
| If rsp_bisect is successful, it will print the first file in the rsp file that | |
| when using the bad build directory's version causes the test script to return a | |
| different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0 | |
| will be a copy of foo.rsp with the relevant file using the version in | |
| buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file | |
| using the version in buildbad/. | |
| """ | |
| import argparse | |
| import os | |
| import subprocess | |
| import sys | |
| def is_path(s): | |
| return "/" in s | |
| def run_test(test): | |
| """Runs the test and returns whether it was successful or not.""" | |
| return subprocess.run([test], capture_output=True).returncode == 0 | |
| def modify_rsp(rsp_entries, other_rel_path, modify_after_num): | |
| """Create a modified rsp file for use in bisection. | |
| Returns a new list from rsp. | |
| For each file in rsp after the first modify_after_num files, prepend | |
| other_rel_path. | |
| """ | |
| ret = [] | |
| for r in rsp_entries: | |
| if is_path(r): | |
| if modify_after_num == 0: | |
| r = os.path.join(other_rel_path, r) | |
| else: | |
| modify_after_num -= 1 | |
| ret.append(r) | |
| assert modify_after_num == 0 | |
| return ret | |
| def test_modified_rsp(test, modified_rsp_entries, rsp_path): | |
| """Write the rsp file to disk and run the test.""" | |
| with open(rsp_path, "w") as f: | |
| f.write(" ".join(modified_rsp_entries)) | |
| return run_test(test) | |
| def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path): | |
| """Bisect over rsp entries. | |
| Args: | |
| zero_result: the test result when modify_after_num is 0. | |
| Returns: | |
| The index of the file in the rsp file where the test result changes. | |
| """ | |
| lower = 0 | |
| upper = num_files_in_rsp | |
| while lower != upper - 1: | |
| assert lower < upper - 1 | |
| mid = int((lower + upper) / 2) | |
| assert lower != mid and mid != upper | |
| print("Trying {} ({}-{})".format(mid, lower, upper)) | |
| result = test_modified_rsp( | |
| test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path | |
| ) | |
| if zero_result == result: | |
| lower = mid | |
| else: | |
| upper = mid | |
| return upper | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument( | |
| "--test", help="Binary to test if current setup is good or bad", required=True | |
| ) | |
| parser.add_argument("--rsp", help="rsp file", required=True) | |
| parser.add_argument( | |
| "--other-rel-path", | |
| help="Relative path from current build directory to other build " | |
| + 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"', | |
| required=True, | |
| ) | |
| args = parser.parse_args() | |
| with open(args.rsp, "r") as f: | |
| rsp_entries = f.read() | |
| rsp_entries = rsp_entries.split() | |
| num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a)) | |
| if num_files_in_rsp == 0: | |
| print("No files in rsp?") | |
| return 1 | |
| print("{} files in rsp".format(num_files_in_rsp)) | |
| try: | |
| print("Initial testing") | |
| test0 = test_modified_rsp( | |
| args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp | |
| ) | |
| test_all = test_modified_rsp( | |
| args.test, | |
| modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp), | |
| args.rsp, | |
| ) | |
| if test0 == test_all: | |
| print("Test returned same exit code for both build directories") | |
| return 1 | |
| print("First build directory returned " + ("0" if test_all else "1")) | |
| result = bisect( | |
| args.test, | |
| test0, | |
| rsp_entries, | |
| num_files_in_rsp, | |
| args.other_rel_path, | |
| args.rsp, | |
| ) | |
| print( | |
| "First file change: {} ({})".format( | |
| list(filter(is_path, rsp_entries))[result - 1], result | |
| ) | |
| ) | |
| rsp_out_0 = args.rsp + ".0" | |
| rsp_out_1 = args.rsp + ".1" | |
| with open(rsp_out_0, "w") as f: | |
| f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1))) | |
| with open(rsp_out_1, "w") as f: | |
| f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result))) | |
| print( | |
| "Bisection point rsp files written to {} and {}".format( | |
| rsp_out_0, rsp_out_1 | |
| ) | |
| ) | |
| finally: | |
| # Always make sure to write the original rsp file contents back so it's | |
| # less of a pain to rerun this script. | |
| with open(args.rsp, "w") as f: | |
| f.write(" ".join(rsp_entries)) | |
| if __name__ == "__main__": | |
| sys.exit(main()) |