utils/rsp_bisect.py - llvm-project/llvm - Git at Google

 #!/usr/bin/env python3
 # ===----------------------------------------------------------------------===##
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 # ===----------------------------------------------------------------------===##
 """Script to bisect over files in an rsp file.

 This is mostly used for detecting which file contains a miscompile between two
 compiler revisions. It does this by bisecting over an rsp file. Between two
 build directories, this script will make the rsp file reference the current
 build directory's version of some set of the rsp's object files/libraries, and
 reference the other build directory's version of the same files for the
 remaining set of object files/libraries.

 Build the target in two separate directories with the two compiler revisions,
 keeping the rsp file around since ninja by default deletes the rsp file after
 building.
 $ ninja -d keeprsp mytarget

 Create a script to build the target and run an interesting test. Get the
 command to build the target via
 $ ninja -t commands | grep mytarget
 The command to build the target should reference the rsp file.
 This script doesn't care if the test script returns 0 or 1 for specifically the
 successful or failing test, just that the test script returns a different
 return code for success vs failure.
 Since the command that `ninja -t commands` is run from the build directory,
 usually the test script cd's to the build directory.

 $ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp
     --other_rel_path=../Other
 where --other_rel_path is the relative path from the first build directory to
 the other build directory. This is prepended to files in the rsp.


 For a full example, if the foo target is suspected to contain a miscompile in
 some file, have two different build directories, buildgood/ and buildbad/ and
 run
 $ ninja -d keeprsp foo
 in both so we have two versions of all relevant object files that may contain a
 miscompile, one built by a good compiler and one by a bad compiler.

 In buildgood/, run
 $ ninja -t commands | grep '-o .*foo'
 to get the command to link the files together. It may look something like
   clang -o foo @foo.rsp

 Now create a test script that runs the link step and whatever test reproduces a
 miscompile and returns a non-zero exit code when there is a miscompile. For
 example
 ```
   #!/bin/bash
   # immediately bail out of script if any command returns a non-zero return code
   set -e
   clang -o foo @foo.rsp
   ./foo
 ```

 With buildgood/ as the working directory, run
 $ path/to/llvm-project/llvm/utils/rsp_bisect.py \
     --test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/
 If rsp_bisect is successful, it will print the first file in the rsp file that
 when using the bad build directory's version causes the test script to return a
 different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0
 will be a copy of foo.rsp with the relevant file using the version in
 buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file
 using the version in buildbad/.

 """

 import argparse
 import os
 import subprocess
 import sys


 def is_path(s):
     return "/" in s


 def run_test(test):
     """Runs the test and returns whether it was successful or not."""
     return subprocess.run([test], capture_output=True).returncode == 0


 def modify_rsp(rsp_entries, other_rel_path, modify_after_num):
     """Create a modified rsp file for use in bisection.

     Returns a new list from rsp.
     For each file in rsp after the first modify_after_num files, prepend
     other_rel_path.
     """
     ret = []
     for r in rsp_entries:
         if is_path(r):
             if modify_after_num == 0:
                 r = os.path.join(other_rel_path, r)
             else:
                 modify_after_num -= 1
         ret.append(r)
     assert modify_after_num == 0
     return ret


 def test_modified_rsp(test, modified_rsp_entries, rsp_path):
     """Write the rsp file to disk and run the test."""
     with open(rsp_path, "w") as f:
         f.write(" ".join(modified_rsp_entries))
     return run_test(test)


 def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):
     """Bisect over rsp entries.

     Args:
         zero_result: the test result when modify_after_num is 0.

     Returns:
         The index of the file in the rsp file where the test result changes.
     """
     lower = 0
     upper = num_files_in_rsp
     while lower != upper - 1:
         assert lower < upper - 1
         mid = int((lower + upper) / 2)
         assert lower != mid and mid != upper
         print("Trying {} ({}-{})".format(mid, lower, upper))
         result = test_modified_rsp(
             test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path
         )
         if zero_result == result:
             lower = mid
         else:
             upper = mid
     return upper


 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--test", help="Binary to test if current setup is good or bad", required=True
     )
     parser.add_argument("--rsp", help="rsp file", required=True)
     parser.add_argument(
         "--other-rel-path",
         help="Relative path from current build directory to other build "
         + 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',
         required=True,
     )
     args = parser.parse_args()

     with open(args.rsp, "r") as f:
         rsp_entries = f.read()
     rsp_entries = rsp_entries.split()
     num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))
     if num_files_in_rsp == 0:
         print("No files in rsp?")
         return 1
     print("{} files in rsp".format(num_files_in_rsp))

     try:
         print("Initial testing")
         test0 = test_modified_rsp(
             args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp
         )
         test_all = test_modified_rsp(
             args.test,
             modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),
             args.rsp,
         )

         if test0 == test_all:
             print("Test returned same exit code for both build directories")
             return 1

         print("First build directory returned " + ("0" if test_all else "1"))

         result = bisect(
             args.test,
             test0,
             rsp_entries,
             num_files_in_rsp,
             args.other_rel_path,
             args.rsp,
         )
         print(
             "First file change: {} ({})".format(
                 list(filter(is_path, rsp_entries))[result - 1], result
             )
         )

         rsp_out_0 = args.rsp + ".0"
         rsp_out_1 = args.rsp + ".1"
         with open(rsp_out_0, "w") as f:
             f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))
         with open(rsp_out_1, "w") as f:
             f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result)))
         print(
             "Bisection point rsp files written to {} and {}".format(
                 rsp_out_0, rsp_out_1
             )
         )
     finally:
         # Always make sure to write the original rsp file contents back so it's
         # less of a pain to rerun this script.
         with open(args.rsp, "w") as f:
             f.write(" ".join(rsp_entries))


 if __name__ == "__main__":
     sys.exit(main())
	#!/usr/bin/env python3
	# ===----------------------------------------------------------------------===##
	#
	# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	# See https://llvm.org/LICENSE.txt for license information.
	# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	#
	# ===----------------------------------------------------------------------===##
	"""Script to bisect over files in an rsp file.

	This is mostly used for detecting which file contains a miscompile between two
	compiler revisions. It does this by bisecting over an rsp file. Between two
	build directories, this script will make the rsp file reference the current
	build directory's version of some set of the rsp's object files/libraries, and
	reference the other build directory's version of the same files for the
	remaining set of object files/libraries.

	Build the target in two separate directories with the two compiler revisions,
	keeping the rsp file around since ninja by default deletes the rsp file after
	building.
	$ ninja -d keeprsp mytarget

	Create a script to build the target and run an interesting test. Get the
	command to build the target via
	$ ninja -t commands \| grep mytarget
	The command to build the target should reference the rsp file.
	This script doesn't care if the test script returns 0 or 1 for specifically the
	successful or failing test, just that the test script returns a different
	return code for success vs failure.
	Since the command that `ninja -t commands` is run from the build directory,
	usually the test script cd's to the build directory.

	$ rsp_bisect.py --test=path/to/test_script --rsp=path/to/build/target.rsp
	--other_rel_path=../Other
	where --other_rel_path is the relative path from the first build directory to
	the other build directory. This is prepended to files in the rsp.


	For a full example, if the foo target is suspected to contain a miscompile in
	some file, have two different build directories, buildgood/ and buildbad/ and
	run
	$ ninja -d keeprsp foo
	in both so we have two versions of all relevant object files that may contain a
	miscompile, one built by a good compiler and one by a bad compiler.

	In buildgood/, run
	$ ninja -t commands \| grep '-o .*foo'
	to get the command to link the files together. It may look something like
	clang -o foo @foo.rsp

	Now create a test script that runs the link step and whatever test reproduces a
	miscompile and returns a non-zero exit code when there is a miscompile. For
	example
	```
	#!/bin/bash
	# immediately bail out of script if any command returns a non-zero return code
	set -e
	clang -o foo @foo.rsp
	./foo
	```

	With buildgood/ as the working directory, run
	$ path/to/llvm-project/llvm/utils/rsp_bisect.py \
	--test=path/to/test_script --rsp=./foo.rsp --other_rel_path=../buildbad/
	If rsp_bisect is successful, it will print the first file in the rsp file that
	when using the bad build directory's version causes the test script to return a
	different return code. foo.rsp.0 and foo.rsp.1 will also be written. foo.rsp.0
	will be a copy of foo.rsp with the relevant file using the version in
	buildgood/, and foo.rsp.1 will be a copy of foo.rsp with the relevant file
	using the version in buildbad/.

	"""

	import argparse
	import os
	import subprocess
	import sys


	def is_path(s):
	return "/" in s


	def run_test(test):
	"""Runs the test and returns whether it was successful or not."""
	return subprocess.run([test], capture_output=True).returncode == 0


	def modify_rsp(rsp_entries, other_rel_path, modify_after_num):
	"""Create a modified rsp file for use in bisection.

	Returns a new list from rsp.
	For each file in rsp after the first modify_after_num files, prepend
	other_rel_path.
	"""
	ret = []
	for r in rsp_entries:
	if is_path(r):
	if modify_after_num == 0:
	r = os.path.join(other_rel_path, r)
	else:
	modify_after_num -= 1
	ret.append(r)
	assert modify_after_num == 0
	return ret


	def test_modified_rsp(test, modified_rsp_entries, rsp_path):
	"""Write the rsp file to disk and run the test."""
	with open(rsp_path, "w") as f:
	f.write(" ".join(modified_rsp_entries))
	return run_test(test)


	def bisect(test, zero_result, rsp_entries, num_files_in_rsp, other_rel_path, rsp_path):
	"""Bisect over rsp entries.

	Args:
	zero_result: the test result when modify_after_num is 0.

	Returns:
	The index of the file in the rsp file where the test result changes.
	"""
	lower = 0
	upper = num_files_in_rsp
	while lower != upper - 1:
	assert lower < upper - 1
	mid = int((lower + upper) / 2)
	assert lower != mid and mid != upper
	print("Trying {} ({}-{})".format(mid, lower, upper))
	result = test_modified_rsp(
	test, modify_rsp(rsp_entries, other_rel_path, mid), rsp_path
	)
	if zero_result == result:
	lower = mid
	else:
	upper = mid
	return upper


	def main():
	parser = argparse.ArgumentParser()
	parser.add_argument(
	"--test", help="Binary to test if current setup is good or bad", required=True
	)
	parser.add_argument("--rsp", help="rsp file", required=True)
	parser.add_argument(
	"--other-rel-path",
	help="Relative path from current build directory to other build "
	+ 'directory, e.g. from "out/Default" to "out/Other" specify "../Other"',
	required=True,
	)
	args = parser.parse_args()

	with open(args.rsp, "r") as f:
	rsp_entries = f.read()
	rsp_entries = rsp_entries.split()
	num_files_in_rsp = sum(1 for a in rsp_entries if is_path(a))
	if num_files_in_rsp == 0:
	print("No files in rsp?")
	return 1
	print("{} files in rsp".format(num_files_in_rsp))

	try:
	print("Initial testing")
	test0 = test_modified_rsp(
	args.test, modify_rsp(rsp_entries, args.other_rel_path, 0), args.rsp
	)
	test_all = test_modified_rsp(
	args.test,
	modify_rsp(rsp_entries, args.other_rel_path, num_files_in_rsp),
	args.rsp,
	)

	if test0 == test_all:
	print("Test returned same exit code for both build directories")
	return 1

	print("First build directory returned " + ("0" if test_all else "1"))

	result = bisect(
	args.test,
	test0,
	rsp_entries,
	num_files_in_rsp,
	args.other_rel_path,
	args.rsp,
	)
	print(
	"First file change: {} ({})".format(
	list(filter(is_path, rsp_entries))[result - 1], result
	)
	)

	rsp_out_0 = args.rsp + ".0"
	rsp_out_1 = args.rsp + ".1"
	with open(rsp_out_0, "w") as f:
	f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result - 1)))
	with open(rsp_out_1, "w") as f:
	f.write(" ".join(modify_rsp(rsp_entries, args.other_rel_path, result)))
	print(
	"Bisection point rsp files written to {} and {}".format(
	rsp_out_0, rsp_out_1
	)
	)
	finally:
	# Always make sure to write the original rsp file contents back so it's
	# less of a pain to rerun this script.
	with open(args.rsp, "w") as f:
	f.write(" ".join(rsp_entries))


	if __name__ == "__main__":
	sys.exit(main())