utils/update_test_checks.py - llvm-project/llvm - Git at Google

 #!/usr/bin/env python3

 """A script to generate FileCheck statements for 'opt' regression tests.

 This script is a utility to update LLVM opt test cases with new
 FileCheck patterns. It can either update all of the tests in the file or
 a single test function.

 Example usage:

 # Default to using `opt` as found in your PATH.
 $ update_test_checks.py test/foo.ll

 # Override the path lookup.
 $ update_test_checks.py --tool-binary=../bin/opt test/foo.ll

 # Use a custom tool instead of `opt`.
 $ update_test_checks.py --tool=yourtool test/foo.ll

 Workflow:
 1. Make a compiler patch that requires updating some number of FileCheck lines
    in regression test files.
 2. Save the patch and revert it from your local work area.
 3. Update the RUN-lines in the affected regression tests to look canonical.
    Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
 4. Refresh the FileCheck lines for either the entire file or select functions by
    running this script.
 5. Commit the fresh baseline of checks.
 6. Apply your patch from step 1 and rebuild your local binaries.
 7. Re-run this script on affected regression tests.
 8. Check the diffs to ensure the script has done something reasonable.
 9. Submit a patch including the regression test diffs for review.
 """

 from __future__ import print_function

 import argparse
 import os  # Used to advertise this file's name ("autogenerated_note").
 import re
 import sys

 from UpdateTestChecks import common


 def main():
     from argparse import RawTextHelpFormatter

     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=RawTextHelpFormatter
     )
     parser.add_argument(
         "--tool",
         default="opt",
         help='The name of the tool used to generate the test case (defaults to "opt")',
     )
     parser.add_argument(
         "--tool-binary",
         "--opt-binary",
         help="The tool binary used to generate the test case",
     )
     parser.add_argument("--function", help="The function in the test file to update")
     parser.add_argument(
         "-p", "--preserve-names", action="store_true", help="Do not scrub IR names"
     )
     parser.add_argument(
         "--function-signature",
         action="store_true",
         help="Keep function signature information around for the check line",
     )
     parser.add_argument(
         "--scrub-attributes",
         action="store_true",
         help="Remove attribute annotations (#0) from the end of check line",
     )
     parser.add_argument(
         "--check-attributes",
         action="store_true",
         help='Check "Function Attributes" for functions',
     )
     parser.add_argument(
         "--check-globals",
         action="store_true",
         help="Check global entries (global variables, metadata, attribute sets, ...) for functions",
     )
     parser.add_argument("tests", nargs="+")
     initial_args = common.parse_commandline_args(parser)

     script_name = os.path.basename(__file__)

     if initial_args.tool_binary:
         tool_basename = os.path.basename(initial_args.tool_binary)
         if not re.match(r"^%s(-\d+)?(\.exe)?$" % (initial_args.tool), tool_basename):
             common.error("Unexpected tool name: " + tool_basename)
             sys.exit(1)

     for ti in common.itertests(
         initial_args.tests, parser, script_name="utils/" + script_name
     ):
         # If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
         if ti.args.scrub_attributes:
             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
                 common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
             )
         else:
             common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
                 common.SCRUB_TRAILING_WHITESPACE_RE
             )

         tool_basename = ti.args.tool

         prefix_list = []
         for l in ti.run_lines:
             if "|" not in l:
                 common.warn("Skipping unparsable RUN line: " + l)
                 continue

             commands = [cmd.strip() for cmd in l.split("|")]
             assert len(commands) >= 2
             preprocess_cmd = None
             if len(commands) > 2:
                 preprocess_cmd = " | ".join(commands[:-2])
             tool_cmd = commands[-2]
             filecheck_cmd = commands[-1]
             common.verify_filecheck_prefixes(filecheck_cmd)
             if not tool_cmd.startswith(tool_basename + " "):
                 common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
                 continue

             if not filecheck_cmd.startswith("FileCheck "):
                 common.warn("Skipping non-FileChecked RUN line: " + l)
                 continue

             tool_cmd_args = tool_cmd[len(tool_basename) :].strip()
             tool_cmd_args = tool_cmd_args.replace("< %s", "").replace("%s", "").strip()
             check_prefixes = common.get_check_prefixes(filecheck_cmd)

             # FIXME: We should use multiple check prefixes to common check lines. For
             # now, we just ignore all but the last.
             prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))

         global_vars_seen_dict = {}
         builder = common.FunctionTestBuilder(
             run_list=prefix_list, flags=ti.args, scrubber_args=[], path=ti.path
         )

         tool_binary = ti.args.tool_binary
         if not tool_binary:
             tool_binary = tool_basename

         for prefixes, tool_args, preprocess_cmd in prefix_list:
             common.debug("Extracted tool cmd: " + tool_basename + " " + tool_args)
             common.debug("Extracted FileCheck prefixes: " + str(prefixes))

             raw_tool_output = common.invoke_tool(
                 tool_binary,
                 tool_args,
                 ti.path,
                 preprocess_cmd=preprocess_cmd,
                 verbose=ti.args.verbose,
             )
             builder.process_run_line(
                 common.OPT_FUNCTION_RE,
                 common.scrub_body,
                 raw_tool_output,
                 prefixes,
                 False,
             )
             builder.processed_prefixes(prefixes)

         func_dict = builder.finish_and_get_func_dict()
         is_in_function = False
         is_in_function_start = False
         has_checked_pre_function_globals = False
         prefix_set = set(
             [prefix for prefixes, _, _ in prefix_list for prefix in prefixes]
         )
         common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
         output_lines = []

         include_generated_funcs = common.find_arg_in_test(
             ti,
             lambda args: ti.args.include_generated_funcs,
             "--include-generated-funcs",
             True,
         )
         generated_prefixes = []
         if include_generated_funcs:
             # Generate the appropriate checks for each function.  We need to emit
             # these in the order according to the generated output so that CHECK-LABEL
             # works properly.  func_order provides that.

             # We can't predict where various passes might insert functions so we can't
             # be sure the input function order is maintained.  Therefore, first spit
             # out all the source lines.
             common.dump_input_lines(output_lines, ti, prefix_set, ";")

             args = ti.args
             if args.check_globals:
                 generated_prefixes.extend(
                     common.add_global_checks(
                         builder.global_var_dict(),
                         ";",
                         prefix_list,
                         output_lines,
                         global_vars_seen_dict,
                         args.preserve_names,
                         True,
                     )
                 )

             # Now generate all the checks.
             generated_prefixes.extend(
                 common.add_checks_at_end(
                     output_lines,
                     prefix_list,
                     builder.func_order(),
                     ";",
                     lambda my_output_lines, prefixes, func: common.add_ir_checks(
                         my_output_lines,
                         ";",
                         prefixes,
                         func_dict,
                         func,
                         False,
                         args.function_signature,
                         args.version,
                         global_vars_seen_dict,
                         is_filtered=builder.is_filtered(),
                     ),
                 )
             )
         else:
             # "Normal" mode.
             for input_line_info in ti.iterlines(output_lines):
                 input_line = input_line_info.line
                 args = input_line_info.args
                 if is_in_function_start:
                     if input_line == "":
                         continue
                     if input_line.lstrip().startswith(";"):
                         m = common.CHECK_RE.match(input_line)
                         if not m or m.group(1) not in prefix_set:
                             output_lines.append(input_line)
                             continue

                     # Print out the various check lines here.
                     generated_prefixes.extend(
                         common.add_ir_checks(
                             output_lines,
                             ";",
                             prefix_list,
                             func_dict,
                             func_name,
                             args.preserve_names,
                             args.function_signature,
                             args.version,
                             global_vars_seen_dict,
                             is_filtered=builder.is_filtered(),
                         )
                     )
                     is_in_function_start = False

                 m = common.IR_FUNCTION_RE.match(input_line)
                 if m and not has_checked_pre_function_globals:
                     if args.check_globals:
                         generated_prefixes.extend(
                             common.add_global_checks(
                                 builder.global_var_dict(),
                                 ";",
                                 prefix_list,
                                 output_lines,
                                 global_vars_seen_dict,
                                 args.preserve_names,
                                 True,
                             )
                         )
                     has_checked_pre_function_globals = True

                 if common.should_add_line_to_output(
                     input_line, prefix_set, not is_in_function
                 ):
                     # This input line of the function body will go as-is into the output.
                     # Except make leading whitespace uniform: 2 spaces.
                     input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(
                         r"  ", input_line
                     )
                     output_lines.append(input_line)
                     if input_line.strip() == "}":
                         is_in_function = False
                         continue

                 if is_in_function:
                     continue

                 m = common.IR_FUNCTION_RE.match(input_line)
                 if not m:
                     continue
                 func_name = m.group(1)
                 if args.function is not None and func_name != args.function:
                     # When filtering on a specific function, skip all others.
                     continue
                 is_in_function = is_in_function_start = True

         if args.check_globals:
             generated_prefixes.extend(
                 common.add_global_checks(
                     builder.global_var_dict(),
                     ";",
                     prefix_list,
                     output_lines,
                     global_vars_seen_dict,
                     args.preserve_names,
                     False,
                 )
             )
         if ti.args.gen_unused_prefix_body:
             output_lines.extend(
                 ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes)
             )
         common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))

         with open(ti.path, "wb") as f:
             f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])


 if __name__ == "__main__":
     main()
	#!/usr/bin/env python3

	"""A script to generate FileCheck statements for 'opt' regression tests.

	This script is a utility to update LLVM opt test cases with new
	FileCheck patterns. It can either update all of the tests in the file or
	a single test function.

	Example usage:

	# Default to using `opt` as found in your PATH.
	$ update_test_checks.py test/foo.ll

	# Override the path lookup.
	$ update_test_checks.py --tool-binary=../bin/opt test/foo.ll

	# Use a custom tool instead of `opt`.
	$ update_test_checks.py --tool=yourtool test/foo.ll

	Workflow:
	1. Make a compiler patch that requires updating some number of FileCheck lines
	in regression test files.
	2. Save the patch and revert it from your local work area.
	3. Update the RUN-lines in the affected regression tests to look canonical.
	Example: "; RUN: opt < %s -instcombine -S \| FileCheck %s"
	4. Refresh the FileCheck lines for either the entire file or select functions by
	running this script.
	5. Commit the fresh baseline of checks.
	6. Apply your patch from step 1 and rebuild your local binaries.
	7. Re-run this script on affected regression tests.
	8. Check the diffs to ensure the script has done something reasonable.
	9. Submit a patch including the regression test diffs for review.
	"""

	from __future__ import print_function

	import argparse
	import os # Used to advertise this file's name ("autogenerated_note").
	import re
	import sys

	from UpdateTestChecks import common


	def main():
	from argparse import RawTextHelpFormatter

	parser = argparse.ArgumentParser(
	description=__doc__, formatter_class=RawTextHelpFormatter
	)
	parser.add_argument(
	"--tool",
	default="opt",
	help='The name of the tool used to generate the test case (defaults to "opt")',
	)
	parser.add_argument(
	"--tool-binary",
	"--opt-binary",
	help="The tool binary used to generate the test case",
	)
	parser.add_argument("--function", help="The function in the test file to update")
	parser.add_argument(
	"-p", "--preserve-names", action="store_true", help="Do not scrub IR names"
	)
	parser.add_argument(
	"--function-signature",
	action="store_true",
	help="Keep function signature information around for the check line",
	)
	parser.add_argument(
	"--scrub-attributes",
	action="store_true",
	help="Remove attribute annotations (#0) from the end of check line",
	)
	parser.add_argument(
	"--check-attributes",
	action="store_true",
	help='Check "Function Attributes" for functions',
	)
	parser.add_argument(
	"--check-globals",
	action="store_true",
	help="Check global entries (global variables, metadata, attribute sets, ...) for functions",
	)
	parser.add_argument("tests", nargs="+")
	initial_args = common.parse_commandline_args(parser)

	script_name = os.path.basename(__file__)

	if initial_args.tool_binary:
	tool_basename = os.path.basename(initial_args.tool_binary)
	if not re.match(r"^%s(-\d+)?(\.exe)?$" % (initial_args.tool), tool_basename):
	common.error("Unexpected tool name: " + tool_basename)
	sys.exit(1)

	for ti in common.itertests(
	initial_args.tests, parser, script_name="utils/" + script_name
	):
	# If requested we scrub trailing attribute annotations, e.g., '#0', together with whitespaces
	if ti.args.scrub_attributes:
	common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
	common.SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE
	)
	else:
	common.SCRUB_TRAILING_WHITESPACE_TEST_RE = (
	common.SCRUB_TRAILING_WHITESPACE_RE
	)

	tool_basename = ti.args.tool

	prefix_list = []
	for l in ti.run_lines:
	if "\|" not in l:
	common.warn("Skipping unparsable RUN line: " + l)
	continue

	commands = [cmd.strip() for cmd in l.split("\|")]
	assert len(commands) >= 2
	preprocess_cmd = None
	if len(commands) > 2:
	preprocess_cmd = " \| ".join(commands[:-2])
	tool_cmd = commands[-2]
	filecheck_cmd = commands[-1]
	common.verify_filecheck_prefixes(filecheck_cmd)
	if not tool_cmd.startswith(tool_basename + " "):
	common.warn("Skipping non-%s RUN line: %s" % (tool_basename, l))
	continue

	if not filecheck_cmd.startswith("FileCheck "):
	common.warn("Skipping non-FileChecked RUN line: " + l)
	continue

	tool_cmd_args = tool_cmd[len(tool_basename) :].strip()
	tool_cmd_args = tool_cmd_args.replace("< %s", "").replace("%s", "").strip()
	check_prefixes = common.get_check_prefixes(filecheck_cmd)

	# FIXME: We should use multiple check prefixes to common check lines. For
	# now, we just ignore all but the last.
	prefix_list.append((check_prefixes, tool_cmd_args, preprocess_cmd))

	global_vars_seen_dict = {}
	builder = common.FunctionTestBuilder(
	run_list=prefix_list, flags=ti.args, scrubber_args=[], path=ti.path
	)

	tool_binary = ti.args.tool_binary
	if not tool_binary:
	tool_binary = tool_basename

	for prefixes, tool_args, preprocess_cmd in prefix_list:
	common.debug("Extracted tool cmd: " + tool_basename + " " + tool_args)
	common.debug("Extracted FileCheck prefixes: " + str(prefixes))

	raw_tool_output = common.invoke_tool(
	tool_binary,
	tool_args,
	ti.path,
	preprocess_cmd=preprocess_cmd,
	verbose=ti.args.verbose,
	)
	builder.process_run_line(
	common.OPT_FUNCTION_RE,
	common.scrub_body,
	raw_tool_output,
	prefixes,
	False,
	)
	builder.processed_prefixes(prefixes)

	func_dict = builder.finish_and_get_func_dict()
	is_in_function = False
	is_in_function_start = False
	has_checked_pre_function_globals = False
	prefix_set = set(
	[prefix for prefixes, _, _ in prefix_list for prefix in prefixes]
	)
	common.debug("Rewriting FileCheck prefixes:", str(prefix_set))
	output_lines = []

	include_generated_funcs = common.find_arg_in_test(
	ti,
	lambda args: ti.args.include_generated_funcs,
	"--include-generated-funcs",
	True,
	)
	generated_prefixes = []
	if include_generated_funcs:
	# Generate the appropriate checks for each function. We need to emit
	# these in the order according to the generated output so that CHECK-LABEL
	# works properly. func_order provides that.

	# We can't predict where various passes might insert functions so we can't
	# be sure the input function order is maintained. Therefore, first spit
	# out all the source lines.
	common.dump_input_lines(output_lines, ti, prefix_set, ";")

	args = ti.args
	if args.check_globals:
	generated_prefixes.extend(
	common.add_global_checks(
	builder.global_var_dict(),
	";",
	prefix_list,
	output_lines,
	global_vars_seen_dict,
	args.preserve_names,
	True,
	)
	)

	# Now generate all the checks.
	generated_prefixes.extend(
	common.add_checks_at_end(
	output_lines,
	prefix_list,
	builder.func_order(),
	";",
	lambda my_output_lines, prefixes, func: common.add_ir_checks(
	my_output_lines,
	";",
	prefixes,
	func_dict,
	func,
	False,
	args.function_signature,
	args.version,
	global_vars_seen_dict,
	is_filtered=builder.is_filtered(),
	),
	)
	)
	else:
	# "Normal" mode.
	for input_line_info in ti.iterlines(output_lines):
	input_line = input_line_info.line
	args = input_line_info.args
	if is_in_function_start:
	if input_line == "":
	continue
	if input_line.lstrip().startswith(";"):
	m = common.CHECK_RE.match(input_line)
	if not m or m.group(1) not in prefix_set:
	output_lines.append(input_line)
	continue

	# Print out the various check lines here.
	generated_prefixes.extend(
	common.add_ir_checks(
	output_lines,
	";",
	prefix_list,
	func_dict,
	func_name,
	args.preserve_names,
	args.function_signature,
	args.version,
	global_vars_seen_dict,
	is_filtered=builder.is_filtered(),
	)
	)
	is_in_function_start = False

	m = common.IR_FUNCTION_RE.match(input_line)
	if m and not has_checked_pre_function_globals:
	if args.check_globals:
	generated_prefixes.extend(
	common.add_global_checks(
	builder.global_var_dict(),
	";",
	prefix_list,
	output_lines,
	global_vars_seen_dict,
	args.preserve_names,
	True,
	)
	)
	has_checked_pre_function_globals = True

	if common.should_add_line_to_output(
	input_line, prefix_set, not is_in_function
	):
	# This input line of the function body will go as-is into the output.
	# Except make leading whitespace uniform: 2 spaces.
	input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(
	r" ", input_line
	)
	output_lines.append(input_line)
	if input_line.strip() == "}":
	is_in_function = False
	continue

	if is_in_function:
	continue

	m = common.IR_FUNCTION_RE.match(input_line)
	if not m:
	continue
	func_name = m.group(1)
	if args.function is not None and func_name != args.function:
	# When filtering on a specific function, skip all others.
	continue
	is_in_function = is_in_function_start = True

	if args.check_globals:
	generated_prefixes.extend(
	common.add_global_checks(
	builder.global_var_dict(),
	";",
	prefix_list,
	output_lines,
	global_vars_seen_dict,
	args.preserve_names,
	False,
	)
	)
	if ti.args.gen_unused_prefix_body:
	output_lines.extend(
	ti.get_checks_for_unused_prefixes(prefix_list, generated_prefixes)
	)
	common.debug("Writing %d lines to %s..." % (len(output_lines), ti.path))

	with open(ti.path, "wb") as f:
	f.writelines(["{}\n".format(l).encode("utf-8") for l in output_lines])


	if __name__ == "__main__":
	main()