.ci/generate_test_report_lib.py - llvm-project - Git at Google

 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 """Library to parse JUnit XML files and return a markdown report."""

 from typing import TypedDict
 import platform

 from junitparser import JUnitXml, Failure


 # This data structure should match the definition in llvm-zorg in
 # premerge/advisor/advisor_lib.py
 class FailureExplanation(TypedDict):
     name: str
     explained: bool
     reason: str | None


 SEE_BUILD_FILE_STR = "Download the build's log file to see the details."
 UNRELATED_FAILURES_STR = (
     "If these failures are unrelated to your changes (for example "
     "tests are broken or flaky at HEAD), please open an issue at "
     "https://github.com/llvm/llvm-project/issues and add the "
     "`infrastructure` label."
 )
 # The maximum number of lines to pull from a ninja failure.
 NINJA_LOG_SIZE_THRESHOLD = 500


 def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
     """Parses an individual ninja log."""
     failures = []
     index = 0
     while index < len(ninja_log):
         while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
             index += 1
         if index == len(ninja_log):
             # We hit the end of the log without finding a build failure, go to
             # the next log.
             return failures
         # If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested
         # ninja invocations. The sub-ninja will print that a subcommand failed,
         # and then the outer ninja will list the command that failed. We should
         # ignore the outer failure.
         if ninja_log[index - 1].startswith("ninja: build stopped:"):
             index += 1
             continue
         # We are trying to parse cases like the following:
         #
         # [4/5] test/4.stamp
         # FAILED: touch test/4.stamp
         # touch test/4.stamp
         #
         # index will point to the line that starts with Failed:. The progress
         # indicator is sometimes the line before this ([4/5] test/4.stamp) and
         # will contain a pretty printed version of the target being built
         # (test/4.stamp) when accurate. We instead parse the failed line rather
         # than the progress indicator as the progress indicator may not be
         # aligned with the failure.
         failing_action = ninja_log[index].split("FAILED: ")[1]
         failure_log = []

         # Parse the lines above the FAILED: string if the line does not come
         # immediately after a progress indicator to ensure that we capture the
         # entire failure message.
         if not ninja_log[index - 1].startswith("["):
             before_index = index - 1
             while before_index > 0 and not ninja_log[before_index].startswith("["):
                 failure_log.append(ninja_log[before_index])
                 before_index = before_index - 1
             failure_log.reverse()

         # Parse the failure information, which comes after the FAILED: tag.
         while (
             index < len(ninja_log)
             and not ninja_log[index].startswith("[")
             and not ninja_log[index].startswith("ninja: build stopped:")
             and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
         ):
             failure_log.append(ninja_log[index])
             index += 1
         failures.append((failing_action, "\n".join(failure_log)))
     return failures


 def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
     """Extracts failure messages from ninja output.

     This function takes stdout/stderr from ninja in the form of a list of files
     represented as a list of lines. This function then returns tuples containing
     the name of the target and the error message.

     Args:
       ninja_logs: A list of files in the form of a list of lines representing the log
         files captured from ninja.

     Returns:
       A list of tuples. The first string is the name of the target that failed. The
       second string is the error message.
     """
     failures = []
     for ninja_log in ninja_logs:
         log_failures = _parse_ninja_log(ninja_log)
         failures.extend(log_failures)
     return failures


 def _format_failures(
     failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]
 ) -> list[str]:
     """Formats failures into summary views for the report."""
     output = []
     for build_failure in failures:
         failed_action, failure_message = build_failure
         failure_explanation = None
         if failed_action in failure_explanations:
             failure_explanation = failure_explanations[failed_action]
         output.append("<details>")
         if failure_explanation:
             output.extend(
                 [
                     f"<summary>{failed_action} (Likely Already Failing)</summary>" "",
                     failure_explanation["reason"],
                     "",
                 ]
             )
         else:
             output.extend([f"<summary>{failed_action}</summary>", ""])
         output.extend(
             [
                 "```",
                 failure_message,
                 "```",
                 "</details>",
             ]
         )
     return output


 def get_failures(junit_objects) -> dict[str, list[tuple[str, str]]]:
     failures = {}
     for results in junit_objects:
         for testsuite in results:
             for test in testsuite:
                 if (
                     not test.is_passed
                     and test.result
                     and isinstance(test.result[0], Failure)
                 ):
                     if failures.get(testsuite.name) is None:
                         failures[testsuite.name] = []
                     failures[testsuite.name].append(
                         (test.classname + "/" + test.name, test.result[0].text)
                     )
     return failures


 def are_all_failures_explained(
     failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]
 ) -> bool:
     for failed_action, _ in failures:
         if failed_action not in failure_explanations:
             return False
         else:
             assert failure_explanations[failed_action]["explained"]
     return True


 # Set size_limit to limit the byte size of the report. The default is 1MB as this
 # is the most that can be put into an annotation. If the generated report exceeds
 # this limit and failures are listed, it will be generated again without failures
 # listed. This minimal report will always fit into an annotation.
 # If include failures is False, total number of test will be reported but their names
 # and output will not be.
 def generate_report(
     title,
     return_code,
     junit_objects,
     ninja_logs: list[list[str]],
     size_limit=1024 * 1024,
     list_failures=True,
     failure_explanations_list: list[FailureExplanation] = [],
 ) -> tuple[str, bool]:
     failures = get_failures(junit_objects)
     tests_run = 0
     tests_skipped = 0
     tests_failed = 0

     failure_explanations: dict[str, FailureExplanation] = {}
     for failure_explanation in failure_explanations_list:
         if not failure_explanation["explained"]:
             continue
         failure_explanations[failure_explanation["name"]] = failure_explanation
     all_failures_explained = True
     if failures:
         for _, failures_list in failures.items():
             all_failures_explained &= are_all_failures_explained(
                 failures_list, failure_explanations
             )

     for results in junit_objects:
         for testsuite in results:
             tests_run += testsuite.tests
             tests_skipped += testsuite.skipped
             tests_failed += testsuite.failures

     report = [f"# {title}", ""]

     if tests_run == 0:
         if return_code == 0:
             report.extend(
                 [
                     ":white_check_mark: The build succeeded and no tests ran. "
                     "This is expected in some build configurations."
                 ]
             )
         else:
             ninja_failures = find_failure_in_ninja_logs(ninja_logs)
             all_failures_explained &= are_all_failures_explained(
                 ninja_failures, failure_explanations
             )
             if not ninja_failures:
                 all_failures_explained = False
                 report.extend(
                     [
                         "The build failed before running any tests. Detailed "
                         "information about the build failure could not be "
                         "automatically obtained.",
                         "",
                         SEE_BUILD_FILE_STR,
                         "",
                         UNRELATED_FAILURES_STR,
                     ]
                 )
             else:
                 report.extend(
                     [
                         "The build failed before running any tests. Click on a "
                         "failure below to see the details.",
                         "",
                     ]
                 )
                 report.extend(_format_failures(ninja_failures, failure_explanations))
                 report.extend(
                     [
                         "",
                         UNRELATED_FAILURES_STR,
                     ]
                 )
         return ("\n".join(report), all_failures_explained)

     tests_passed = tests_run - tests_skipped - tests_failed

     def plural(num_tests):
         return "test" if num_tests == 1 else "tests"

     if tests_passed:
         report.append(f"* {tests_passed} {plural(tests_passed)} passed")
     if tests_skipped:
         report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
     if tests_failed:
         report.append(f"* {tests_failed} {plural(tests_failed)} failed")

     if not list_failures:
         report.extend(
             [
                 "",
                 "Failed tests and their output was too large to report. "
                 + SEE_BUILD_FILE_STR,
             ]
         )
     elif failures:
         report.extend(
             ["", "## Failed Tests", "(click on a test name to see its output)"]
         )

         for testsuite_name, failures in failures.items():
             report.extend(["", f"### {testsuite_name}"])
             report.extend(_format_failures(failures, failure_explanations))
     elif return_code != 0:
         # No tests failed but the build was in a failed state. Bring this to the user's
         # attention.
         ninja_failures = find_failure_in_ninja_logs(ninja_logs)
         if not ninja_failures:
             all_failures_explained = False
             report.extend(
                 [
                     "",
                     "All executed tests passed, but another part of the build **failed**. "
                     "Information about the build failure could not be automatically "
                     "obtained.",
                     "",
                     SEE_BUILD_FILE_STR,
                 ]
             )
         else:
             all_failures_explained &= are_all_failures_explained(
                 ninja_failures, failure_explanations
             )
             report.extend(
                 [
                     "",
                     "All executed tests passed, but another part of the build **failed**. Click on "
                     "a failure below to see the details.",
                     "",
                 ]
             )
             report.extend(_format_failures(ninja_failures, failure_explanations))
     else:
         report.extend(
             ["", ":white_check_mark: The build succeeded and all tests passed."]
         )

     if failures or return_code != 0:
         report.extend(["", UNRELATED_FAILURES_STR])

     report = "\n".join(report)
     if len(report.encode("utf-8")) > size_limit:
         return generate_report(
             title,
             return_code,
             junit_objects,
             ninja_logs,
             size_limit,
             list_failures=False,
         )

     return (report, all_failures_explained)


 def load_info_from_files(build_log_files):
     junit_files = [
         junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
     ]
     ninja_log_files = [
         ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
     ]
     ninja_logs = []
     for ninja_log_file in ninja_log_files:
         with open(ninja_log_file, "r") as ninja_log_file_handle:
             ninja_logs.append(
                 [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
             )
     return [JUnitXml.fromfile(p) for p in junit_files], ninja_logs


 def generate_report_from_files(title, return_code, build_log_files):
     junit_objects, ninja_logs = load_info_from_files(build_log_files)
     return generate_report(title, return_code, junit_objects, ninja_logs)


 def compute_platform_title() -> str:
     logo = ":window:" if platform.system() == "Windows" else ":penguin:"
     # On Linux the machine value is x86_64 on Windows it is AMD64.
     if platform.machine() == "x86_64" or platform.machine() == "AMD64":
         arch = "x64"
     else:
         arch = platform.machine()
     return f"{logo} {platform.system()} {arch} Test Results"
	# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	# See https://llvm.org/LICENSE.txt for license information.
	# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	"""Library to parse JUnit XML files and return a markdown report."""

	from typing import TypedDict
	import platform

	from junitparser import JUnitXml, Failure


	# This data structure should match the definition in llvm-zorg in
	# premerge/advisor/advisor_lib.py
	class FailureExplanation(TypedDict):
	name: str
	explained: bool
	reason: str \| None


	SEE_BUILD_FILE_STR = "Download the build's log file to see the details."
	UNRELATED_FAILURES_STR = (
	"If these failures are unrelated to your changes (for example "
	"tests are broken or flaky at HEAD), please open an issue at "
	"https://github.com/llvm/llvm-project/issues and add the "
	"`infrastructure` label."
	)
	# The maximum number of lines to pull from a ninja failure.
	NINJA_LOG_SIZE_THRESHOLD = 500


	def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
	"""Parses an individual ninja log."""
	failures = []
	index = 0
	while index < len(ninja_log):
	while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
	index += 1
	if index == len(ninja_log):
	# We hit the end of the log without finding a build failure, go to
	# the next log.
	return failures
	# If we are doing a build with LLVM_ENABLE_RUNTIMES, we can have nested
	# ninja invocations. The sub-ninja will print that a subcommand failed,
	# and then the outer ninja will list the command that failed. We should
	# ignore the outer failure.
	if ninja_log[index - 1].startswith("ninja: build stopped:"):
	index += 1
	continue
	# We are trying to parse cases like the following:
	#
	# [4/5] test/4.stamp
	# FAILED: touch test/4.stamp
	# touch test/4.stamp
	#
	# index will point to the line that starts with Failed:. The progress
	# indicator is sometimes the line before this ([4/5] test/4.stamp) and
	# will contain a pretty printed version of the target being built
	# (test/4.stamp) when accurate. We instead parse the failed line rather
	# than the progress indicator as the progress indicator may not be
	# aligned with the failure.
	failing_action = ninja_log[index].split("FAILED: ")[1]
	failure_log = []

	# Parse the lines above the FAILED: string if the line does not come
	# immediately after a progress indicator to ensure that we capture the
	# entire failure message.
	if not ninja_log[index - 1].startswith("["):
	before_index = index - 1
	while before_index > 0 and not ninja_log[before_index].startswith("["):
	failure_log.append(ninja_log[before_index])
	before_index = before_index - 1
	failure_log.reverse()

	# Parse the failure information, which comes after the FAILED: tag.
	while (
	index < len(ninja_log)
	and not ninja_log[index].startswith("[")
	and not ninja_log[index].startswith("ninja: build stopped:")
	and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
	):
	failure_log.append(ninja_log[index])
	index += 1
	failures.append((failing_action, "\n".join(failure_log)))
	return failures


	def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
	"""Extracts failure messages from ninja output.

	This function takes stdout/stderr from ninja in the form of a list of files
	represented as a list of lines. This function then returns tuples containing
	the name of the target and the error message.

	Args:
	ninja_logs: A list of files in the form of a list of lines representing the log
	files captured from ninja.

	Returns:
	A list of tuples. The first string is the name of the target that failed. The
	second string is the error message.
	"""
	failures = []
	for ninja_log in ninja_logs:
	log_failures = _parse_ninja_log(ninja_log)
	failures.extend(log_failures)
	return failures


	def _format_failures(
	failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]
	) -> list[str]:
	"""Formats failures into summary views for the report."""
	output = []
	for build_failure in failures:
	failed_action, failure_message = build_failure
	failure_explanation = None
	if failed_action in failure_explanations:
	failure_explanation = failure_explanations[failed_action]
	output.append("<details>")
	if failure_explanation:
	output.extend(
	[
	f"<summary>{failed_action} (Likely Already Failing)</summary>" "",
	failure_explanation["reason"],
	"",
	]
	)
	else:
	output.extend([f"<summary>{failed_action}</summary>", ""])
	output.extend(
	[
	"```",
	failure_message,
	"```",
	"</details>",
	]
	)
	return output


	def get_failures(junit_objects) -> dict[str, list[tuple[str, str]]]:
	failures = {}
	for results in junit_objects:
	for testsuite in results:
	for test in testsuite:
	if (
	not test.is_passed
	and test.result
	and isinstance(test.result[0], Failure)
	):
	if failures.get(testsuite.name) is None:
	failures[testsuite.name] = []
	failures[testsuite.name].append(
	(test.classname + "/" + test.name, test.result[0].text)
	)
	return failures


	def are_all_failures_explained(
	failures: list[tuple[str, str]], failure_explanations: dict[str, FailureExplanation]
	) -> bool:
	for failed_action, _ in failures:
	if failed_action not in failure_explanations:
	return False
	else:
	assert failure_explanations[failed_action]["explained"]
	return True


	# Set size_limit to limit the byte size of the report. The default is 1MB as this
	# is the most that can be put into an annotation. If the generated report exceeds
	# this limit and failures are listed, it will be generated again without failures
	# listed. This minimal report will always fit into an annotation.
	# If include failures is False, total number of test will be reported but their names
	# and output will not be.
	def generate_report(
	title,
	return_code,
	junit_objects,
	ninja_logs: list[list[str]],
	size_limit=1024 * 1024,
	list_failures=True,
	failure_explanations_list: list[FailureExplanation] = [],
	) -> tuple[str, bool]:
	failures = get_failures(junit_objects)
	tests_run = 0
	tests_skipped = 0
	tests_failed = 0

	failure_explanations: dict[str, FailureExplanation] = {}
	for failure_explanation in failure_explanations_list:
	if not failure_explanation["explained"]:
	continue
	failure_explanations[failure_explanation["name"]] = failure_explanation
	all_failures_explained = True
	if failures:
	for _, failures_list in failures.items():
	all_failures_explained &= are_all_failures_explained(
	failures_list, failure_explanations
	)

	for results in junit_objects:
	for testsuite in results:
	tests_run += testsuite.tests
	tests_skipped += testsuite.skipped
	tests_failed += testsuite.failures

	report = [f"# {title}", ""]

	if tests_run == 0:
	if return_code == 0:
	report.extend(
	[
	":white_check_mark: The build succeeded and no tests ran. "
	"This is expected in some build configurations."
	]
	)
	else:
	ninja_failures = find_failure_in_ninja_logs(ninja_logs)
	all_failures_explained &= are_all_failures_explained(
	ninja_failures, failure_explanations
	)
	if not ninja_failures:
	all_failures_explained = False
	report.extend(
	[
	"The build failed before running any tests. Detailed "
	"information about the build failure could not be "
	"automatically obtained.",
	"",
	SEE_BUILD_FILE_STR,
	"",
	UNRELATED_FAILURES_STR,
	]
	)
	else:
	report.extend(
	[
	"The build failed before running any tests. Click on a "
	"failure below to see the details.",
	"",
	]
	)
	report.extend(_format_failures(ninja_failures, failure_explanations))
	report.extend(
	[
	"",
	UNRELATED_FAILURES_STR,
	]
	)
	return ("\n".join(report), all_failures_explained)

	tests_passed = tests_run - tests_skipped - tests_failed

	def plural(num_tests):
	return "test" if num_tests == 1 else "tests"

	if tests_passed:
	report.append(f"* {tests_passed} {plural(tests_passed)} passed")
	if tests_skipped:
	report.append(f"* {tests_skipped} {plural(tests_skipped)} skipped")
	if tests_failed:
	report.append(f"* {tests_failed} {plural(tests_failed)} failed")

	if not list_failures:
	report.extend(
	[
	"",
	"Failed tests and their output was too large to report. "
	+ SEE_BUILD_FILE_STR,
	]
	)
	elif failures:
	report.extend(
	["", "## Failed Tests", "(click on a test name to see its output)"]
	)

	for testsuite_name, failures in failures.items():
	report.extend(["", f"### {testsuite_name}"])
	report.extend(_format_failures(failures, failure_explanations))
	elif return_code != 0:
	# No tests failed but the build was in a failed state. Bring this to the user's
	# attention.
	ninja_failures = find_failure_in_ninja_logs(ninja_logs)
	if not ninja_failures:
	all_failures_explained = False
	report.extend(
	[
	"",
	"All executed tests passed, but another part of the build failed. "
	"Information about the build failure could not be automatically "
	"obtained.",
	"",
	SEE_BUILD_FILE_STR,
	]
	)
	else:
	all_failures_explained &= are_all_failures_explained(
	ninja_failures, failure_explanations
	)
	report.extend(
	[
	"",
	"All executed tests passed, but another part of the build failed. Click on "
	"a failure below to see the details.",
	"",
	]
	)
	report.extend(_format_failures(ninja_failures, failure_explanations))
	else:
	report.extend(
	["", ":white_check_mark: The build succeeded and all tests passed."]
	)

	if failures or return_code != 0:
	report.extend(["", UNRELATED_FAILURES_STR])

	report = "\n".join(report)
	if len(report.encode("utf-8")) > size_limit:
	return generate_report(
	title,
	return_code,
	junit_objects,
	ninja_logs,
	size_limit,
	list_failures=False,
	)

	return (report, all_failures_explained)


	def load_info_from_files(build_log_files):
	junit_files = [
	junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
	]
	ninja_log_files = [
	ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
	]
	ninja_logs = []
	for ninja_log_file in ninja_log_files:
	with open(ninja_log_file, "r") as ninja_log_file_handle:
	ninja_logs.append(
	[log_line.strip() for log_line in ninja_log_file_handle.readlines()]
	)
	return [JUnitXml.fromfile(p) for p in junit_files], ninja_logs


	def generate_report_from_files(title, return_code, build_log_files):
	junit_objects, ninja_logs = load_info_from_files(build_log_files)
	return generate_report(title, return_code, junit_objects, ninja_logs)


	def compute_platform_title() -> str:
	logo = ":window:" if platform.system() == "Windows" else ":penguin:"
	# On Linux the machine value is x86_64 on Windows it is AMD64.
	if platform.machine() == "x86_64" or platform.machine() == "AMD64":
	arch = "x64"
	else:
	arch = platform.machine()
	return f"{logo} {platform.system()} {arch} Test Results"