[libc++] Improve error resilience when running historical benchmarks In benchmark-historical, don't skip gathering the results when the lit command has failed. Indeed, it is expected to fail as part of normal operation since it should be pretty frequent for at least one test to fail when running historical benchmarks. Instead, gather whatever results we have. Also, output the build log in spec.gen.py upon failure so we can see the reason for the failure.
diff --git a/libcxx/test/benchmarks/spec.gen.py b/libcxx/test/benchmarks/spec.gen.py index b4b31d4..ea7b75b 100644 --- a/libcxx/test/benchmarks/spec.gen.py +++ b/libcxx/test/benchmarks/spec.gen.py
@@ -74,5 +74,5 @@ # Parse the results into a LNT-compatible format. This also errors out if there are no CSV files, which # means that the benchmark didn't run properly (the `runcpu` command above never reports a failure). - print(f'RUN: %{{libcxx-dir}}/utils/parse-spec-results %T/result/*.train.csv --output-format=lnt > %T/results.lnt') + print(f'RUN: %{{libcxx-dir}}/utils/parse-spec-results %T/result/*.train.csv --output-format=lnt > %T/results.lnt || ! cat %T/result/*.log') print(f'RUN: cat %T/results.lnt')
diff --git a/libcxx/utils/benchmark-historical b/libcxx/utils/benchmark-historical index 448d7bd..4d8d65b 100755 --- a/libcxx/utils/benchmark-historical +++ b/libcxx/utils/benchmark-historical
@@ -90,10 +90,7 @@ logging.info(f'Running {pretty}') continue - if subprocess.call(test_cmd) != 0: - logging.error(f'Failed to run the tests for commit {commit}, skipping') - continue - + subprocess.call(test_cmd) output_file.parent.mkdir(parents=True, exist_ok=True) consolidate_cmd = [(PARENT_DIR / 'consolidate-benchmarks'), build_dir, '--output', output_file] subprocess.check_call(consolidate_cmd)