Speeding up llvm-cov export with multithreaded renderFiles implementation.
Summary:
CoverageExporterJson::renderFiles accounts for most of the execution time given a large profdata file with multiple binaries.
Proposed solution is to generate JSON for each file in parallel and sort at the end to preserve deterministic output. Also added flags to skip generating parts of the output to trim the output size.
Patch by Sajjad Mirza (@sajjadm).
Reviewers: Dor1s, vsk
Reviewed By: Dor1s, vsk
Subscribers: liaoyuke, mgrang, jdoerfert, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D59277
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@356178 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/docs/CommandGuide/llvm-cov.rst b/docs/CommandGuide/llvm-cov.rst
index 6e90760..7cc7d3f 100644
--- a/docs/CommandGuide/llvm-cov.rst
+++ b/docs/CommandGuide/llvm-cov.rst
@@ -419,3 +419,16 @@
.. option:: -ignore-filename-regex=<PATTERN>
Skip source code files with file paths that match the given regular expression.
+
+ .. option:: -skip-expansions
+
+ Skip exporting macro expansion coverage data.
+
+ .. option:: -skip-functions
+
+ Skip exporting per-function coverage data.
+
+ .. option:: -num-threads=N, -j=N
+
+ Use N threads to export coverage data. When N=0, llvm-cov auto-detects an
+ appropriate number of threads to use. This is the default.
diff --git a/test/tools/llvm-cov/export_functions.test b/test/tools/llvm-cov/export_functions.test
new file mode 100644
index 0000000..03f355f
--- /dev/null
+++ b/test/tools/llvm-cov/export_functions.test
@@ -0,0 +1,10 @@
+# Test that llvm-cov export produces function data by default and that it can be
+# turned off with a flag.
+
+RUN: llvm-cov export %S/Inputs/report.covmapping -instr-profile %S/Inputs/report.profdata 2>&1 | FileCheck %s
+RUN: llvm-cov export %S/Inputs/report.covmapping -instr-profile %S/Inputs/report.profdata -skip-functions 2>&1 | FileCheck -check-prefix=SKIP-FUNCTIONS %s
+
+CHECK: "functions":[
+SKIP-FUNCTIONS-NOT: "functions":[
+
+
diff --git a/test/tools/llvm-cov/showExpansions.cpp b/test/tools/llvm-cov/showExpansions.cpp
index f9d63e9..5b165da 100644
--- a/test/tools/llvm-cov/showExpansions.cpp
+++ b/test/tools/llvm-cov/showExpansions.cpp
@@ -25,3 +25,6 @@
return 0;
}
// RUN: llvm-cov export %S/Inputs/showExpansions.covmapping -instr-profile %S/Inputs/showExpansions.profdata 2>&1 | FileCheck %S/Inputs/showExpansions.json
+
+// RUN: llvm-cov export %S/Inputs/showExpansions.covmapping -instr-profile %S/Inputs/showExpansions.profdata -skip-expansions 2>&1 | FileCheck %s -check-prefix=SKIP-EXPANSIONS
+// SKIP-EXPANSIONS-NOT: "expansions"
diff --git a/tools/llvm-cov/CodeCoverage.cpp b/tools/llvm-cov/CodeCoverage.cpp
index 43311a4..f707e3c 100644
--- a/tools/llvm-cov/CodeCoverage.cpp
+++ b/tools/llvm-cov/CodeCoverage.cpp
@@ -1006,10 +1006,23 @@
int CodeCoverageTool::doExport(int argc, const char **argv,
CommandLineParserType commandLineParser) {
+ cl::OptionCategory ExportCategory("Exporting options");
+
+ cl::opt<bool> SkipExpansions("skip-expansions", cl::Optional,
+ cl::desc("Don't export expanded source regions"),
+ cl::cat(ExportCategory));
+
+ cl::opt<bool> SkipFunctions("skip-functions", cl::Optional,
+ cl::desc("Don't export per-function data"),
+ cl::cat(ExportCategory));
+
auto Err = commandLineParser(argc, argv);
if (Err)
return Err;
+ ViewOpts.SkipExpansions = SkipExpansions;
+ ViewOpts.SkipFunctions = SkipFunctions;
+
if (ViewOpts.Format != CoverageViewOptions::OutputFormat::Text &&
ViewOpts.Format != CoverageViewOptions::OutputFormat::Lcov) {
error("Coverage data can only be exported as textual JSON or an "
diff --git a/tools/llvm-cov/CoverageExporterJson.cpp b/tools/llvm-cov/CoverageExporterJson.cpp
index e6361d4..181d428 100644
--- a/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/tools/llvm-cov/CoverageExporterJson.cpp
@@ -42,7 +42,14 @@
#include "CoverageExporterJson.h"
#include "CoverageReport.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/Support/JSON.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/Threading.h"
+#include <algorithm>
+#include <mutex>
+#include <utility>
/// The semantic version combined as a string.
#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.0"
@@ -127,13 +134,15 @@
json::Object renderFile(const coverage::CoverageMapping &Coverage,
const std::string &Filename,
const FileCoverageSummary &FileReport,
- bool ExportSummaryOnly) {
+ const CoverageViewOptions &Options) {
json::Object File({{"filename", Filename}});
- if (!ExportSummaryOnly) {
+ if (!Options.ExportSummaryOnly) {
// Calculate and render detailed coverage information for given file.
auto FileCoverage = Coverage.getCoverageForFile(Filename);
File["segments"] = renderFileSegments(FileCoverage, FileReport);
- File["expansions"] = renderFileExpansions(FileCoverage, FileReport);
+ if (!Options.SkipExpansions) {
+ File["expansions"] = renderFileExpansions(FileCoverage, FileReport);
+ }
}
File["summary"] = renderSummary(FileReport);
return File;
@@ -142,11 +151,28 @@
json::Array renderFiles(const coverage::CoverageMapping &Coverage,
ArrayRef<std::string> SourceFiles,
ArrayRef<FileCoverageSummary> FileReports,
- bool ExportSummaryOnly) {
+ const CoverageViewOptions &Options) {
+ auto NumThreads = Options.NumThreads;
+ if (NumThreads == 0) {
+ NumThreads = std::max(1U, std::min(llvm::heavyweight_hardware_concurrency(),
+ unsigned(SourceFiles.size())));
+ }
+ ThreadPool Pool(NumThreads);
json::Array FileArray;
- for (unsigned I = 0, E = SourceFiles.size(); I < E; ++I)
- FileArray.push_back(renderFile(Coverage, SourceFiles[I], FileReports[I],
- ExportSummaryOnly));
+ std::mutex FileArrayMutex;
+
+ for (unsigned I = 0, E = SourceFiles.size(); I < E; ++I) {
+ auto &SourceFile = SourceFiles[I];
+ auto &FileReport = FileReports[I];
+ Pool.async([&] {
+ auto File = renderFile(Coverage, SourceFile, FileReport, Options);
+ {
+ std::lock_guard<std::mutex> Lock(FileArrayMutex);
+ FileArray.push_back(std::move(File));
+ }
+ });
+ }
+ Pool.wait();
return FileArray;
}
@@ -177,12 +203,22 @@
FileCoverageSummary Totals = FileCoverageSummary("Totals");
auto FileReports = CoverageReport::prepareFileReports(Coverage, Totals,
SourceFiles, Options);
- auto Export =
- json::Object({{"files", renderFiles(Coverage, SourceFiles, FileReports,
- Options.ExportSummaryOnly)},
- {"totals", renderSummary(Totals)}});
- // Skip functions-level information for summary-only export mode.
- if (!Options.ExportSummaryOnly)
+ auto Files = renderFiles(Coverage, SourceFiles, FileReports, Options);
+ // Sort files in order of their names.
+ std::sort(Files.begin(), Files.end(),
+ [](const json::Value &A, const json::Value &B) {
+ const json::Object *ObjA = A.getAsObject();
+ const json::Object *ObjB = B.getAsObject();
+ assert(ObjA != nullptr && "Value A was not an Object");
+ assert(ObjB != nullptr && "Value B was not an Object");
+ const StringRef FilenameA = ObjA->getString("filename").getValue();
+ const StringRef FilenameB = ObjB->getString("filename").getValue();
+ return FilenameA.compare(FilenameB) < 0;
+ });
+ auto Export = json::Object(
+ {{"files", std::move(Files)}, {"totals", renderSummary(Totals)}});
+ // Skip functions-level information if necessary.
+ if (!Options.ExportSummaryOnly && !Options.SkipFunctions)
Export["functions"] = renderFunctions(Coverage.getCoveredFunctions());
auto ExportArray = json::Array({std::move(Export)});
diff --git a/tools/llvm-cov/CoverageViewOptions.h b/tools/llvm-cov/CoverageViewOptions.h
index 0a3e211..dde0c69 100644
--- a/tools/llvm-cov/CoverageViewOptions.h
+++ b/tools/llvm-cov/CoverageViewOptions.h
@@ -34,6 +34,8 @@
bool ShowRegionSummary;
bool ShowInstantiationSummary;
bool ExportSummaryOnly;
+ bool SkipExpansions;
+ bool SkipFunctions;
OutputFormat Format;
std::string ShowOutputDirectory;
std::vector<std::string> DemanglerOpts;
