| //===- DependencyScannerImpl.cpp - Implements module dependency scanning --===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "clang/DependencyScanning/DependencyScannerImpl.h" | |
| #include "clang/Basic/DiagnosticFrontend.h" | |
| #include "clang/Basic/DiagnosticSerialization.h" | |
| #include "clang/DependencyScanning/DependencyScanningFilesystem.h" | |
| #include "clang/DependencyScanning/DependencyScanningService.h" | |
| #include "clang/DependencyScanning/DependencyScanningWorker.h" | |
| #include "clang/Frontend/FrontendActions.h" | |
| #include "llvm/ADT/IntrusiveRefCntPtr.h" | |
| #include "llvm/ADT/ScopeExit.h" | |
| #include "llvm/Option/Option.h" | |
| #include "llvm/Support/AdvisoryLock.h" | |
| #include "llvm/Support/CrashRecoveryContext.h" | |
| #include "llvm/Support/VirtualFileSystem.h" | |
| #include "llvm/TargetParser/Host.h" | |
| #include <mutex> | |
| #include <thread> | |
| using namespace clang; | |
| using namespace dependencies; | |
| namespace { | |
| /// Forwards the gatherered dependencies to the consumer. | |
| class DependencyConsumerForwarder : public DependencyFileGenerator { | |
| public: | |
| DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts, | |
| StringRef WorkingDirectory, DependencyConsumer &C) | |
| : DependencyFileGenerator(*Opts), WorkingDirectory(WorkingDirectory), | |
| Opts(std::move(Opts)), C(C) {} | |
| void finishedMainFile(DiagnosticsEngine &Diags) override { | |
| C.handleDependencyOutputOpts(*Opts); | |
| llvm::SmallString<256> CanonPath; | |
| for (const auto &File : getDependencies()) { | |
| CanonPath = File; | |
| llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true); | |
| llvm::sys::path::make_absolute(WorkingDirectory, CanonPath); | |
| C.handleFileDependency(CanonPath); | |
| } | |
| } | |
| private: | |
| StringRef WorkingDirectory; | |
| std::unique_ptr<DependencyOutputOptions> Opts; | |
| DependencyConsumer &C; | |
| }; | |
| static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts, | |
| const HeaderSearchOptions &ExistingHSOpts, | |
| DiagnosticsEngine *Diags, | |
| const LangOptions &LangOpts) { | |
| if (LangOpts.Modules) { | |
| if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { | |
| if (Diags) { | |
| Diags->Report(diag::warn_pch_vfsoverlay_mismatch); | |
| auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) { | |
| if (VFSOverlays.empty()) { | |
| Diags->Report(diag::note_pch_vfsoverlay_empty) << Type; | |
| } else { | |
| std::string Files = llvm::join(VFSOverlays, "\n"); | |
| Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files; | |
| } | |
| }; | |
| VFSNote(0, HSOpts.VFSOverlayFiles); | |
| VFSNote(1, ExistingHSOpts.VFSOverlayFiles); | |
| } | |
| } | |
| } | |
| return false; | |
| } | |
| using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); | |
| /// A listener that collects the imported modules and the input | |
| /// files. While visiting, collect vfsoverlays and file inputs that determine | |
| /// whether prebuilt modules fully resolve in stable directories. | |
| class PrebuiltModuleListener : public ASTReaderListener { | |
| public: | |
| PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, | |
| llvm::SmallVector<std::string> &NewModuleFiles, | |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, | |
| const HeaderSearchOptions &HSOpts, | |
| const LangOptions &LangOpts, DiagnosticsEngine &Diags, | |
| const ArrayRef<StringRef> StableDirs) | |
| : PrebuiltModuleFiles(PrebuiltModuleFiles), | |
| NewModuleFiles(NewModuleFiles), | |
| PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts), | |
| ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {} | |
| bool needsImportVisitation() const override { return true; } | |
| bool needsInputFileVisitation() override { return true; } | |
| bool needsSystemInputFileVisitation() override { return true; } | |
| /// Accumulate the modules are transitively depended on by the initial | |
| /// prebuilt module. | |
| void visitImport(StringRef ModuleName, StringRef Filename) override { | |
| if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) | |
| NewModuleFiles.push_back(Filename.str()); | |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename); | |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; | |
| if (PrebuiltMapEntry.second) | |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); | |
| if (auto It = PrebuiltModulesASTMap.find(CurrentFile); | |
| It != PrebuiltModulesASTMap.end() && CurrentFile != Filename) | |
| PrebuiltModule.addDependent(It->getKey()); | |
| } | |
| /// For each input file discovered, check whether it's external path is in a | |
| /// stable directory. Traversal is stopped if the current module is not | |
| /// considered stable. | |
| bool visitInputFileAsRequested(StringRef FilenameAsRequested, | |
| StringRef Filename, bool isSystem, | |
| bool isOverridden, time_t StoredTime, | |
| bool isExplicitModule) override { | |
| if (StableDirs.empty()) | |
| return false; | |
| auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); | |
| if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) || | |
| (!PrebuiltEntryIt->second.isInStableDir())) | |
| return false; | |
| PrebuiltEntryIt->second.setInStableDir( | |
| isPathInStableDir(StableDirs, Filename)); | |
| return PrebuiltEntryIt->second.isInStableDir(); | |
| } | |
| /// Update which module that is being actively traversed. | |
| void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind, | |
| bool DirectlyImported) override { | |
| // If the CurrentFile is not | |
| // considered stable, update any of it's transitive dependents. | |
| auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); | |
| if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) && | |
| !PrebuiltEntryIt->second.isInStableDir()) | |
| PrebuiltEntryIt->second.updateDependentsNotInStableDirs( | |
| PrebuiltModulesASTMap); | |
| CurrentFile = Filename.str(); | |
| } | |
| /// Check the header search options for a given module when considering | |
| /// if the module comes from stable directories. | |
| bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, | |
| StringRef ModuleFilename, StringRef ContextHash, | |
| bool Complain) override { | |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); | |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; | |
| if (PrebuiltMapEntry.second) | |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); | |
| if (PrebuiltModule.isInStableDir()) | |
| PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts)); | |
| return false; | |
| } | |
| /// Accumulate vfsoverlays used to build these prebuilt modules. | |
| bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, | |
| bool Complain) override { | |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); | |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; | |
| if (PrebuiltMapEntry.second) | |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); | |
| PrebuiltModule.setVFS( | |
| llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles)); | |
| return checkHeaderSearchPaths( | |
| HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts); | |
| } | |
| private: | |
| PrebuiltModuleFilesT &PrebuiltModuleFiles; | |
| llvm::SmallVector<std::string> &NewModuleFiles; | |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap; | |
| const HeaderSearchOptions &ExistingHSOpts; | |
| const LangOptions &ExistingLangOpts; | |
| DiagnosticsEngine &Diags; | |
| std::string CurrentFile; | |
| const ArrayRef<StringRef> StableDirs; | |
| }; | |
| /// Visit the given prebuilt module and collect all of the modules it | |
| /// transitively imports and contributing input files. | |
| static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, | |
| CompilerInstance &CI, | |
| PrebuiltModuleFilesT &ModuleFiles, | |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, | |
| DiagnosticsEngine &Diags, | |
| const ArrayRef<StringRef> StableDirs) { | |
| // List of module files to be processed. | |
| llvm::SmallVector<std::string> Worklist; | |
| PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap, | |
| CI.getHeaderSearchOpts(), CI.getLangOpts(), | |
| Diags, StableDirs); | |
| Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename), | |
| serialization::MK_ExplicitModule, | |
| /*DirectlyImported=*/true); | |
| if (ASTReader::readASTFileControlBlock( | |
| PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(), | |
| CI.getPCHContainerReader(), | |
| /*FindModuleFileExtensions=*/false, Listener, | |
| /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate)) | |
| return true; | |
| while (!Worklist.empty()) { | |
| // FIXME: This is assuming the PCH only refers to explicitly-built modules, | |
| // which technically is not guaranteed. To remove the assumption, we'd need | |
| // to also rework how the module files are handled to the scan, specifically | |
| // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain | |
| // paths to ModuleFileName. | |
| Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()), | |
| serialization::MK_ExplicitModule, | |
| /*DirectlyImported=*/false); | |
| if (ASTReader::readASTFileControlBlock( | |
| Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), | |
| CI.getPCHContainerReader(), | |
| /*FindModuleFileExtensions=*/false, Listener, | |
| /*ValidateDiagnosticOptions=*/false)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| /// Transform arbitrary file name into an object-like file name. | |
| static std::string makeObjFileName(StringRef FileName) { | |
| SmallString<128> ObjFileName(FileName); | |
| llvm::sys::path::replace_extension(ObjFileName, "o"); | |
| return std::string(ObjFileName); | |
| } | |
| /// Deduce the dependency target based on the output file and input files. | |
| static std::string | |
| deduceDepTarget(const std::string &OutputFile, | |
| const SmallVectorImpl<FrontendInputFile> &InputFiles) { | |
| if (OutputFile != "-") | |
| return OutputFile; | |
| if (InputFiles.empty() || !InputFiles.front().isFile()) | |
| return "clang-scan-deps\\ dependency"; | |
| return makeObjFileName(InputFiles.front().getFile()); | |
| } | |
| // Clang implements -D and -U by splatting text into a predefines buffer. This | |
| // allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and | |
| // define the same macro, or adding C++ style comments before the macro name. | |
| // | |
| // This function checks that the first non-space characters in the macro | |
| // obviously form an identifier that can be uniqued on without lexing. Failing | |
| // to do this could lead to changing the final definition of a macro. | |
| // | |
| // We could set up a preprocessor and actually lex the name, but that's very | |
| // heavyweight for a situation that will almost never happen in practice. | |
| static std::optional<StringRef> getSimpleMacroName(StringRef Macro) { | |
| StringRef Name = Macro.split("=").first.ltrim(" \t"); | |
| std::size_t I = 0; | |
| auto FinishName = [&]() -> std::optional<StringRef> { | |
| StringRef SimpleName = Name.slice(0, I); | |
| if (SimpleName.empty()) | |
| return std::nullopt; | |
| return SimpleName; | |
| }; | |
| for (; I != Name.size(); ++I) { | |
| switch (Name[I]) { | |
| case '(': // Start of macro parameter list | |
| case ' ': // End of macro name | |
| case '\t': | |
| return FinishName(); | |
| case '_': | |
| continue; | |
| default: | |
| if (llvm::isAlnum(Name[I])) | |
| continue; | |
| return std::nullopt; | |
| } | |
| } | |
| return FinishName(); | |
| } | |
| } // namespace | |
| void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) { | |
| using MacroOpt = std::pair<StringRef, std::size_t>; | |
| std::vector<MacroOpt> SimpleNames; | |
| SimpleNames.reserve(PPOpts.Macros.size()); | |
| std::size_t Index = 0; | |
| for (const auto &M : PPOpts.Macros) { | |
| auto SName = getSimpleMacroName(M.first); | |
| // Skip optimizing if we can't guarantee we can preserve relative order. | |
| if (!SName) | |
| return; | |
| SimpleNames.emplace_back(*SName, Index); | |
| ++Index; | |
| } | |
| llvm::stable_sort(SimpleNames, llvm::less_first()); | |
| // Keep the last instance of each macro name by going in reverse | |
| auto NewEnd = std::unique( | |
| SimpleNames.rbegin(), SimpleNames.rend(), | |
| [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; }); | |
| SimpleNames.erase(SimpleNames.begin(), NewEnd.base()); | |
| // Apply permutation. | |
| decltype(PPOpts.Macros) NewMacros; | |
| NewMacros.reserve(SimpleNames.size()); | |
| for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) { | |
| std::size_t OriginalIndex = SimpleNames[I].second; | |
| // We still emit undefines here as they may be undefining a predefined macro | |
| NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex])); | |
| } | |
| std::swap(PPOpts.Macros, NewMacros); | |
| } | |
| namespace { | |
| class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { | |
| DependencyScanningWorkerFilesystem *DepFS; | |
| public: | |
| ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) { | |
| FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) { | |
| auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS); | |
| if (DFS) { | |
| assert(!DepFS && "Found multiple scanning VFSs"); | |
| DepFS = DFS; | |
| } | |
| }); | |
| assert(DepFS && "Did not find scanning VFS"); | |
| } | |
| std::unique_ptr<DependencyDirectivesGetter> | |
| cloneFor(FileManager &FileMgr) override { | |
| return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr); | |
| } | |
| std::optional<ArrayRef<dependency_directives_scan::Directive>> | |
| operator()(FileEntryRef File) override { | |
| return DepFS->getDirectiveTokens(File.getName()); | |
| } | |
| }; | |
| /// Sanitize diagnostic options for dependency scan. | |
| void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { | |
| // Don't print 'X warnings and Y errors generated'. | |
| DiagOpts.ShowCarets = false; | |
| // Don't write out diagnostic file. | |
| DiagOpts.DiagnosticSerializationFile.clear(); | |
| // Don't emit warnings except for scanning specific warnings. | |
| // TODO: It would be useful to add a more principled way to ignore all | |
| // warnings that come from source code. The issue is that we need to | |
| // ignore warnings that could be surpressed by | |
| // `#pragma clang diagnostic`, while still allowing some scanning | |
| // warnings for things we're not ready to turn into errors yet. | |
| // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example. | |
| llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) { | |
| return llvm::StringSwitch<bool>(Warning) | |
| .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false) | |
| .StartsWith("no-error=", false) | |
| .Default(true); | |
| }); | |
| } | |
| } // namespace | |
| std::unique_ptr<DiagnosticOptions> | |
| dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) { | |
| std::vector<const char *> CLI; | |
| for (const std::string &Arg : CommandLine) | |
| CLI.push_back(Arg.c_str()); | |
| auto DiagOpts = CreateAndPopulateDiagOpts(CLI); | |
| sanitizeDiagOpts(*DiagOpts); | |
| return DiagOpts; | |
| } | |
| DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts( | |
| ArrayRef<std::string> CommandLine, | |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) { | |
| std::vector<const char *> CCommandLine(CommandLine.size(), nullptr); | |
| llvm::transform(CommandLine, CCommandLine.begin(), | |
| [](const std::string &Str) { return Str.c_str(); }); | |
| DiagOpts = CreateAndPopulateDiagOpts(CCommandLine); | |
| sanitizeDiagOpts(*DiagOpts); | |
| DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC, | |
| /*ShouldOwnClient=*/false); | |
| } | |
| std::unique_ptr<CompilerInvocation> | |
| dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine, | |
| DiagnosticsEngine &Diags) { | |
| llvm::opt::ArgStringList Argv; | |
| for (const std::string &Str : ArrayRef(CommandLine).drop_front()) | |
| Argv.push_back(Str.c_str()); | |
| auto Invocation = std::make_unique<CompilerInvocation>(); | |
| if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) { | |
| // FIXME: Should we just go on like cc1_main does? | |
| return nullptr; | |
| } | |
| return Invocation; | |
| } | |
| void dependencies::initializeScanCompilerInstance( | |
| CompilerInstance &ScanInstance, | |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, | |
| DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, | |
| IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) { | |
| ScanInstance.setBuildingModule(false); | |
| ScanInstance.createVirtualFileSystem(FS, DiagConsumer); | |
| ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); | |
| if (Service.getOpts().Format == ScanningOutputFormat::P1689) | |
| ScanInstance.getDiagnostics().setIgnoreAllWarnings(true); | |
| ScanInstance.createFileManager(); | |
| ScanInstance.createSourceManager(); | |
| // Use DepFS for getting the dependency directives if requested to do so. | |
| if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) | |
| ScanInstance.setDependencyDirectivesGetter( | |
| std::make_unique<ScanningDependencyDirectivesGetter>( | |
| ScanInstance.getFileManager())); | |
| } | |
| std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation( | |
| const CompilerInvocation &Invocation, | |
| const DependencyScanningService &Service, | |
| DependencyActionController &Controller) { | |
| auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation); | |
| sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts()); | |
| ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = | |
| true; | |
| if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession) | |
| ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp = | |
| Service.getOpts().BuildSessionTimestamp; | |
| ScanInvocation->getFrontendOpts().DisableFree = false; | |
| ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false; | |
| ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false; | |
| ScanInvocation->getFrontendOpts().GenReducedBMI = false; | |
| ScanInvocation->getFrontendOpts().ModuleOutputPath.clear(); | |
| // This will prevent us compiling individual modules asynchronously since | |
| // FileManager is not thread-safe, but it does improve performance for now. | |
| ScanInvocation->getFrontendOpts().ModulesShareFileManager = true; | |
| ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw"; | |
| ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage = | |
| any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS); | |
| // Consider different header search and diagnostic options to create | |
| // different modules. This avoids the unsound aliasing of module PCMs. | |
| // | |
| // TODO: Implement diagnostic bucketing to reduce the impact of strict | |
| // context hashing. | |
| ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true; | |
| ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; | |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; | |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; | |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = | |
| true; | |
| ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; | |
| // Ensure that the scanner does not create new dependency collectors, | |
| // and thus won't write out the extra '.d' files to disk. | |
| ScanInvocation->getDependencyOutputOpts() = {}; | |
| Controller.initializeScanInvocation(*ScanInvocation); | |
| return ScanInvocation; | |
| } | |
| llvm::SmallVector<StringRef> | |
| dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) { | |
| // Create a collection of stable directories derived from the ScanInstance | |
| // for determining whether module dependencies would fully resolve from | |
| // those directories. | |
| llvm::SmallVector<StringRef> StableDirs; | |
| const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; | |
| if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot)) | |
| StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; | |
| return StableDirs; | |
| } | |
| std::optional<PrebuiltModulesAttrsMap> | |
| dependencies::computePrebuiltModulesASTMap( | |
| CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) { | |
| // Store a mapping of prebuilt module files and their properties like header | |
| // search options. This will prevent the implicit build to create duplicate | |
| // modules and will force reuse of the existing prebuilt module files | |
| // instead. | |
| PrebuiltModulesAttrsMap PrebuiltModulesASTMap; | |
| if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) | |
| if (visitPrebuiltModule( | |
| ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, | |
| ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, | |
| PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs)) | |
| return {}; | |
| return PrebuiltModulesASTMap; | |
| } | |
| std::unique_ptr<DependencyOutputOptions> | |
| dependencies::createDependencyOutputOptions( | |
| const CompilerInvocation &Invocation) { | |
| auto Opts = std::make_unique<DependencyOutputOptions>( | |
| Invocation.getDependencyOutputOpts()); | |
| // We need at least one -MT equivalent for the generator of make dependency | |
| // files to work. | |
| if (Opts->Targets.empty()) | |
| Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile, | |
| Invocation.getFrontendOpts().Inputs)}; | |
| Opts->IncludeSystemHeaders = true; | |
| return Opts; | |
| } | |
| std::shared_ptr<ModuleDepCollector> | |
| dependencies::initializeScanInstanceDependencyCollector( | |
| CompilerInstance &ScanInstance, | |
| std::unique_ptr<DependencyOutputOptions> DepOutputOpts, | |
| StringRef WorkingDirectory, DependencyConsumer &Consumer, | |
| DependencyScanningService &Service, CompilerInvocation &Inv, | |
| DependencyActionController &Controller, | |
| PrebuiltModulesAttrsMap PrebuiltModulesASTMap, | |
| llvm::SmallVector<StringRef> &StableDirs) { | |
| std::shared_ptr<ModuleDepCollector> MDC; | |
| switch (Service.getOpts().Format) { | |
| case ScanningOutputFormat::Make: | |
| ScanInstance.addDependencyCollector( | |
| std::make_shared<DependencyConsumerForwarder>( | |
| std::move(DepOutputOpts), WorkingDirectory, Consumer)); | |
| break; | |
| case ScanningOutputFormat::P1689: | |
| case ScanningOutputFormat::Full: | |
| MDC = std::make_shared<ModuleDepCollector>( | |
| Service, std::move(DepOutputOpts), ScanInstance, Consumer, Controller, | |
| Inv, std::move(PrebuiltModulesASTMap), StableDirs); | |
| ScanInstance.addDependencyCollector(MDC); | |
| break; | |
| } | |
| return MDC; | |
| } | |
| /// Manages (and terminates) the asynchronous compilation of modules. | |
| class AsyncModuleCompiles { | |
| std::mutex Mutex; | |
| bool Stop = false; | |
| // FIXME: Have the service own a thread pool and use that instead. | |
| std::vector<std::thread> Compiles; | |
| public: | |
| /// Registers the module compilation, unless this instance is about to be | |
| /// destroyed. | |
| void add(llvm::unique_function<void()> Compile) { | |
| std::lock_guard<std::mutex> Lock(Mutex); | |
| if (!Stop) | |
| Compiles.emplace_back(std::move(Compile)); | |
| } | |
| ~AsyncModuleCompiles() { | |
| { | |
| // Prevent registration of further module compiles. | |
| std::lock_guard<std::mutex> Lock(Mutex); | |
| Stop = true; | |
| } | |
| // Wait for outstanding module compiles to finish. | |
| for (std::thread &Compile : Compiles) | |
| Compile.join(); | |
| } | |
| }; | |
| struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction { | |
| DependencyScanningService &Service; | |
| DependencyActionController &Controller; | |
| AsyncModuleCompiles &Compiles; | |
| SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service, | |
| DependencyActionController &Controller, | |
| AsyncModuleCompiles &Compiles) | |
| : Service(Service), Controller(Controller), Compiles(Compiles) {} | |
| bool BeginSourceFileAction(CompilerInstance &CI) override; | |
| }; | |
| /// The preprocessor callback that takes care of initiating an asynchronous | |
| /// module compilation if needed. | |
| struct AsyncModuleCompile : PPCallbacks { | |
| CompilerInstance &CI; | |
| DependencyScanningService &Service; | |
| DependencyActionController &Controller; | |
| AsyncModuleCompiles &Compiles; | |
| AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service, | |
| DependencyActionController &Controller, | |
| AsyncModuleCompiles &Compiles) | |
| : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {} | |
| void moduleLoadSkipped(Module *M) override { | |
| M = M->getTopLevelModule(); | |
| HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); | |
| ModuleCache &ModCache = CI.getModuleCache(); | |
| ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M); | |
| uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName); | |
| // Someone else already built/validated the PCM. | |
| if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp) | |
| return; | |
| if (!CI.getASTReader()) | |
| CI.createASTReader(); | |
| SmallVector<ASTReader::ImportedModule, 0> Imported; | |
| // Only calling ReadASTCore() to avoid the expensive eager deserialization | |
| // of the clang::Module objects in ReadAST(). | |
| // FIXME: Consider doing this in the new thread depending on how expensive | |
| // the read turns out to be. | |
| switch (CI.getASTReader()->ReadASTCore( | |
| ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(), | |
| nullptr, Imported, {}, {}, {}, | |
| ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing | | |
| ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) { | |
| case ASTReader::Success: | |
| // We successfully read a valid, up-to-date PCM. | |
| // FIXME: This could update the timestamp. Regular calls to | |
| // ASTReader::ReadAST() would do so unless they encountered corrupted | |
| // AST block, corrupted extension block, or did not read the expected | |
| // top-level module. | |
| return; | |
| case ASTReader::OutOfDate: | |
| case ASTReader::Missing: | |
| // The most interesting case. | |
| break; | |
| default: | |
| // Let the regular scan diagnose this. | |
| return; | |
| } | |
| auto Lock = ModCache.getLock(ModuleFileName); | |
| bool Owned; | |
| llvm::Error LockErr = Lock->tryLock().moveInto(Owned); | |
| // Someone else is building the PCM right now. | |
| if (!LockErr && !Owned) | |
| return; | |
| // We should build the PCM. | |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = | |
| llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>( | |
| Service.getSharedCache(), Service.getOpts().MakeVFS()); | |
| VFS = createVFSFromCompilerInvocation(CI.getInvocation(), | |
| CI.getDiagnostics(), std::move(VFS)); | |
| auto DC = std::make_unique<DiagnosticConsumer>(); | |
| auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries()); | |
| CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC, | |
| std::move(MC)); | |
| auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName, | |
| CloneConfig); | |
| auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName, | |
| CloneConfig); | |
| auto ModController = Controller.clone(); | |
| // Note: This lock belongs to a module cache that might not outlive the | |
| // thread. This works, because the in-process lock only refers to an object | |
| // managed by the service, which does outlive the thread. | |
| Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1), | |
| ModCI2 = std::move(ModCI2), DC = std::move(DC), | |
| ModController = std::move(ModController), Service = &Service, | |
| Compiles = &Compiles] { | |
| llvm::CrashRecoveryContext CRC; | |
| (void)CRC.RunSafely([&] { | |
| // Quickly discovers and compiles modules for the real scan below. | |
| SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController, | |
| *Compiles); | |
| (void)ModCI1->ExecuteAction(Action1); | |
| // The real scan below. | |
| ModCI2->getPreprocessorOpts().SingleModuleParseMode = false; | |
| GenerateModuleFromModuleMapAction Action2; | |
| (void)ModCI2->ExecuteAction(Action2); | |
| }); | |
| }); | |
| } | |
| }; | |
| /// Runs the preprocessor on a TU with single-module-parse-mode and compiles | |
| /// modules asynchronously without blocking or importing them. | |
| struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction { | |
| DependencyScanningService &Service; | |
| DependencyActionController &Controller; | |
| AsyncModuleCompiles &Compiles; | |
| SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service, | |
| DependencyActionController &Controller, | |
| AsyncModuleCompiles &Compiles) | |
| : Service(Service), Controller(Controller), Compiles(Compiles) {} | |
| bool BeginSourceFileAction(CompilerInstance &CI) override { | |
| CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; | |
| CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>( | |
| CI, Service, Controller, Compiles)); | |
| return true; | |
| } | |
| }; | |
| bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction( | |
| CompilerInstance &CI) { | |
| CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; | |
| CI.getPreprocessor().addPPCallbacks( | |
| std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles)); | |
| return true; | |
| } | |
| bool DependencyScanningAction::runInvocation( | |
| std::string Executable, | |
| std::unique_ptr<CompilerInvocation> OriginalInvocation, | |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, | |
| std::shared_ptr<PCHContainerOperations> PCHContainerOps, | |
| DiagnosticConsumer *DiagConsumer) { | |
| // Making sure that we canonicalize the defines early to avoid unnecessary | |
| // variants in both the scanner and in the resulting explicit command lines. | |
| if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros)) | |
| canonicalizeDefines(OriginalInvocation->getPreprocessorOpts()); | |
| if (Scanned) { | |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; | |
| // Scanning runs once for the first -cc1 invocation in a chain of driver | |
| // jobs. For any dependent jobs, reuse the scanning result and just | |
| // update the new invocation. | |
| // FIXME: to support multi-arch builds, each arch requires a separate scan | |
| if (MDC) | |
| MDC->applyDiscoveredDependencies(*OriginalInvocation); | |
| if (!Controller.finalize(ScanInstance, *OriginalInvocation)) | |
| return false; | |
| Consumer.handleBuildCommand( | |
| {Executable, OriginalInvocation->getCC1CommandLine()}); | |
| return true; | |
| } | |
| Scanned = true; | |
| // Create a compiler instance to handle the actual work. | |
| auto ScanInvocation = | |
| createScanCompilerInvocation(*OriginalInvocation, Service, Controller); | |
| // Quickly discovers and compiles modules for the real scan below. | |
| std::optional<AsyncModuleCompiles> AsyncCompiles; | |
| if (Service.getOpts().AsyncScanModules) { | |
| auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); | |
| auto ScanInstanceStorage = std::make_unique<CompilerInstance>( | |
| std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps, | |
| std::move(ModCache)); | |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; | |
| DiagnosticConsumer DiagConsumer; | |
| initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service, | |
| DepFS); | |
| // FIXME: Do this only once. | |
| SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance); | |
| auto MaybePrebuiltModulesASTMap = | |
| computePrebuiltModulesASTMap(ScanInstance, StableDirs); | |
| if (!MaybePrebuiltModulesASTMap) | |
| return false; | |
| // Normally this would be handled by GeneratePCHAction | |
| if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH) | |
| ScanInstance.getLangOpts().CompilingPCH = true; | |
| AsyncCompiles.emplace(); | |
| SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles); | |
| (void)ScanInstance.ExecuteAction(Action); | |
| } | |
| auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); | |
| ScanInstanceStorage.emplace(std::move(ScanInvocation), | |
| std::move(PCHContainerOps), std::move(ModCache)); | |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; | |
| initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service, | |
| DepFS); | |
| llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance); | |
| auto MaybePrebuiltModulesASTMap = | |
| computePrebuiltModulesASTMap(ScanInstance, StableDirs); | |
| if (!MaybePrebuiltModulesASTMap) | |
| return false; | |
| auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation); | |
| MDC = initializeScanInstanceDependencyCollector( | |
| ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer, | |
| Service, *OriginalInvocation, Controller, *MaybePrebuiltModulesASTMap, | |
| StableDirs); | |
| if (ScanInstance.getDiagnostics().hasErrorOccurred()) | |
| return false; | |
| if (!Controller.initialize(ScanInstance, *OriginalInvocation)) | |
| return false; | |
| ReadPCHAndPreprocessAction Action; | |
| const bool Result = ScanInstance.ExecuteAction(Action); | |
| if (Result) { | |
| if (MDC) | |
| MDC->applyDiscoveredDependencies(*OriginalInvocation); | |
| if (!Controller.finalize(ScanInstance, *OriginalInvocation)) | |
| return false; | |
| Consumer.handleBuildCommand( | |
| {Executable, OriginalInvocation->getCC1CommandLine()}); | |
| } | |
| return Result; | |
| } |