| //===- DependencyScannerImpl.cpp - Implements module dependency scanning --===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "clang/DependencyScanning/DependencyScannerImpl.h" |
| #include "clang/Basic/DiagnosticFrontend.h" |
| #include "clang/Basic/DiagnosticSerialization.h" |
| #include "clang/DependencyScanning/DependencyActionController.h" |
| #include "clang/DependencyScanning/DependencyConsumer.h" |
| #include "clang/DependencyScanning/DependencyScanningFilesystem.h" |
| #include "clang/DependencyScanning/DependencyScanningService.h" |
| #include "clang/DependencyScanning/DependencyScanningWorker.h" |
| #include "clang/Frontend/FrontendActions.h" |
| #include "llvm/ADT/IntrusiveRefCntPtr.h" |
| #include "llvm/ADT/ScopeExit.h" |
| #include "llvm/Option/Option.h" |
| #include "llvm/Support/AdvisoryLock.h" |
| #include "llvm/Support/CrashRecoveryContext.h" |
| #include "llvm/Support/VirtualFileSystem.h" |
| #include "llvm/TargetParser/Host.h" |
| |
| #include <mutex> |
| #include <thread> |
| |
| using namespace clang; |
| using namespace dependencies; |
| |
| static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts, |
| const HeaderSearchOptions &ExistingHSOpts, |
| DiagnosticsEngine *Diags, |
| const LangOptions &LangOpts) { |
| if (LangOpts.Modules) { |
| if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { |
| if (Diags) { |
| Diags->Report(diag::warn_pch_vfsoverlay_mismatch); |
| auto VFSNote = [&](int Type, ArrayRef<std::string> VFSOverlays) { |
| if (VFSOverlays.empty()) { |
| Diags->Report(diag::note_pch_vfsoverlay_empty) << Type; |
| } else { |
| std::string Files = llvm::join(VFSOverlays, "\n"); |
| Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files; |
| } |
| }; |
| VFSNote(0, HSOpts.VFSOverlayFiles); |
| VFSNote(1, ExistingHSOpts.VFSOverlayFiles); |
| } |
| } |
| } |
| return false; |
| } |
| |
| namespace { |
| |
| using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); |
| |
| /// A listener that collects the imported modules and the input |
| /// files. While visiting, collect vfsoverlays and file inputs that determine |
| /// whether prebuilt modules fully resolve in stable directories. |
| class PrebuiltModuleListener : public ASTReaderListener { |
| public: |
| PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, |
| llvm::SmallVector<std::string> &NewModuleFiles, |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, |
| const HeaderSearchOptions &HSOpts, |
| const LangOptions &LangOpts, DiagnosticsEngine &Diags, |
| const ArrayRef<StringRef> StableDirs) |
| : PrebuiltModuleFiles(PrebuiltModuleFiles), |
| NewModuleFiles(NewModuleFiles), |
| PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts), |
| ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {} |
| |
| bool needsImportVisitation() const override { return true; } |
| bool needsInputFileVisitation() override { return true; } |
| bool needsSystemInputFileVisitation() override { return true; } |
| |
| /// Accumulate the modules are transitively depended on by the initial |
| /// prebuilt module. |
| void visitImport(StringRef ModuleName, StringRef Filename) override { |
| if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) |
| NewModuleFiles.push_back(Filename.str()); |
| |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename); |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| if (PrebuiltMapEntry.second) |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| |
| if (auto It = PrebuiltModulesASTMap.find(CurrentFile); |
| It != PrebuiltModulesASTMap.end() && CurrentFile != Filename) |
| PrebuiltModule.addDependent(It->getKey()); |
| } |
| |
| /// For each input file discovered, check whether it's external path is in a |
| /// stable directory. Traversal is stopped if the current module is not |
| /// considered stable. |
| bool visitInputFileAsRequested(StringRef FilenameAsRequested, |
| StringRef Filename, bool isSystem, |
| bool isOverridden, time_t StoredTime, |
| bool isExplicitModule) override { |
| if (StableDirs.empty()) |
| return false; |
| auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); |
| if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) || |
| (!PrebuiltEntryIt->second.isInStableDir())) |
| return false; |
| |
| PrebuiltEntryIt->second.setInStableDir( |
| isPathInStableDir(StableDirs, Filename)); |
| return PrebuiltEntryIt->second.isInStableDir(); |
| } |
| |
| /// Update which module that is being actively traversed. |
| void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind, |
| bool DirectlyImported) override { |
| // If the CurrentFile is not |
| // considered stable, update any of it's transitive dependents. |
| auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); |
| if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) && |
| !PrebuiltEntryIt->second.isInStableDir()) |
| PrebuiltEntryIt->second.updateDependentsNotInStableDirs( |
| PrebuiltModulesASTMap); |
| CurrentFile = Filename.str(); |
| } |
| |
| /// Check the header search options for a given module when considering |
| /// if the module comes from stable directories. |
| bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, |
| StringRef ModuleFilename, StringRef ContextHash, |
| bool Complain) override { |
| |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| if (PrebuiltMapEntry.second) |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| |
| if (PrebuiltModule.isInStableDir()) |
| PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts)); |
| |
| return false; |
| } |
| |
| /// Accumulate vfsoverlays used to build these prebuilt modules. |
| bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, |
| bool Complain) override { |
| |
| auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); |
| PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; |
| if (PrebuiltMapEntry.second) |
| PrebuiltModule.setInStableDir(!StableDirs.empty()); |
| |
| PrebuiltModule.setVFS( |
| llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles)); |
| |
| return checkHeaderSearchPaths( |
| HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts); |
| } |
| |
| private: |
| PrebuiltModuleFilesT &PrebuiltModuleFiles; |
| llvm::SmallVector<std::string> &NewModuleFiles; |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap; |
| const HeaderSearchOptions &ExistingHSOpts; |
| const LangOptions &ExistingLangOpts; |
| DiagnosticsEngine &Diags; |
| std::string CurrentFile; |
| const ArrayRef<StringRef> StableDirs; |
| }; |
| |
| /// Visit the given prebuilt module and collect all of the modules it |
| /// transitively imports and contributing input files. |
| static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, |
| CompilerInstance &CI, |
| PrebuiltModuleFilesT &ModuleFiles, |
| PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, |
| DiagnosticsEngine &Diags, |
| const ArrayRef<StringRef> StableDirs) { |
| // List of module files to be processed. |
| llvm::SmallVector<std::string> Worklist; |
| |
| PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap, |
| CI.getHeaderSearchOpts(), CI.getLangOpts(), |
| Diags, StableDirs); |
| |
| Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename), |
| serialization::MK_ExplicitModule, |
| /*DirectlyImported=*/true); |
| if (ASTReader::readASTFileControlBlock( |
| PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(), |
| CI.getPCHContainerReader(), |
| /*FindModuleFileExtensions=*/false, Listener, |
| /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate)) |
| return true; |
| |
| while (!Worklist.empty()) { |
| // FIXME: This is assuming the PCH only refers to explicitly-built modules, |
| // which technically is not guaranteed. To remove the assumption, we'd need |
| // to also rework how the module files are handled to the scan, specifically |
| // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain |
| // paths to ModuleFileName. |
| Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()), |
| serialization::MK_ExplicitModule, |
| /*DirectlyImported=*/false); |
| if (ASTReader::readASTFileControlBlock( |
| Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), |
| CI.getPCHContainerReader(), |
| /*FindModuleFileExtensions=*/false, Listener, |
| /*ValidateDiagnosticOptions=*/false)) |
| return true; |
| } |
| return false; |
| } |
| |
| /// Transform arbitrary file name into an object-like file name. |
| static std::string makeObjFileName(StringRef FileName) { |
| SmallString<128> ObjFileName(FileName); |
| llvm::sys::path::replace_extension(ObjFileName, "o"); |
| return std::string(ObjFileName); |
| } |
| |
| /// Deduce the dependency target based on the output file and input files. |
| static std::string |
| deduceDepTarget(const std::string &OutputFile, |
| const SmallVectorImpl<FrontendInputFile> &InputFiles) { |
| if (OutputFile != "-") |
| return OutputFile; |
| |
| if (InputFiles.empty() || !InputFiles.front().isFile()) |
| return "clang-scan-deps\\ dependency"; |
| |
| return makeObjFileName(InputFiles.front().getFile()); |
| } |
| |
| // Clang implements -D and -U by splatting text into a predefines buffer. This |
| // allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and |
| // define the same macro, or adding C++ style comments before the macro name. |
| // |
| // This function checks that the first non-space characters in the macro |
| // obviously form an identifier that can be uniqued on without lexing. Failing |
| // to do this could lead to changing the final definition of a macro. |
| // |
| // We could set up a preprocessor and actually lex the name, but that's very |
| // heavyweight for a situation that will almost never happen in practice. |
| static std::optional<StringRef> getSimpleMacroName(StringRef Macro) { |
| StringRef Name = Macro.split("=").first.ltrim(" \t"); |
| std::size_t I = 0; |
| |
| auto FinishName = [&]() -> std::optional<StringRef> { |
| StringRef SimpleName = Name.slice(0, I); |
| if (SimpleName.empty()) |
| return std::nullopt; |
| return SimpleName; |
| }; |
| |
| for (; I != Name.size(); ++I) { |
| switch (Name[I]) { |
| case '(': // Start of macro parameter list |
| case ' ': // End of macro name |
| case '\t': |
| return FinishName(); |
| case '_': |
| continue; |
| default: |
| if (llvm::isAlnum(Name[I])) |
| continue; |
| return std::nullopt; |
| } |
| } |
| return FinishName(); |
| } |
| } // namespace |
| |
| void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) { |
| using MacroOpt = std::pair<StringRef, std::size_t>; |
| std::vector<MacroOpt> SimpleNames; |
| SimpleNames.reserve(PPOpts.Macros.size()); |
| std::size_t Index = 0; |
| for (const auto &M : PPOpts.Macros) { |
| auto SName = getSimpleMacroName(M.first); |
| // Skip optimizing if we can't guarantee we can preserve relative order. |
| if (!SName) |
| return; |
| SimpleNames.emplace_back(*SName, Index); |
| ++Index; |
| } |
| |
| llvm::stable_sort(SimpleNames, llvm::less_first()); |
| // Keep the last instance of each macro name by going in reverse |
| auto NewEnd = std::unique( |
| SimpleNames.rbegin(), SimpleNames.rend(), |
| [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; }); |
| SimpleNames.erase(SimpleNames.begin(), NewEnd.base()); |
| |
| // Apply permutation. |
| decltype(PPOpts.Macros) NewMacros; |
| NewMacros.reserve(SimpleNames.size()); |
| for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) { |
| std::size_t OriginalIndex = SimpleNames[I].second; |
| // We still emit undefines here as they may be undefining a predefined macro |
| NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex])); |
| } |
| std::swap(PPOpts.Macros, NewMacros); |
| } |
| |
| namespace { |
| class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { |
| DependencyScanningWorkerFilesystem *DepFS; |
| |
| public: |
| ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) { |
| FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) { |
| auto *DFS = llvm::dyn_cast<DependencyScanningWorkerFilesystem>(&FS); |
| if (DFS) { |
| assert(!DepFS && "Found multiple scanning VFSs"); |
| DepFS = DFS; |
| } |
| }); |
| assert(DepFS && "Did not find scanning VFS"); |
| } |
| |
| std::unique_ptr<DependencyDirectivesGetter> |
| cloneFor(FileManager &FileMgr) override { |
| return std::make_unique<ScanningDependencyDirectivesGetter>(FileMgr); |
| } |
| |
| std::optional<ArrayRef<dependency_directives_scan::Directive>> |
| operator()(FileEntryRef File) override { |
| return DepFS->getDirectiveTokens(File.getName()); |
| } |
| }; |
| |
| /// Sanitize diagnostic options for dependency scan. |
| void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { |
| // Don't print 'X warnings and Y errors generated'. |
| DiagOpts.ShowCarets = false; |
| // Don't write out diagnostic file. |
| DiagOpts.DiagnosticSerializationFile.clear(); |
| // Don't emit warnings except for scanning specific warnings. |
| // TODO: It would be useful to add a more principled way to ignore all |
| // warnings that come from source code. The issue is that we need to |
| // ignore warnings that could be surpressed by |
| // `#pragma clang diagnostic`, while still allowing some scanning |
| // warnings for things we're not ready to turn into errors yet. |
| // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example. |
| llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) { |
| return llvm::StringSwitch<bool>(Warning) |
| .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false) |
| .StartsWith("no-error=", false) |
| .Default(true); |
| }); |
| } |
| } // namespace |
| |
| std::unique_ptr<DiagnosticOptions> |
| dependencies::createDiagOptions(ArrayRef<std::string> CommandLine) { |
| std::vector<const char *> CLI; |
| for (const std::string &Arg : CommandLine) |
| CLI.push_back(Arg.c_str()); |
| auto DiagOpts = CreateAndPopulateDiagOpts(CLI); |
| sanitizeDiagOpts(*DiagOpts); |
| return DiagOpts; |
| } |
| |
| DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts( |
| ArrayRef<std::string> CommandLine, |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, DiagnosticConsumer &DC) { |
| std::vector<const char *> CCommandLine(CommandLine.size(), nullptr); |
| llvm::transform(CommandLine, CCommandLine.begin(), |
| [](const std::string &Str) { return Str.c_str(); }); |
| DiagOpts = CreateAndPopulateDiagOpts(CCommandLine); |
| sanitizeDiagOpts(*DiagOpts); |
| DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC, |
| /*ShouldOwnClient=*/false); |
| } |
| |
| std::unique_ptr<CompilerInvocation> |
| dependencies::createCompilerInvocation(ArrayRef<std::string> CommandLine, |
| DiagnosticsEngine &Diags) { |
| llvm::opt::ArgStringList Argv; |
| for (const std::string &Str : ArrayRef(CommandLine).drop_front()) |
| Argv.push_back(Str.c_str()); |
| |
| auto Invocation = std::make_unique<CompilerInvocation>(); |
| if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) { |
| // FIXME: Should we just go on like cc1_main does? |
| return nullptr; |
| } |
| return Invocation; |
| } |
| |
| void dependencies::initializeScanCompilerInstance( |
| CompilerInstance &ScanInstance, |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, |
| DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, |
| IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS) { |
| ScanInstance.setBuildingModule(false); |
| ScanInstance.createVirtualFileSystem(FS, DiagConsumer); |
| ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); |
| if (!Service.getOpts().EmitWarnings) |
| ScanInstance.getDiagnostics().setIgnoreAllWarnings(true); |
| ScanInstance.createFileManager(); |
| ScanInstance.createSourceManager(); |
| |
| // Use DepFS for getting the dependency directives if requested to do so. |
| if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) |
| ScanInstance.setDependencyDirectivesGetter( |
| std::make_unique<ScanningDependencyDirectivesGetter>( |
| ScanInstance.getFileManager())); |
| } |
| |
| std::shared_ptr<CompilerInvocation> dependencies::createScanCompilerInvocation( |
| const CompilerInvocation &Invocation, |
| const DependencyScanningService &Service, |
| DependencyActionController &Controller) { |
| auto ScanInvocation = std::make_shared<CompilerInvocation>(Invocation); |
| |
| sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts()); |
| |
| ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = |
| true; |
| |
| if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession) |
| ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp = |
| Service.getOpts().BuildSessionTimestamp; |
| |
| ScanInvocation->getFrontendOpts().DisableFree = false; |
| ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false; |
| ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false; |
| ScanInvocation->getFrontendOpts().GenReducedBMI = false; |
| ScanInvocation->getFrontendOpts().ModuleOutputPath.clear(); |
| // This will prevent us compiling individual modules asynchronously since |
| // FileManager is not thread-safe, but it does improve performance for now. |
| ScanInvocation->getFrontendOpts().ModulesShareFileManager = true; |
| ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw"; |
| ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage = |
| any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS); |
| |
| // Consider different header search and diagnostic options to create |
| // different modules. This avoids the unsound aliasing of module PCMs. |
| // |
| // TODO: Implement diagnostic bucketing to reduce the impact of strict |
| // context hashing. |
| ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true; |
| ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; |
| ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = |
| true; |
| ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; |
| |
| // Application extension only affects the handling of availability attributes, |
| // which cannot change the dependencies. |
| ScanInvocation->getLangOpts().AppExt = false; |
| |
| // Ensure that the scanner does not create new dependency collectors, |
| // and thus won't write out the extra '.d' files to disk. |
| ScanInvocation->getDependencyOutputOpts() = {}; |
| |
| Controller.initializeScanInvocation(*ScanInvocation); |
| |
| return ScanInvocation; |
| } |
| |
| llvm::SmallVector<StringRef> |
| dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) { |
| // Create a collection of stable directories derived from the ScanInstance |
| // for determining whether module dependencies would fully resolve from |
| // those directories. |
| llvm::SmallVector<StringRef> StableDirs; |
| const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; |
| if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot)) |
| StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; |
| return StableDirs; |
| } |
| |
| std::optional<PrebuiltModulesAttrsMap> |
| dependencies::computePrebuiltModulesASTMap( |
| CompilerInstance &ScanInstance, llvm::SmallVector<StringRef> &StableDirs) { |
| // Store a mapping of prebuilt module files and their properties like header |
| // search options. This will prevent the implicit build to create duplicate |
| // modules and will force reuse of the existing prebuilt module files |
| // instead. |
| PrebuiltModulesAttrsMap PrebuiltModulesASTMap; |
| |
| if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) |
| if (visitPrebuiltModule( |
| ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, |
| ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, |
| PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs)) |
| return {}; |
| |
| return PrebuiltModulesASTMap; |
| } |
| |
| std::unique_ptr<DependencyOutputOptions> |
| dependencies::createDependencyOutputOptions( |
| const CompilerInvocation &Invocation) { |
| auto Opts = std::make_unique<DependencyOutputOptions>( |
| Invocation.getDependencyOutputOpts()); |
| // We need at least one -MT equivalent for the generator of make dependency |
| // files to work. |
| if (Opts->Targets.empty()) |
| Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile, |
| Invocation.getFrontendOpts().Inputs)}; |
| Opts->IncludeSystemHeaders = true; |
| |
| return Opts; |
| } |
| |
| std::shared_ptr<ModuleDepCollector> |
| dependencies::initializeScanInstanceDependencyCollector( |
| CompilerInstance &ScanInstance, |
| std::unique_ptr<DependencyOutputOptions> DepOutputOpts, |
| DependencyScanningService &Service, CompilerInvocation &Inv, |
| DependencyActionController &Controller, |
| PrebuiltModulesAttrsMap PrebuiltModulesASTMap, |
| SmallVector<StringRef> &StableDirs) { |
| auto MDC = std::make_shared<ModuleDepCollector>( |
| Service, std::move(DepOutputOpts), ScanInstance, Controller, Inv, |
| std::move(PrebuiltModulesASTMap), StableDirs); |
| ScanInstance.addDependencyCollector(MDC); |
| return MDC; |
| } |
| |
| /// Manages (and terminates) the asynchronous compilation of modules. |
| class AsyncModuleCompiles { |
| std::mutex Mutex; |
| bool Stop = false; |
| // FIXME: Have the service own a thread pool and use that instead. |
| std::vector<std::thread> Compiles; |
| |
| public: |
| /// Registers the module compilation, unless this instance is about to be |
| /// destroyed. |
| void add(llvm::unique_function<void()> Compile) { |
| std::lock_guard<std::mutex> Lock(Mutex); |
| if (!Stop) |
| Compiles.emplace_back(std::move(Compile)); |
| } |
| |
| ~AsyncModuleCompiles() { |
| { |
| // Prevent registration of further module compiles. |
| std::lock_guard<std::mutex> Lock(Mutex); |
| Stop = true; |
| } |
| |
| // Wait for outstanding module compiles to finish. |
| for (std::thread &Compile : Compiles) |
| Compile.join(); |
| } |
| }; |
| |
| struct SingleModuleWithAsyncModuleCompiles : PreprocessOnlyAction { |
| DependencyScanningService &Service; |
| DependencyActionController &Controller; |
| AsyncModuleCompiles &Compiles; |
| |
| SingleModuleWithAsyncModuleCompiles(DependencyScanningService &Service, |
| DependencyActionController &Controller, |
| AsyncModuleCompiles &Compiles) |
| : Service(Service), Controller(Controller), Compiles(Compiles) {} |
| |
| bool BeginSourceFileAction(CompilerInstance &CI) override; |
| }; |
| |
| /// The preprocessor callback that takes care of initiating an asynchronous |
| /// module compilation if needed. |
| struct AsyncModuleCompile : PPCallbacks { |
| CompilerInstance &CI; |
| DependencyScanningService &Service; |
| DependencyActionController &Controller; |
| AsyncModuleCompiles &Compiles; |
| |
| AsyncModuleCompile(CompilerInstance &CI, DependencyScanningService &Service, |
| DependencyActionController &Controller, |
| AsyncModuleCompiles &Compiles) |
| : CI(CI), Service(Service), Controller(Controller), Compiles(Compiles) {} |
| |
| void moduleLoadSkipped(Module *M) override { |
| M = M->getTopLevelModule(); |
| |
| HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); |
| ModuleCache &ModCache = CI.getModuleCache(); |
| ModuleFileName ModuleFileName = HS.getCachedModuleFileName(M); |
| |
| uint64_t Timestamp = ModCache.getModuleTimestamp(ModuleFileName); |
| // Someone else already built/validated the PCM. |
| if (Timestamp > CI.getHeaderSearchOpts().BuildSessionTimestamp) |
| return; |
| |
| if (!CI.getASTReader()) |
| CI.createASTReader(); |
| SmallVector<ASTReader::ImportedModule, 0> Imported; |
| // Only calling ReadASTCore() to avoid the expensive eager deserialization |
| // of the clang::Module objects in ReadAST(). |
| // FIXME: Consider doing this in the new thread depending on how expensive |
| // the read turns out to be. |
| switch (CI.getASTReader()->ReadASTCore( |
| ModuleFileName, serialization::MK_ImplicitModule, SourceLocation(), |
| nullptr, Imported, {}, {}, {}, |
| ASTReader::ARR_OutOfDate | ASTReader::ARR_Missing | |
| ASTReader::ARR_TreatModuleWithErrorsAsOutOfDate)) { |
| case ASTReader::Success: |
| // We successfully read a valid, up-to-date PCM. |
| // FIXME: This could update the timestamp. Regular calls to |
| // ASTReader::ReadAST() would do so unless they encountered corrupted |
| // AST block, corrupted extension block, or did not read the expected |
| // top-level module. |
| return; |
| case ASTReader::OutOfDate: |
| case ASTReader::Missing: |
| // The most interesting case. |
| break; |
| default: |
| // Let the regular scan diagnose this. |
| return; |
| } |
| |
| auto Lock = ModCache.getLock(ModuleFileName); |
| bool Owned; |
| llvm::Error LockErr = Lock->tryLock().moveInto(Owned); |
| // Someone else is building the PCM right now. |
| if (!LockErr && !Owned) |
| return; |
| // We should build the PCM. |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS = |
| llvm::makeIntrusiveRefCnt<DependencyScanningWorkerFilesystem>( |
| Service, Service.getOpts().MakeVFS()); |
| VFS = createVFSFromCompilerInvocation(CI.getInvocation(), |
| CI.getDiagnostics(), std::move(VFS)); |
| auto DC = std::make_unique<DiagnosticConsumer>(); |
| auto MC = makeInProcessModuleCache(Service.getModuleCacheEntries()); |
| CompilerInstance::ThreadSafeCloneConfig CloneConfig(std::move(VFS), *DC, |
| std::move(MC)); |
| auto ModCI1 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName, |
| CloneConfig); |
| auto ModCI2 = CI.cloneForModuleCompile(SourceLocation(), M, ModuleFileName, |
| CloneConfig); |
| |
| auto ModController = Controller.clone(); |
| |
| // Note: This lock belongs to a module cache that might not outlive the |
| // thread. This works, because the in-process lock only refers to an object |
| // managed by the service, which does outlive the thread. |
| Compiles.add([Lock = std::move(Lock), ModCI1 = std::move(ModCI1), |
| ModCI2 = std::move(ModCI2), DC = std::move(DC), |
| ModController = std::move(ModController), Service = &Service, |
| Compiles = &Compiles] { |
| llvm::CrashRecoveryContext CRC; |
| (void)CRC.RunSafely([&] { |
| // Quickly discovers and compiles modules for the real scan below. |
| SingleModuleWithAsyncModuleCompiles Action1(*Service, *ModController, |
| *Compiles); |
| (void)ModCI1->ExecuteAction(Action1); |
| // The real scan below. |
| ModCI2->getPreprocessorOpts().SingleModuleParseMode = false; |
| GenerateModuleFromModuleMapAction Action2; |
| (void)ModCI2->ExecuteAction(Action2); |
| }); |
| }); |
| } |
| }; |
| |
| /// Runs the preprocessor on a TU with single-module-parse-mode and compiles |
| /// modules asynchronously without blocking or importing them. |
| struct SingleTUWithAsyncModuleCompiles : PreprocessOnlyAction { |
| DependencyScanningService &Service; |
| DependencyActionController &Controller; |
| AsyncModuleCompiles &Compiles; |
| |
| SingleTUWithAsyncModuleCompiles(DependencyScanningService &Service, |
| DependencyActionController &Controller, |
| AsyncModuleCompiles &Compiles) |
| : Service(Service), Controller(Controller), Compiles(Compiles) {} |
| |
| bool BeginSourceFileAction(CompilerInstance &CI) override { |
| CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; |
| CI.getPreprocessor().addPPCallbacks(std::make_unique<AsyncModuleCompile>( |
| CI, Service, Controller, Compiles)); |
| return true; |
| } |
| }; |
| |
| bool SingleModuleWithAsyncModuleCompiles::BeginSourceFileAction( |
| CompilerInstance &CI) { |
| CI.getInvocation().getPreprocessorOpts().SingleModuleParseMode = true; |
| CI.getPreprocessor().addPPCallbacks( |
| std::make_unique<AsyncModuleCompile>(CI, Service, Controller, Compiles)); |
| return true; |
| } |
| |
| bool DependencyScanningAction::runInvocation( |
| std::string Executable, |
| std::unique_ptr<CompilerInvocation> OriginalInvocation, |
| IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, |
| std::shared_ptr<PCHContainerOperations> PCHContainerOps, |
| DiagnosticConsumer *DiagConsumer) { |
| // Making sure that we canonicalize the defines early to avoid unnecessary |
| // variants in both the scanner and in the resulting explicit command lines. |
| if (any(Service.getOpts().OptimizeArgs & ScanningOptimizations::Macros)) |
| canonicalizeDefines(OriginalInvocation->getPreprocessorOpts()); |
| |
| if (Scanned) { |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; |
| |
| // Scanning runs once for the first -cc1 invocation in a chain of driver |
| // jobs. For any dependent jobs, reuse the scanning result and just |
| // update the new invocation. |
| // FIXME: to support multi-arch builds, each arch requires a separate scan |
| if (MDC) |
| MDC->applyDiscoveredDependencies(*OriginalInvocation); |
| |
| if (!Controller.finalize(ScanInstance, *OriginalInvocation)) |
| return false; |
| |
| Consumer.handleBuildCommand( |
| {Executable, OriginalInvocation->getCC1CommandLine()}); |
| return true; |
| } |
| |
| Scanned = true; |
| |
| // Create a compiler instance to handle the actual work. |
| auto ScanInvocation = |
| createScanCompilerInvocation(*OriginalInvocation, Service, Controller); |
| |
| // Quickly discovers and compiles modules for the real scan below. |
| std::optional<AsyncModuleCompiles> AsyncCompiles; |
| if (Service.getOpts().AsyncScanModules) { |
| auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); |
| auto ScanInstanceStorage = std::make_unique<CompilerInstance>( |
| std::make_shared<CompilerInvocation>(*ScanInvocation), PCHContainerOps, |
| std::move(ModCache)); |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; |
| |
| DiagnosticConsumer DiagConsumer; |
| initializeScanCompilerInstance(ScanInstance, FS, &DiagConsumer, Service, |
| DepFS); |
| |
| // FIXME: Do this only once. |
| SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance); |
| auto MaybePrebuiltModulesASTMap = |
| computePrebuiltModulesASTMap(ScanInstance, StableDirs); |
| if (!MaybePrebuiltModulesASTMap) |
| return false; |
| |
| // Normally this would be handled by GeneratePCHAction |
| if (ScanInstance.getFrontendOpts().ProgramAction == frontend::GeneratePCH) |
| ScanInstance.getLangOpts().CompilingPCH = true; |
| |
| AsyncCompiles.emplace(); |
| SingleTUWithAsyncModuleCompiles Action(Service, Controller, *AsyncCompiles); |
| (void)ScanInstance.ExecuteAction(Action); |
| } |
| |
| auto ModCache = makeInProcessModuleCache(Service.getModuleCacheEntries()); |
| ScanInstanceStorage.emplace(std::move(ScanInvocation), |
| std::move(PCHContainerOps), std::move(ModCache)); |
| CompilerInstance &ScanInstance = *ScanInstanceStorage; |
| |
| initializeScanCompilerInstance(ScanInstance, FS, DiagConsumer, Service, |
| DepFS); |
| |
| llvm::SmallVector<StringRef> StableDirs = getInitialStableDirs(ScanInstance); |
| auto MaybePrebuiltModulesASTMap = |
| computePrebuiltModulesASTMap(ScanInstance, StableDirs); |
| if (!MaybePrebuiltModulesASTMap) |
| return false; |
| |
| auto DepOutputOpts = createDependencyOutputOptions(*OriginalInvocation); |
| |
| MDC = initializeScanInstanceDependencyCollector( |
| ScanInstance, std::move(DepOutputOpts), Service, *OriginalInvocation, |
| Controller, *MaybePrebuiltModulesASTMap, StableDirs); |
| |
| if (ScanInstance.getDiagnostics().hasErrorOccurred()) |
| return false; |
| |
| if (!Controller.initialize(ScanInstance, *OriginalInvocation)) |
| return false; |
| |
| ReadPCHAndPreprocessAction Action; |
| const bool Result = ScanInstance.ExecuteAction(Action); |
| |
| if (Result) { |
| if (MDC) { |
| MDC->run(Consumer); |
| MDC->applyDiscoveredDependencies(*OriginalInvocation); |
| } |
| |
| if (!Controller.finalize(ScanInstance, *OriginalInvocation)) |
| return false; |
| |
| Consumer.handleBuildCommand( |
| {Executable, OriginalInvocation->getCC1CommandLine()}); |
| } |
| |
| return Result; |
| } |
