| //===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" |
| #include "clang/Lex/DependencyDirectivesSourceMinimizer.h" |
| #include "llvm/Support/MemoryBuffer.h" |
| #include "llvm/Support/Threading.h" |
| |
| using namespace clang; |
| using namespace tooling; |
| using namespace dependencies; |
| |
| CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( |
| StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { |
| // Load the file and its content from the file system. |
| llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MaybeFile = |
| FS.openFileForRead(Filename); |
| if (!MaybeFile) |
| return MaybeFile.getError(); |
| llvm::ErrorOr<llvm::vfs::Status> Stat = (*MaybeFile)->status(); |
| if (!Stat) |
| return Stat.getError(); |
| |
| llvm::vfs::File &F = **MaybeFile; |
| llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MaybeBuffer = |
| F.getBuffer(Stat->getName()); |
| if (!MaybeBuffer) |
| return MaybeBuffer.getError(); |
| |
| llvm::SmallString<1024> MinimizedFileContents; |
| // Minimize the file down to directives that might affect the dependencies. |
| const auto &Buffer = *MaybeBuffer; |
| SmallVector<minimize_source_to_dependency_directives::Token, 64> Tokens; |
| if (!Minimize || minimizeSourceToDependencyDirectives( |
| Buffer->getBuffer(), MinimizedFileContents, Tokens)) { |
| // Use the original file unless requested otherwise, or |
| // if the minimization failed. |
| // FIXME: Propage the diagnostic if desired by the client. |
| CachedFileSystemEntry Result; |
| Result.MaybeStat = std::move(*Stat); |
| Result.Contents.reserve(Buffer->getBufferSize() + 1); |
| Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); |
| // Implicitly null terminate the contents for Clang's lexer. |
| Result.Contents.push_back('\0'); |
| Result.Contents.pop_back(); |
| return Result; |
| } |
| |
| CachedFileSystemEntry Result; |
| size_t Size = MinimizedFileContents.size(); |
| Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), |
| Stat->getLastModificationTime(), |
| Stat->getUser(), Stat->getGroup(), Size, |
| Stat->getType(), Stat->getPermissions()); |
| // The contents produced by the minimizer must be null terminated. |
| assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && |
| "not null terminated contents"); |
| // Even though there's an implicit null terminator in the minimized contents, |
| // we want to temporarily make it explicit. This will ensure that the |
| // std::move will preserve it even if it needs to do a copy if the |
| // SmallString still has the small capacity. |
| MinimizedFileContents.push_back('\0'); |
| Result.Contents = std::move(MinimizedFileContents); |
| // Now make the null terminator implicit again, so that Clang's lexer can find |
| // it right where the buffer ends. |
| Result.Contents.pop_back(); |
| |
| // Compute the skipped PP ranges that speedup skipping over inactive |
| // preprocessor blocks. |
| llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32> |
| SkippedRanges; |
| minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, |
| SkippedRanges); |
| PreprocessorSkippedRangeMapping Mapping; |
| for (const auto &Range : SkippedRanges) { |
| if (Range.Length < 16) { |
| // Ignore small ranges as non-profitable. |
| // FIXME: This is a heuristic, its worth investigating the tradeoffs |
| // when it should be applied. |
| continue; |
| } |
| Mapping[Range.Offset] = Range.Length; |
| } |
| Result.PPSkippedRangeMapping = std::move(Mapping); |
| |
| return Result; |
| } |
| |
| CachedFileSystemEntry |
| CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { |
| assert(Stat.isDirectory() && "not a directory!"); |
| auto Result = CachedFileSystemEntry(); |
| Result.MaybeStat = std::move(Stat); |
| return Result; |
| } |
| |
| DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() { |
| // This heuristic was chosen using a empirical testing on a |
| // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache |
| // sharding gives a performance edge by reducing the lock contention. |
| // FIXME: A better heuristic might also consider the OS to account for |
| // the different cost of lock contention on different OSes. |
| NumShards = |
| std::max(2u, llvm::hardware_concurrency().compute_thread_count() / 4); |
| CacheShards = std::make_unique<CacheShard[]>(NumShards); |
| } |
| |
| DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & |
| DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) { |
| CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; |
| std::unique_lock<std::mutex> LockGuard(Shard.CacheLock); |
| auto It = Shard.Cache.try_emplace(Key); |
| return It.first->getValue(); |
| } |
| |
| DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & |
| DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) { |
| SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal; |
| return Cache.get(Key); |
| } |
| |
| /// Whitelist file extensions that should be minimized, treating no extension as |
| /// a source file that should be minimized. |
| /// |
| /// This is kinda hacky, it would be better if we knew what kind of file Clang |
| /// was expecting instead. |
| static bool shouldMinimizeBasedOnExtension(StringRef Filename) { |
| StringRef Ext = llvm::sys::path::extension(Filename); |
| if (Ext.empty()) |
| return true; // C++ standard library |
| return llvm::StringSwitch<bool>(Ext) |
| .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true) |
| .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true) |
| .CasesLower(".m", ".mm", true) |
| .CasesLower(".i", ".ii", ".mi", ".mmi", true) |
| .CasesLower(".def", ".inc", true) |
| .Default(false); |
| } |
| |
| |
| static bool shouldCacheStatFailures(StringRef Filename) { |
| StringRef Ext = llvm::sys::path::extension(Filename); |
| if (Ext.empty()) |
| return false; // This may be the module cache directory. |
| // Only cache stat failures on source files. |
| return shouldMinimizeBasedOnExtension(Filename); |
| } |
| |
| void DependencyScanningWorkerFilesystem::disableMinimization( |
| StringRef RawFilename) { |
| llvm::SmallString<256> Filename; |
| llvm::sys::path::native(RawFilename, Filename); |
| NotToBeMinimized.insert(Filename); |
| } |
| |
| bool DependencyScanningWorkerFilesystem::shouldMinimize(StringRef RawFilename) { |
| if (!shouldMinimizeBasedOnExtension(RawFilename)) |
| return false; |
| |
| llvm::SmallString<256> Filename; |
| llvm::sys::path::native(RawFilename, Filename); |
| return !NotToBeMinimized.contains(Filename); |
| } |
| |
| CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry( |
| llvm::ErrorOr<llvm::vfs::Status> &&MaybeStatus, StringRef Filename, |
| bool ShouldMinimize) { |
| if (!MaybeStatus) |
| return CachedFileSystemEntry(MaybeStatus.getError()); |
| |
| if (MaybeStatus->isDirectory()) |
| return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus)); |
| |
| return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(), |
| ShouldMinimize); |
| } |
| |
| llvm::ErrorOr<const CachedFileSystemEntry *> |
| DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( |
| const StringRef Filename) { |
| bool ShouldMinimize = shouldMinimize(Filename); |
| |
| if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize)) |
| return Entry; |
| |
| // FIXME: Handle PCM/PCH files. |
| // FIXME: Handle module map files. |
| |
| DependencyScanningFilesystemSharedCache::SharedFileSystemEntry |
| &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize); |
| const CachedFileSystemEntry *Result; |
| { |
| std::unique_lock<std::mutex> LockGuard(SharedCacheEntry.ValueLock); |
| CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; |
| |
| if (!CacheEntry.isValid()) { |
| auto MaybeStatus = getUnderlyingFS().status(Filename); |
| if (!MaybeStatus && !shouldCacheStatFailures(Filename)) |
| // HACK: We need to always restat non source files if the stat fails. |
| // This is because Clang first looks up the module cache and module |
| // files before building them, and then looks for them again. If we |
| // cache the stat failure, it won't see them the second time. |
| return MaybeStatus.getError(); |
| CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename, |
| ShouldMinimize); |
| } |
| |
| Result = &CacheEntry; |
| } |
| |
| // Store the result in the local cache. |
| Cache.setCachedEntry(Filename, ShouldMinimize, Result); |
| return Result; |
| } |
| |
| llvm::ErrorOr<llvm::vfs::Status> |
| DependencyScanningWorkerFilesystem::status(const Twine &Path) { |
| SmallString<256> OwnedFilename; |
| StringRef Filename = Path.toStringRef(OwnedFilename); |
| const llvm::ErrorOr<const CachedFileSystemEntry *> Result = |
| getOrCreateFileSystemEntry(Filename); |
| if (!Result) |
| return Result.getError(); |
| return (*Result)->getStatus(); |
| } |
| |
| namespace { |
| |
| /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using |
| /// this subclass. |
| class MinimizedVFSFile final : public llvm::vfs::File { |
| public: |
| MinimizedVFSFile(std::unique_ptr<llvm::MemoryBuffer> Buffer, |
| llvm::vfs::Status Stat) |
| : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} |
| |
| static llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
| create(const CachedFileSystemEntry *Entry, |
| ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings); |
| |
| llvm::ErrorOr<llvm::vfs::Status> status() override { return Stat; } |
| |
| llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> |
| getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, |
| bool IsVolatile) override { |
| return std::move(Buffer); |
| } |
| |
| std::error_code close() override { return {}; } |
| |
| private: |
| std::unique_ptr<llvm::MemoryBuffer> Buffer; |
| llvm::vfs::Status Stat; |
| }; |
| |
| } // end anonymous namespace |
| |
| llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> MinimizedVFSFile::create( |
| const CachedFileSystemEntry *Entry, |
| ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { |
| if (Entry->isDirectory()) |
| return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( |
| std::make_error_code(std::errc::is_a_directory)); |
| llvm::ErrorOr<StringRef> Contents = Entry->getContents(); |
| if (!Contents) |
| return Contents.getError(); |
| auto Result = std::make_unique<MinimizedVFSFile>( |
| llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), |
| /*RequiresNullTerminator=*/false), |
| *Entry->getStatus()); |
| if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) |
| (*PPSkipMappings)[Result->Buffer->getBufferStart()] = |
| &Entry->getPPSkippedRangeMapping(); |
| return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>( |
| std::unique_ptr<llvm::vfs::File>(std::move(Result))); |
| } |
| |
| llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
| DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { |
| SmallString<256> OwnedFilename; |
| StringRef Filename = Path.toStringRef(OwnedFilename); |
| |
| const llvm::ErrorOr<const CachedFileSystemEntry *> Result = |
| getOrCreateFileSystemEntry(Filename); |
| if (!Result) |
| return Result.getError(); |
| return MinimizedVFSFile::create(Result.get(), PPSkipMappings); |
| } |