blob: 5239bc3d185a1b46f3aea393654b2f4a2314d005 [file] [log] [blame]
//===--- ParsedAST.cpp -------------------------------------------*- C++-*-===//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "ParsedAST.h"
#include "../clang-tidy/ClangTidyCheck.h"
#include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
#include "../clang-tidy/ClangTidyModuleRegistry.h"
#include "AST.h"
#include "Compiler.h"
#include "Config.h"
#include "Diagnostics.h"
#include "Feature.h"
#include "FeatureModule.h"
#include "Headers.h"
#include "HeuristicResolver.h"
#include "IncludeCleaner.h"
#include "IncludeFixer.h"
#include "Preamble.h"
#include "SourceCode.h"
#include "TidyProvider.h"
#include "index/CanonicalIncludes.h"
#include "index/Index.h"
#include "support/Logger.h"
#include "support/Trace.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Decl.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LangOptions.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendActions.h"
#include "clang/Frontend/Utils.h"
#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Lex/Lexer.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Sema/Sema.h"
#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/PCHContainerOperations.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "clang/Tooling/Syntax/Tokens.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <memory>
#include <vector>
// Force the linker to link in Clang-tidy modules.
// clangd doesn't support the static analyzer.
#include "../clang-tidy/ClangTidyForceLinker.h"
namespace clang {
namespace clangd {
namespace {
template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
return Vec.capacity() * sizeof(T);
class DeclTrackingASTConsumer : public ASTConsumer {
DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
: TopLevelDecls(TopLevelDecls) {}
bool HandleTopLevelDecl(DeclGroupRef DG) override {
for (Decl *D : DG) {
auto &SM = D->getASTContext().getSourceManager();
if (!isInsideMainFile(D->getLocation(), SM))
if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
if (isImplicitTemplateInstantiation(ND))
// ObjCMethodDecl are not actually top-level decls.
if (isa<ObjCMethodDecl>(D))
return true;
std::vector<Decl *> &TopLevelDecls;
class ClangdFrontendAction : public SyntaxOnlyAction {
std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
return std::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
std::vector<Decl *> TopLevelDecls;
// When using a preamble, only preprocessor events outside its bounds are seen.
// This is almost what we want: replaying transitive preprocessing wastes time.
// However this confuses clang-tidy checks: they don't see any #includes!
// So we replay the *non-transitive* #includes that appear in the main-file.
// It would be nice to replay other events (macro definitions, ifdefs etc) but
// this addresses the most common cases fairly cheaply.
class ReplayPreamble : private PPCallbacks {
// Attach preprocessor hooks such that preamble events will be injected at
// the appropriate time.
// Events will be delivered to the *currently registered* PP callbacks.
static void attach(std::vector<Inclusion> Includes, CompilerInstance &Clang,
const PreambleBounds &PB) {
auto &PP = Clang.getPreprocessor();
auto *ExistingCallbacks = PP.getPPCallbacks();
// No need to replay events if nobody is listening.
if (!ExistingCallbacks)
PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(new ReplayPreamble(
std::move(Includes), ExistingCallbacks, Clang.getSourceManager(), PP,
Clang.getLangOpts(), PB)));
// We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
// around, creating a chaining wrapper. Guard against other implementations.
assert(PP.getPPCallbacks() != ExistingCallbacks &&
"Expected chaining implementation");
ReplayPreamble(std::vector<Inclusion> Includes, PPCallbacks *Delegate,
const SourceManager &SM, Preprocessor &PP,
const LangOptions &LangOpts, const PreambleBounds &PB)
: Includes(std::move(Includes)), Delegate(Delegate), SM(SM), PP(PP) {
// Only tokenize the preamble section of the main file, as we are not
// interested in the rest of the tokens.
MainFileTokens = syntax::tokenize(
syntax::FileRange(SM.getMainFileID(), 0, PB.Size), SM, LangOpts);
// In a normal compile, the preamble traverses the following structure:
// mainfile.cpp
// <built-in>
// ... macro definitions like __cplusplus ...
// <command-line>
// ... macro definitions for args like -Dfoo=bar ...
// "header1.h"
// ... header file contents ...
// "header2.h"
// ... header file contents ...
// ... main file contents ...
// When using a preamble, the "header1" and "header2" subtrees get skipped.
// We insert them right after the built-in header, which still appears.
void FileChanged(SourceLocation Loc, FileChangeReason Reason,
SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
// It'd be nice if there was a better way to identify built-in headers...
if (Reason == FileChangeReason::ExitFile &&
SM.getBufferOrFake(PrevFID).getBufferIdentifier() == "<built-in>")
void replay() {
for (const auto &Inc : Includes) {
llvm::Optional<FileEntryRef> File;
if (Inc.Resolved != "")
File = expectedToOptional(SM.getFileManager().getFileRef(Inc.Resolved));
// Re-lex the #include directive to find its interesting parts.
auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset);
auto HashTok = llvm::partition_point(MainFileTokens,
[&HashLoc](const syntax::Token &T) {
return T.location() < HashLoc;
assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash);
auto IncludeTok = std::next(HashTok);
assert(IncludeTok != MainFileTokens.end());
auto FileTok = std::next(IncludeTok);
assert(FileTok != MainFileTokens.end());
// Create a fake import/include token, none of the callers seem to care
// about clang::Token::Flags.
Token SynthesizedIncludeTok;
// Same here, create a fake one for Filename, including angles or quotes.
Token SynthesizedFilenameTok;
// Note that we can't make use of FileTok->length/text in here as in the
// case of angled includes this will contain tok::less instead of
// filename. Whereas Inc.Written contains the full header name including
// quotes/angles.
const FileEntry *FE = File ? &File->getFileEntry() : nullptr;
llvm::StringRef WrittenFilename =
HashTok->location(), SynthesizedIncludeTok, WrittenFilename,
Inc.Written.front() == '<',
syntax::FileRange(SM, SynthesizedFilenameTok.getLocation(),
FE, "SearchPath", "RelPath",
/*Imported=*/nullptr, Inc.FileKind);
if (File)
Delegate->FileSkipped(*File, SynthesizedFilenameTok, Inc.FileKind);
else {
llvm::SmallString<1> UnusedRecovery;
Delegate->FileNotFound(WrittenFilename, UnusedRecovery);
const std::vector<Inclusion> Includes;
PPCallbacks *Delegate;
const SourceManager &SM;
Preprocessor &PP;
std::vector<syntax::Token> MainFileTokens;
} // namespace
ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
std::unique_ptr<clang::CompilerInvocation> CI,
llvm::ArrayRef<Diag> CompilerInvocationDiags,
std::shared_ptr<const PreambleData> Preamble) {
trace::Span Tracer("BuildAST");
SPAN_ATTACH(Tracer, "File", Filename);
auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
if (Preamble && Preamble->StatCache)
VFS = Preamble->StatCache->getConsumingFS(std::move(VFS));
// Command-line parsing sets DisableFree to true by default, but we don't want
// to leak memory in clangd.
CI->getFrontendOpts().DisableFree = false;
const PrecompiledPreamble *PreamblePCH =
Preamble ? &Preamble->Preamble : nullptr;
// This is on-by-default in windows to allow parsing SDK headers, but it
// breaks many features. Disable it for the main-file (not preamble).
CI->getLangOpts()->DelayedTemplateParsing = false;
std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
if (Inputs.FeatureModules) {
for (auto &M : *Inputs.FeatureModules) {
if (auto Listener = M.astListeners())
StoreDiags ASTDiags;
[&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
[&](const auto &L) { L->sawDiagnostic(D, Diag); });
llvm::Optional<PreamblePatch> Patch;
bool PreserveDiags = true;
// We might use an ignoring diagnostic consumer if they are going to be
// dropped later on to not pay for extra latency by processing them.
DiagnosticConsumer *DiagConsumer = &ASTDiags;
IgnoreDiagnostics DropDiags;
if (Preamble) {
Patch = PreamblePatch::createFullPatch(Filename, Inputs, *Preamble);
PreserveDiags = Patch->preserveDiagnostics();
if (!PreserveDiags)
DiagConsumer = &DropDiags;
auto Clang = prepareCompilerInstance(
std::move(CI), PreamblePCH,
llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS,
if (!Clang) {
// The last diagnostic contains information about the reason of this
// failure.
std::vector<Diag> Diags(ASTDiags.take());
elog("Failed to prepare a compiler instance: {0}",
!Diags.empty() ? static_cast<DiagBase &>(Diags.back()).Message
: "unknown error");
return None;
if (!PreserveDiags) {
// Skips some analysis.
Clang->getDiagnosticOpts().IgnoreWarnings = true;
auto Action = std::make_unique<ClangdFrontendAction>();
const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
if (!Action->BeginSourceFile(*Clang, MainInput)) {
log("BeginSourceFile() failed when building AST for {0}",
return None;
// If we saw an include guard in the preamble section of the main file,
// mark the main-file as include-guarded.
// This information is part of the HeaderFileInfo but is not loaded from the
// preamble as the file's size is part of its identity and may have changed.
// (The rest of HeaderFileInfo is not relevant for our purposes).
if (Preamble && Preamble->MainIsIncludeGuarded) {
const SourceManager &SM = Clang->getSourceManager();
const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID());
// Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
// Clang-tidy has some limitations to ensure reasonable performance:
// - checks don't see all preprocessor events in the preamble
// - matchers run only over the main-file top-level decls (and can't see
// ancestors outside this scope).
// In practice almost all checks work well without modifications.
std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
ast_matchers::MatchFinder CTFinder;
llvm::Optional<tidy::ClangTidyContext> CTContext;
llvm::Optional<IncludeFixer> FixIncludes;
// No need to run clang-tidy or IncludeFixerif we are not going to surface
// diagnostics.
if (PreserveDiags) {
trace::Span Tracer("ClangTidyInit");
tidy::ClangTidyOptions ClangTidyOpts =
getTidyOptionsForFile(Inputs.ClangTidyProvider, Filename);
dlog("ClangTidy configuration for file {0}: {1}", Filename,
tidy::ClangTidyCheckFactories CTFactories;
for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
tidy::ClangTidyGlobalOptions(), ClangTidyOpts));
CTChecks = CTFactories.createChecks(CTContext.getPointer());
llvm::erase_if(CTChecks, [&](const auto &Check) {
return !Check->isLanguageVersionSupported(CTContext->getLangOpts());
Preprocessor *PP = &Clang->getPreprocessor();
for (const auto &Check : CTChecks) {
Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
const Config &Cfg = Config::current();
ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
const clang::Diagnostic &Info) {
if (Cfg.Diagnostics.SuppressAll ||
isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress))
return DiagnosticsEngine::Ignored;
if (!CTChecks.empty()) {
std::string CheckName = CTContext->getCheckName(Info.getID());
bool IsClangTidyDiag = !CheckName.empty();
if (IsClangTidyDiag) {
if (Cfg.Diagnostics.Suppress.contains(CheckName))
return DiagnosticsEngine::Ignored;
// Check for suppression comment. Skip the check for diagnostics not
// in the main file, because we don't want that function to query the
// source buffer for preamble files. For the same reason, we ask
// shouldSuppressDiagnostic to avoid I/O.
// We let suppression comments take precedence over warning-as-error
// to match clang-tidy's behaviour.
bool IsInsideMainFile =
Info.hasSourceManager() &&
isInsideMainFile(Info.getLocation(), Info.getSourceManager());
SmallVector<tidy::ClangTidyError, 1> TidySuppressedErrors;
if (IsInsideMainFile &&
tidy::shouldSuppressDiagnostic(DiagLevel, Info, *CTContext,
/*AllowIO=*/false)) {
// FIXME: should we expose the suppression error (invalid use of
// NOLINT comments)?
return DiagnosticsEngine::Ignored;
// Check for warning-as-error.
if (DiagLevel == DiagnosticsEngine::Warning &&
CTContext->treatAsError(CheckName)) {
return DiagnosticsEngine::Error;
return DiagLevel;
// Add IncludeFixer which can recover diagnostics caused by missing includes
// (e.g. incomplete type) and attach include insertion fixes to diagnostics.
auto BuildDir = VFS->getCurrentWorkingDirectory();
if (Inputs.Index && !BuildDir.getError()) {
auto Style =
getFormatStyleForFile(Filename, Inputs.Contents, *Inputs.TFS);
auto Inserter = std::make_shared<IncludeInserter>(
Filename, Inputs.Contents, Style, BuildDir.get(),
if (Preamble) {
for (const auto &Inc : Preamble->Includes.MainFileIncludes)
FixIncludes.emplace(Filename, Inserter, *Inputs.Index,
ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl,
const clang::Diagnostic &Info) {
return FixIncludes->fix(DiagLevl, Info);
IncludeStructure Includes;
// If we are using a preamble, copy existing includes.
if (Preamble) {
Includes = Preamble->Includes;
Includes.MainFileIncludes = Patch->preambleIncludes();
// Replay the preamble includes so that clang-tidy checks can see them.
ReplayPreamble::attach(Patch->preambleIncludes(), *Clang,
// Important: collectIncludeStructure is registered *after* ReplayPreamble!
// Otherwise we would collect the replayed includes again...
// (We can't *just* use the replayed includes, they don't have Resolved path).
// Copy over the macros in the preamble region of the main file, and combine
// with non-preamble macros below.
MainFileMacros Macros;
if (Preamble)
Macros = Preamble->Macros;
std::vector<PragmaMark> Marks;
// FIXME: We need to patch the marks for stale preambles.
if (Preamble)
Marks = Preamble->Marks;
collectPragmaMarksCallback(Clang->getSourceManager(), Marks));
// Copy over the includes from the preamble, then combine with the
// non-preamble includes below.
CanonicalIncludes CanonIncludes;
if (Preamble)
CanonIncludes = Preamble->CanonIncludes;
std::unique_ptr<CommentHandler> IWYUHandler =
// Collect tokens of the main file.
syntax::TokenCollector CollectTokens(Clang->getPreprocessor());
if (llvm::Error Err = Action->Execute())
log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(),
// We have to consume the tokens before running clang-tidy to avoid collecting
// tokens from running the preprocessor inside the checks (only
// modernize-use-trailing-return-type does that today).
syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
// Makes SelectionTree build much faster.
std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
// AST traversals should exclude the preamble, to avoid performance cliffs.
if (!CTChecks.empty()) {
// Run the AST-dependent part of the clang-tidy checks.
// (The preprocessor part ran already, via PPCallbacks).
trace::Span Tracer("ClangTidyMatch");
// XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
// However Action->EndSourceFile() would destroy the ASTContext!
// So just inform the preprocessor of EOF, while keeping everything alive.
// UnitDiagsConsumer is local, we can not store it in CompilerInstance that
// has a longer lifetime.
Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
// CompilerInstance won't run this callback, do it directly.
llvm::Optional<std::vector<Diag>> Diags;
// FIXME: Also skip generation of diagnostics alltogether to speed up ast
// builds when we are patching a stale preamble.
if (PreserveDiags) {
Diags = CompilerInvocationDiags;
// Add diagnostics from the preamble, if any.
if (Preamble)
Diags->insert(Diags->end(), Preamble->Diags.begin(),
// Finally, add diagnostics coming from the AST.
std::vector<Diag> D = ASTDiags.take(CTContext.getPointer());
Diags->insert(Diags->end(), D.begin(), D.end());
ParsedAST Result(Inputs.Version, std::move(Preamble), std::move(Clang),
std::move(Action), std::move(Tokens), std::move(Macros),
std::move(Marks), std::move(ParsedDecls), std::move(Diags),
std::move(Includes), std::move(CanonIncludes));
if (Result.Diags) {
auto UnusedHeadersDiags =
issueUnusedIncludesDiagnostics(Result, Inputs.Contents);
return Result;
ParsedAST::ParsedAST(ParsedAST &&Other) = default;
ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
ParsedAST::~ParsedAST() {
if (Action) {
// We already notified the PP of end-of-file earlier, so detach it first.
// We must keep it alive until after EndSourceFile(), Sema relies on this.
auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
Clang->setPreprocessor(nullptr); // Detach so we don't send EOF again.
Action->EndSourceFile(); // Destroy ASTContext and Sema.
// Now Sema is gone, it's safe for PP to go out of scope.
ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
const ASTContext &ParsedAST::getASTContext() const {
return Clang->getASTContext();
Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
return Clang->getPreprocessorPtr();
const Preprocessor &ParsedAST::getPreprocessor() const {
return Clang->getPreprocessor();
llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
return LocalTopLevelDecls;
const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
const std::vector<PragmaMark> &ParsedAST::getMarks() const { return Marks; }
std::size_t ParsedAST::getUsedBytes() const {
auto &AST = getASTContext();
// FIXME(ibiryukov): we do not account for the dynamically allocated part of
// Message and Fixes inside each diagnostic.
std::size_t Total = clangd::getUsedBytes(LocalTopLevelDecls) +
(Diags ? clangd::getUsedBytes(*Diags) : 0);
// FIXME: the rest of the function is almost a direct copy-paste from
// libclang's clang_getCXTUResourceUsage. We could share the implementation.
// Sum up various allocators inside the ast context and the preprocessor.
Total += AST.getASTAllocatedMemory();
Total += AST.getSideTableAllocatedMemory();
Total += AST.Idents.getAllocator().getTotalMemory();
Total += AST.Selectors.getTotalMemory();
Total += AST.getSourceManager().getContentCacheSize();
Total += AST.getSourceManager().getDataStructureSizes();
Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
if (ExternalASTSource *Ext = AST.getExternalSource())
Total += Ext->getMemoryBufferSizes().malloc_bytes;
const Preprocessor &PP = getPreprocessor();
Total += PP.getTotalMemory();
if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
Total += PRec->getTotalMemory();
Total += PP.getHeaderSearchInfo().getTotalMemory();
return Total;
const IncludeStructure &ParsedAST::getIncludeStructure() const {
return Includes;
const CanonicalIncludes &ParsedAST::getCanonicalIncludes() const {
return CanonIncludes;
ParsedAST::ParsedAST(llvm::StringRef Version,
std::shared_ptr<const PreambleData> Preamble,
std::unique_ptr<CompilerInstance> Clang,
std::unique_ptr<FrontendAction> Action,
syntax::TokenBuffer Tokens, MainFileMacros Macros,
std::vector<PragmaMark> Marks,
std::vector<Decl *> LocalTopLevelDecls,
llvm::Optional<std::vector<Diag>> Diags,
IncludeStructure Includes, CanonicalIncludes CanonIncludes)
: Version(Version), Preamble(std::move(Preamble)), Clang(std::move(Clang)),
Action(std::move(Action)), Tokens(std::move(Tokens)),
Macros(std::move(Macros)), Marks(std::move(Marks)),
Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
Resolver = std::make_unique<HeuristicResolver>(getASTContext());
llvm::Optional<llvm::StringRef> ParsedAST::preambleVersion() const {
if (!Preamble)
return llvm::None;
return llvm::StringRef(Preamble->Version);
} // namespace clangd
} // namespace clang