blob: 79165e392df6819159e7d5acbc21b420cf2dec01 [file] [log] [blame]
Sam McCallcf3a5852019-09-04 07:35:00 +00001//===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Preamble.h"
Kadir Cetinkayaecd3e672020-03-11 16:34:01 +010010#include "Compiler.h"
Nathan Ridgee6a971b2021-10-25 02:42:11 -040011#include "Config.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020012#include "Headers.h"
Kadir Cetinkaya717bef62020-04-23 17:44:51 +020013#include "SourceCode.h"
Sam McCallad97ccf2020-04-28 17:49:17 +020014#include "support/Logger.h"
Kadir Cetinkaya06287052020-06-17 11:53:32 +020015#include "support/ThreadsafeFS.h"
Sam McCallad97ccf2020-04-28 17:49:17 +020016#include "support/Trace.h"
Sam McCall4160f4c2020-06-09 15:46:35 +020017#include "clang/AST/DeclTemplate.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020018#include "clang/Basic/Diagnostic.h"
Kadir Cetinkayaa46bbc12021-04-14 19:28:14 +020019#include "clang/Basic/DiagnosticLex.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020020#include "clang/Basic/LangOptions.h"
Sam McCallcf3a5852019-09-04 07:35:00 +000021#include "clang/Basic/SourceLocation.h"
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +020022#include "clang/Basic/SourceManager.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020023#include "clang/Basic/TokenKinds.h"
24#include "clang/Frontend/CompilerInvocation.h"
25#include "clang/Frontend/FrontendActions.h"
Kirill Bobyrevcd0ca5a2021-11-26 14:12:35 +010026#include "clang/Lex/HeaderSearch.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020027#include "clang/Lex/Lexer.h"
Sam McCallcf3a5852019-09-04 07:35:00 +000028#include "clang/Lex/PPCallbacks.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020029#include "clang/Lex/Preprocessor.h"
Sam McCallcf3a5852019-09-04 07:35:00 +000030#include "clang/Lex/PreprocessorOptions.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020031#include "clang/Tooling/CompilationDatabase.h"
32#include "llvm/ADT/ArrayRef.h"
Kadir Cetinkayab742eaa2020-04-02 10:53:45 +020033#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/DenseSet.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020035#include "llvm/ADT/IntrusiveRefCntPtr.h"
Kadir Cetinkaya2dc2e47e2020-06-16 12:16:24 +020036#include "llvm/ADT/None.h"
37#include "llvm/ADT/Optional.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020038#include "llvm/ADT/STLExtras.h"
39#include "llvm/ADT/SmallString.h"
Kadir Cetinkaya717bef62020-04-23 17:44:51 +020040#include "llvm/ADT/StringExtras.h"
Kadir Cetinkaya2214b902020-04-02 10:53:23 +020041#include "llvm/ADT/StringRef.h"
42#include "llvm/ADT/StringSet.h"
43#include "llvm/Support/Error.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/FormatVariadic.h"
46#include "llvm/Support/MemoryBuffer.h"
47#include "llvm/Support/Path.h"
48#include "llvm/Support/VirtualFileSystem.h"
49#include "llvm/Support/raw_ostream.h"
50#include <iterator>
51#include <memory>
52#include <string>
53#include <system_error>
54#include <utility>
55#include <vector>
Sam McCallcf3a5852019-09-04 07:35:00 +000056
57namespace clang {
58namespace clangd {
59namespace {
Kadir Cetinkaya538c2752020-05-14 12:26:47 +020060constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h";
Sam McCallcf3a5852019-09-04 07:35:00 +000061
62bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
63 const tooling::CompileCommand &RHS) {
64 // We don't check for Output, it should not matter to clangd.
65 return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
66 llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
67}
68
Sam McCallcf3a5852019-09-04 07:35:00 +000069class CppFilePreambleCallbacks : public PreambleCallbacks {
70public:
71 CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
Haojian Wu7e3c74b2019-09-24 11:14:06 +000072 : File(File), ParsedCallback(ParsedCallback) {}
Sam McCallcf3a5852019-09-04 07:35:00 +000073
74 IncludeStructure takeIncludes() { return std::move(Includes); }
75
Haojian Wu7e3c74b2019-09-24 11:14:06 +000076 MainFileMacros takeMacros() { return std::move(Macros); }
Sam McCallcf3a5852019-09-04 07:35:00 +000077
David Goldmand75fb1e2021-07-12 12:29:48 -040078 std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
79
Sam McCallcf3a5852019-09-04 07:35:00 +000080 CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
81
Sam McCall69c04ef2021-07-17 02:17:44 +020082 bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
83
Sam McCallcf3a5852019-09-04 07:35:00 +000084 void AfterExecute(CompilerInstance &CI) override {
Sam McCall69c04ef2021-07-17 02:17:44 +020085 if (ParsedCallback) {
86 trace::Span Tracer("Running PreambleCallback");
87 ParsedCallback(CI.getASTContext(), CI.getPreprocessorPtr(),
88 CanonIncludes);
89 }
90
91 const SourceManager &SM = CI.getSourceManager();
92 const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID());
93 IsMainFileIncludeGuarded =
94 CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
95 MainFE);
Sam McCallcf3a5852019-09-04 07:35:00 +000096 }
97
98 void BeforeExecute(CompilerInstance &CI) override {
Ilya Biryukov8b767092019-09-09 15:32:51 +000099 CanonIncludes.addSystemHeadersMapping(CI.getLangOpts());
Haojian Wu7e3c74b2019-09-24 11:14:06 +0000100 LangOpts = &CI.getLangOpts();
Sam McCallcf3a5852019-09-04 07:35:00 +0000101 SourceMgr = &CI.getSourceManager();
Kirill Bobyrevcd0ca5a2021-11-26 14:12:35 +0100102 Compiler = &CI;
Sam McCallcf3a5852019-09-04 07:35:00 +0000103 }
104
105 std::unique_ptr<PPCallbacks> createPPCallbacks() override {
Haojian Wu7e3c74b2019-09-24 11:14:06 +0000106 assert(SourceMgr && LangOpts &&
107 "SourceMgr and LangOpts must be set at this point");
108
Sam McCallcf3a5852019-09-04 07:35:00 +0000109 return std::make_unique<PPChainedCallbacks>(
Kirill Bobyrevcd0ca5a2021-11-26 14:12:35 +0100110 Includes.collect(*Compiler),
David Goldmand75fb1e2021-07-12 12:29:48 -0400111 std::make_unique<PPChainedCallbacks>(
112 std::make_unique<CollectMainFileMacros>(*SourceMgr, Macros),
113 collectPragmaMarksCallback(*SourceMgr, Marks)));
Sam McCallcf3a5852019-09-04 07:35:00 +0000114 }
115
116 CommentHandler *getCommentHandler() override {
117 IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
118 return IWYUHandler.get();
119 }
120
Sam McCall4160f4c2020-06-09 15:46:35 +0200121 bool shouldSkipFunctionBody(Decl *D) override {
122 // Generally we skip function bodies in preambles for speed.
123 // We can make exceptions for functions that are cheap to parse and
124 // instantiate, widely used, and valuable (e.g. commonly produce errors).
125 if (const auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
126 if (const auto *II = FT->getDeclName().getAsIdentifierInfo())
127 // std::make_unique is trivial, and we diagnose bad constructor calls.
128 if (II->isStr("make_unique") && FT->isInStdNamespace())
129 return false;
130 }
131 return true;
132 }
133
Sam McCallcf3a5852019-09-04 07:35:00 +0000134private:
135 PathRef File;
136 PreambleParsedCallback ParsedCallback;
137 IncludeStructure Includes;
138 CanonicalIncludes CanonIncludes;
Haojian Wu7e3c74b2019-09-24 11:14:06 +0000139 MainFileMacros Macros;
David Goldmand75fb1e2021-07-12 12:29:48 -0400140 std::vector<PragmaMark> Marks;
Sam McCall69c04ef2021-07-17 02:17:44 +0200141 bool IsMainFileIncludeGuarded = false;
Sam McCallcf3a5852019-09-04 07:35:00 +0000142 std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
Haojian Wu7e3c74b2019-09-24 11:14:06 +0000143 const clang::LangOptions *LangOpts = nullptr;
144 const SourceManager *SourceMgr = nullptr;
Kirill Bobyrevcd0ca5a2021-11-26 14:12:35 +0100145 const CompilerInstance *Compiler = nullptr;
Sam McCallcf3a5852019-09-04 07:35:00 +0000146};
147
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200148// Represents directives other than includes, where basic textual information is
149// enough.
150struct TextualPPDirective {
151 unsigned DirectiveLine;
152 // Full text that's representing the directive, including the `#`.
153 std::string Text;
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200154 unsigned Offset;
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200155
156 bool operator==(const TextualPPDirective &RHS) const {
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200157 return std::tie(DirectiveLine, Offset, Text) ==
158 std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200159 }
160};
161
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200162// Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
163// 10"). The formatting is copied so that the tokens in Body have PresumedLocs
164// with correct columns and lines.
165std::string spellDirective(llvm::StringRef Prefix,
166 CharSourceRange DirectiveRange,
167 const LangOptions &LangOpts, const SourceManager &SM,
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200168 unsigned &DirectiveLine, unsigned &Offset) {
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200169 std::string SpelledDirective;
170 llvm::raw_string_ostream OS(SpelledDirective);
171 OS << Prefix;
172
173 // Make sure DirectiveRange is a char range and doesn't contain macro ids.
174 DirectiveRange = SM.getExpansionRange(DirectiveRange);
175 if (DirectiveRange.isTokenRange()) {
176 DirectiveRange.setEnd(
177 Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
178 }
179
180 auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
181 DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200182 Offset = DecompLoc.second;
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200183 auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
184
185 // Pad with spaces before DirectiveRange to make sure it will be on right
186 // column when patched.
187 if (Prefix.size() <= TargetColumn) {
188 // There is enough space for Prefix and space before directive, use it.
189 // We try to squeeze the Prefix into the same line whenever we can, as
190 // putting onto a separate line won't work at the beginning of the file.
191 OS << std::string(TargetColumn - Prefix.size(), ' ');
192 } else {
193 // Prefix was longer than the space we had. We produce e.g.:
194 // #line N-1
195 // #define \
196 // X 10
197 OS << "\\\n" << std::string(TargetColumn, ' ');
198 // Decrement because we put an additional line break before
199 // DirectiveRange.begin().
200 --DirectiveLine;
201 }
202 OS << toSourceCode(SM, DirectiveRange.getAsRange());
203 return OS.str();
204}
205
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200206// Collects #define directives inside the main file.
207struct DirectiveCollector : public PPCallbacks {
208 DirectiveCollector(const Preprocessor &PP,
209 std::vector<TextualPPDirective> &TextualDirectives)
210 : LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
211 TextualDirectives(TextualDirectives) {}
212
213 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
214 SrcMgr::CharacteristicKind FileType,
215 FileID PrevFID) override {
216 InMainFile = SM.isWrittenInMainFile(Loc);
217 }
218
219 void MacroDefined(const Token &MacroNameTok,
220 const MacroDirective *MD) override {
221 if (!InMainFile)
222 return;
223 TextualDirectives.emplace_back();
224 TextualPPDirective &TD = TextualDirectives.back();
225
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200226 const auto *MI = MD->getMacroInfo();
227 TD.Text =
228 spellDirective("#define ",
229 CharSourceRange::getTokenRange(
230 MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200231 LangOpts, SM, TD.DirectiveLine, TD.Offset);
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200232 }
233
234private:
235 bool InMainFile = true;
236 const LangOptions &LangOpts;
237 const SourceManager &SM;
238 std::vector<TextualPPDirective> &TextualDirectives;
239};
240
241struct ScannedPreamble {
242 std::vector<Inclusion> Includes;
243 std::vector<TextualPPDirective> TextualDirectives;
Kadir Cetinkaya4317ee22020-06-16 21:21:45 +0200244 PreambleBounds Bounds = {0, false};
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200245};
246
247/// Scans the preprocessor directives in the preamble section of the file by
248/// running preprocessor over \p Contents. Returned includes do not contain
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200249/// resolved paths. \p Cmd is used to build the compiler invocation, which might
250/// stat/read files.
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200251llvm::Expected<ScannedPreamble>
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200252scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
253 class EmptyFS : public ThreadsafeFS {
Sam McCall72568982020-06-29 19:52:09 +0200254 private:
255 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200256 return new llvm::vfs::InMemoryFileSystem;
257 }
Kadir Cetinkayaf693ce42020-06-04 18:26:52 +0200258 };
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200259 EmptyFS FS;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200260 // Build and run Preprocessor over the preamble.
261 ParseInputs PI;
262 PI.Contents = Contents.str();
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200263 PI.TFS = &FS;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200264 PI.CompileCommand = Cmd;
265 IgnoringDiagConsumer IgnoreDiags;
266 auto CI = buildCompilerInvocation(PI, IgnoreDiags);
267 if (!CI)
Sam McCall687e1d72020-09-14 11:33:12 +0200268 return error("failed to create compiler invocation");
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200269 CI->getDiagnosticOpts().IgnoreWarnings = true;
270 auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents);
Kadir Cetinkaya34e39eb2020-05-05 17:55:11 +0200271 // This means we're scanning (though not preprocessing) the preamble section
272 // twice. However, it's important to precisely follow the preamble bounds used
273 // elsewhere.
Duncan P. N. Exon Smith4c55c3b2020-11-05 12:43:58 -0500274 auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
Kadir Cetinkaya34e39eb2020-05-05 17:55:11 +0200275 auto PreambleContents =
276 llvm::MemoryBuffer::getMemBufferCopy(Contents.substr(0, Bounds.Size));
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200277 auto Clang = prepareCompilerInstance(
Kadir Cetinkaya34e39eb2020-05-05 17:55:11 +0200278 std::move(CI), nullptr, std::move(PreambleContents),
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200279 // Provide an empty FS to prevent preprocessor from performing IO. This
280 // also implies missing resolved paths for includes.
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200281 FS.view(llvm::None), IgnoreDiags);
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200282 if (Clang->getFrontendOpts().Inputs.empty())
Sam McCall687e1d72020-09-14 11:33:12 +0200283 return error("compiler instance had no inputs");
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200284 // We are only interested in main file includes.
285 Clang->getPreprocessorOpts().SingleFileParseMode = true;
Kadir Cetinkaya34e39eb2020-05-05 17:55:11 +0200286 PreprocessOnlyAction Action;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200287 if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
Sam McCall687e1d72020-09-14 11:33:12 +0200288 return error("failed BeginSourceFile");
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200289 Preprocessor &PP = Clang->getPreprocessor();
290 IncludeStructure Includes;
Kirill Bobyrevcd0ca5a2021-11-26 14:12:35 +0100291 PP.addPPCallbacks(Includes.collect(*Clang));
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200292 ScannedPreamble SP;
Kadir Cetinkaya4317ee22020-06-16 21:21:45 +0200293 SP.Bounds = Bounds;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200294 PP.addPPCallbacks(
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200295 std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200296 if (llvm::Error Err = Action.Execute())
297 return std::move(Err);
298 Action.EndSourceFile();
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200299 SP.Includes = std::move(Includes.MainFileIncludes);
300 return SP;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200301}
302
303const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
304 switch (IncludeDirective) {
305 case tok::pp_include:
306 return "include";
307 case tok::pp_import:
308 return "import";
309 case tok::pp_include_next:
310 return "include_next";
311 default:
312 break;
313 }
314 llvm_unreachable("not an include directive");
315}
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200316
317// Checks whether \p FileName is a valid spelling of main file.
318bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
319 auto FE = SM.getFileManager().getFile(FileName);
320 return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
321}
322
Sam McCallcf3a5852019-09-04 07:35:00 +0000323} // namespace
324
Sam McCallcf3a5852019-09-04 07:35:00 +0000325std::shared_ptr<const PreambleData>
Kadir Cetinkaya276a95b2020-03-13 11:52:19 +0100326buildPreamble(PathRef FileName, CompilerInvocation CI,
Sam McCallcf3a5852019-09-04 07:35:00 +0000327 const ParseInputs &Inputs, bool StoreInMemory,
328 PreambleParsedCallback PreambleCallback) {
329 // Note that we don't need to copy the input contents, preamble can live
330 // without those.
331 auto ContentsBuffer =
332 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
Duncan P. N. Exon Smith4c55c3b2020-11-05 12:43:58 -0500333 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
Sam McCallcf3a5852019-09-04 07:35:00 +0000334
Sam McCallcf3a5852019-09-04 07:35:00 +0000335 trace::Span Tracer("BuildPreamble");
336 SPAN_ATTACH(Tracer, "File", FileName);
Kadir Cetinkayabce3ac42021-03-12 14:23:45 +0100337 std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
338 if (Inputs.FeatureModules) {
339 for (auto &M : *Inputs.FeatureModules) {
340 if (auto Listener = M.astListeners())
341 ASTListeners.emplace_back(std::move(Listener));
342 }
343 }
Sam McCallcf3a5852019-09-04 07:35:00 +0000344 StoreDiags PreambleDiagnostics;
Kadir Cetinkayabce3ac42021-03-12 14:23:45 +0100345 PreambleDiagnostics.setDiagCallback(
346 [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
347 llvm::for_each(ASTListeners,
348 [&](const auto &L) { L->sawDiagnostic(D, Diag); });
349 });
Sam McCallcf3a5852019-09-04 07:35:00 +0000350 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
351 CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
352 &PreambleDiagnostics, false);
Nathan Ridgee6a971b2021-10-25 02:42:11 -0400353 const Config &Cfg = Config::current();
354 PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
355 const clang::Diagnostic &Info) {
356 if (Cfg.Diagnostics.SuppressAll ||
357 isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress))
358 return DiagnosticsEngine::Ignored;
359 switch (Info.getID()) {
360 case diag::warn_no_newline_eof:
361 case diag::warn_cxx98_compat_no_newline_eof:
362 case diag::ext_no_newline_eof:
363 // If the preamble doesn't span the whole file, drop the no newline at
364 // eof warnings.
365 return Bounds.Size != ContentsBuffer->getBufferSize()
366 ? DiagnosticsEngine::Level::Ignored
367 : DiagLevel;
368 }
369 return DiagLevel;
370 });
Sam McCallcf3a5852019-09-04 07:35:00 +0000371
372 // Skip function bodies when building the preamble to speed up building
373 // the preamble and make it smaller.
374 assert(!CI.getFrontendOpts().SkipFunctionBodies);
375 CI.getFrontendOpts().SkipFunctionBodies = true;
376 // We don't want to write comment locations into PCH. They are racy and slow
377 // to read back. We rely on dynamic index for the comments instead.
378 CI.getPreprocessorOpts().WriteCommentListToPCH = false;
379
Sam McCall69c04ef2021-07-17 02:17:44 +0200380 CppFilePreambleCallbacks CapturedInfo(FileName, PreambleCallback);
Kadir Cetinkaya8d654df52020-06-17 18:09:54 +0200381 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
Sam McCallcf3a5852019-09-04 07:35:00 +0000382 llvm::SmallString<32> AbsFileName(FileName);
Kadir Cetinkayaf693ce42020-06-04 18:26:52 +0200383 VFS->makeAbsolute(AbsFileName);
Sam McCallcf3a5852019-09-04 07:35:00 +0000384 auto StatCache = std::make_unique<PreambleFileStatusCache>(AbsFileName);
385 auto BuiltPreamble = PrecompiledPreamble::Build(
386 CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
Kadir Cetinkayaf693ce42020-06-04 18:26:52 +0200387 StatCache->getProducingFS(VFS),
Sam McCall69c04ef2021-07-17 02:17:44 +0200388 std::make_shared<PCHContainerOperations>(), StoreInMemory, CapturedInfo);
Sam McCallcf3a5852019-09-04 07:35:00 +0000389
390 // When building the AST for the main file, we do want the function
391 // bodies.
392 CI.getFrontendOpts().SkipFunctionBodies = false;
393
394 if (BuiltPreamble) {
Sam McCall2cd33e62020-03-04 00:33:29 +0100395 vlog("Built preamble of size {0} for file {1} version {2}",
396 BuiltPreamble->getSize(), FileName, Inputs.Version);
Sam McCallcf3a5852019-09-04 07:35:00 +0000397 std::vector<Diag> Diags = PreambleDiagnostics.take();
Sam McCall91670f52021-07-21 11:31:52 +0200398 auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
399 Result->Version = Inputs.Version;
400 Result->CompileCommand = Inputs.CompileCommand;
401 Result->Diags = std::move(Diags);
402 Result->Includes = CapturedInfo.takeIncludes();
403 Result->Macros = CapturedInfo.takeMacros();
David Goldmand75fb1e2021-07-12 12:29:48 -0400404 Result->Marks = CapturedInfo.takeMarks();
Sam McCall91670f52021-07-21 11:31:52 +0200405 Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes();
406 Result->StatCache = std::move(StatCache);
Sam McCall69c04ef2021-07-17 02:17:44 +0200407 Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
408 return Result;
Sam McCallcf3a5852019-09-04 07:35:00 +0000409 }
Sam McCall69c04ef2021-07-17 02:17:44 +0200410
411 elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
412 Inputs.Version, BuiltPreamble.getError().message());
413 return nullptr;
Sam McCallcf3a5852019-09-04 07:35:00 +0000414}
415
Kadir Cetinkayac31367e2020-03-15 21:43:00 +0100416bool isPreambleCompatible(const PreambleData &Preamble,
417 const ParseInputs &Inputs, PathRef FileName,
418 const CompilerInvocation &CI) {
419 auto ContentsBuffer =
420 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
Duncan P. N. Exon Smith4c55c3b2020-11-05 12:43:58 -0500421 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
Kadir Cetinkaya8d654df52020-06-17 18:09:54 +0200422 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
Kadir Cetinkayac31367e2020-03-15 21:43:00 +0100423 return compileCommandsAreEqual(Inputs.CompileCommand,
424 Preamble.CompileCommand) &&
Duncan P. N. Exon Smithf4d02fb2020-11-10 10:54:03 -0500425 Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
Kadir Cetinkayac31367e2020-03-15 21:43:00 +0100426}
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200427
Kadir Cetinkaya717bef62020-04-23 17:44:51 +0200428void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
429 for (char C : Text) {
430 switch (C) {
431 case '\\':
432 case '"':
433 OS << '\\';
434 break;
435 default:
436 break;
437 }
438 OS << C;
439 }
440}
441
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200442PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
443 const ParseInputs &Modified,
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200444 const PreambleData &Baseline,
445 PatchType PatchType) {
Kadir Cetinkaya20b2af32020-05-29 12:31:35 +0200446 trace::Span Tracer("CreatePreamblePatch");
447 SPAN_ATTACH(Tracer, "File", FileName);
Kadir Cetinkayab742eaa2020-04-02 10:53:45 +0200448 assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200449 // First scan preprocessor directives in Baseline and Modified. These will be
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200450 // used to figure out newly added directives in Modified. Scanning can fail,
451 // the code just bails out and creates an empty patch in such cases, as:
452 // - If scanning for Baseline fails, no knowledge of existing includes hence
453 // patch will contain all the includes in Modified. Leading to rebuild of
454 // whole preamble, which is terribly slow.
455 // - If scanning for Modified fails, cannot figure out newly added ones so
456 // there's nothing to do but generate an empty patch.
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200457 auto BaselineScan = scanPreamble(
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200458 // Contents needs to be null-terminated.
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200459 Baseline.Preamble.getContents().str(), Modified.CompileCommand);
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200460 if (!BaselineScan) {
461 elog("Failed to scan baseline of {0}: {1}", FileName,
462 BaselineScan.takeError());
463 return PreamblePatch::unmodified(Baseline);
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200464 }
Kadir Cetinkayad2fcc582020-06-17 18:09:54 +0200465 auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200466 if (!ModifiedScan) {
467 elog("Failed to scan modified contents of {0}: {1}", FileName,
468 ModifiedScan.takeError());
469 return PreamblePatch::unmodified(Baseline);
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200470 }
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200471
472 bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
473 bool DirectivesChanged =
474 BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
Kadir Cetinkaya64fe0452021-09-16 11:30:55 +0200475 if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
476 !DirectivesChanged)
Kadir Cetinkayab742eaa2020-04-02 10:53:45 +0200477 return PreamblePatch::unmodified(Baseline);
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200478
479 PreamblePatch PP;
480 // This shouldn't coincide with any real file name.
481 llvm::SmallString<128> PatchName;
482 llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200483 PreamblePatchHeaderName);
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200484 PP.PatchFileName = PatchName.str().str();
Kadir Cetinkaya4317ee22020-06-16 21:21:45 +0200485 PP.ModifiedBounds = ModifiedScan->Bounds;
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200486
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200487 llvm::raw_string_ostream Patch(PP.PatchContents);
Kadir Cetinkaya717bef62020-04-23 17:44:51 +0200488 // Set default filename for subsequent #line directives
489 Patch << "#line 0 \"";
490 // FileName part of a line directive is subject to backslash escaping, which
491 // might lead to problems on windows especially.
492 escapeBackslashAndQuotes(FileName, Patch);
493 Patch << "\"\n";
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200494
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200495 if (IncludesChanged && PatchType == PatchType::All) {
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200496 // We are only interested in newly added includes, record the ones in
497 // Baseline for exclusion.
498 llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
499 /*Resolved=*/llvm::StringRef>
500 ExistingIncludes;
501 for (const auto &Inc : Baseline.Includes.MainFileIncludes)
502 ExistingIncludes[{Inc.Directive, Inc.Written}] = Inc.Resolved;
503 // There might be includes coming from disabled regions, record these for
504 // exclusion too. note that we don't have resolved paths for those.
505 for (const auto &Inc : BaselineScan->Includes)
506 ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
507 // Calculate extra includes that needs to be inserted.
508 for (auto &Inc : ModifiedScan->Includes) {
509 auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
510 // Include already present in the baseline preamble. Set resolved path and
511 // put into preamble includes.
512 if (It != ExistingIncludes.end()) {
513 Inc.Resolved = It->second.str();
514 PP.PreambleIncludes.push_back(Inc);
515 continue;
516 }
517 // Include is new in the modified preamble. Inject it into the patch and
518 // use #line to set the presumed location to where it is spelled.
519 auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
520 Patch << llvm::formatv("#line {0}\n", LineCol.first);
521 Patch << llvm::formatv(
522 "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
523 }
524 }
525
526 if (DirectivesChanged) {
527 // We need to patch all the directives, since they are order dependent. e.g:
528 // #define BAR(X) NEW(X) // Newly introduced in Modified
529 // #define BAR(X) OLD(X) // Exists in the Baseline
530 //
531 // If we've patched only the first directive, the macro definition would've
532 // been wrong for the rest of the file, since patch is applied after the
533 // baseline preamble.
534 //
535 // Note that we deliberately ignore conditional directives and undefs to
536 // reduce complexity. The former might cause problems because scanning is
537 // imprecise and might pick directives from disabled regions.
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200538 for (const auto &TD : ModifiedScan->TextualDirectives) {
539 Patch << "#line " << TD.DirectiveLine << '\n';
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200540 Patch << TD.Text << '\n';
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200541 }
Kadir Cetinkayafcde3d52020-05-14 12:20:33 +0200542 }
543 dlog("Created preamble patch: {0}", Patch.str());
544 Patch.flush();
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200545 return PP;
546}
547
Queen Dela Cruz9c4a1682021-09-16 10:17:37 +0200548PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName,
549 const ParseInputs &Modified,
550 const PreambleData &Baseline) {
551 return create(FileName, Modified, Baseline, PatchType::All);
552}
553
554PreamblePatch PreamblePatch::createMacroPatch(llvm::StringRef FileName,
555 const ParseInputs &Modified,
556 const PreambleData &Baseline) {
557 return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
558}
559
Kadir Cetinkaya2214b902020-04-02 10:53:23 +0200560void PreamblePatch::apply(CompilerInvocation &CI) const {
561 // No need to map an empty file.
562 if (PatchContents.empty())
563 return;
564 auto &PPOpts = CI.getPreprocessorOpts();
565 auto PatchBuffer =
566 // we copy here to ensure contents are still valid if CI outlives the
567 // PreamblePatch.
568 llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
569 // CI will take care of the lifetime of the buffer.
570 PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
571 // The patch will be parsed after loading the preamble ast and before parsing
572 // the main file.
573 PPOpts.Includes.push_back(PatchFileName);
574}
575
Kadir Cetinkayab742eaa2020-04-02 10:53:45 +0200576std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
577 return PreambleIncludes;
578}
579
580PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
581 PreamblePatch PP;
582 PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
Kadir Cetinkaya4317ee22020-06-16 21:21:45 +0200583 PP.ModifiedBounds = Preamble.Preamble.getBounds();
Kadir Cetinkayab742eaa2020-04-02 10:53:45 +0200584 return PP;
585}
586
Kadir Cetinkaya538c2752020-05-14 12:26:47 +0200587SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
588 const SourceManager &SM) {
589 auto DefFile = SM.getFileID(Loc);
590 if (auto *FE = SM.getFileEntryForID(DefFile)) {
591 auto IncludeLoc = SM.getIncludeLoc(DefFile);
592 // Preamble patch is included inside the builtin file.
593 if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
594 FE->getName().endswith(PreamblePatchHeaderName)) {
595 auto Presumed = SM.getPresumedLoc(Loc);
596 // Check that line directive is pointing at main file.
597 if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
598 isMainFile(Presumed.getFilename(), SM)) {
599 Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
600 Presumed.getColumn());
601 }
602 }
603 }
604 return Loc;
605}
Sam McCallcf3a5852019-09-04 07:35:00 +0000606} // namespace clangd
607} // namespace clang