Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/Preprocessor.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
//  This file implements the Preprocessor interface.
10
//
11
//===----------------------------------------------------------------------===//
12
//
13
// Options to support:
14
//   -H       - Print the name of each header file used.
15
//   -d[DNI] - Dump various things.
16
//   -fworking-directory - #line's with preprocessor's working dir.
17
//   -fpreprocessed
18
//   -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19
//   -W*
20
//   -w
21
//
22
// Messages to emit:
23
//   "Multiple include guards may be useful for:\n"
24
//
25
//===----------------------------------------------------------------------===//
26
27
#include "clang/Lex/Preprocessor.h"
28
#include "clang/Basic/Builtins.h"
29
#include "clang/Basic/FileManager.h"
30
#include "clang/Basic/FileSystemStatCache.h"
31
#include "clang/Basic/IdentifierTable.h"
32
#include "clang/Basic/LLVM.h"
33
#include "clang/Basic/LangOptions.h"
34
#include "clang/Basic/Module.h"
35
#include "clang/Basic/SourceLocation.h"
36
#include "clang/Basic/SourceManager.h"
37
#include "clang/Basic/TargetInfo.h"
38
#include "clang/Lex/CodeCompletionHandler.h"
39
#include "clang/Lex/ExternalPreprocessorSource.h"
40
#include "clang/Lex/HeaderSearch.h"
41
#include "clang/Lex/LexDiagnostic.h"
42
#include "clang/Lex/Lexer.h"
43
#include "clang/Lex/LiteralSupport.h"
44
#include "clang/Lex/MacroArgs.h"
45
#include "clang/Lex/MacroInfo.h"
46
#include "clang/Lex/ModuleLoader.h"
47
#include "clang/Lex/Pragma.h"
48
#include "clang/Lex/PreprocessingRecord.h"
49
#include "clang/Lex/PreprocessorLexer.h"
50
#include "clang/Lex/PreprocessorOptions.h"
51
#include "clang/Lex/ScratchBuffer.h"
52
#include "clang/Lex/Token.h"
53
#include "clang/Lex/TokenLexer.h"
54
#include "llvm/ADT/APInt.h"
55
#include "llvm/ADT/ArrayRef.h"
56
#include "llvm/ADT/DenseMap.h"
57
#include "llvm/ADT/STLExtras.h"
58
#include "llvm/ADT/SmallString.h"
59
#include "llvm/ADT/SmallVector.h"
60
#include "llvm/ADT/StringRef.h"
61
#include "llvm/ADT/StringSwitch.h"
62
#include "llvm/Support/Capacity.h"
63
#include "llvm/Support/ErrorHandling.h"
64
#include "llvm/Support/MemoryBuffer.h"
65
#include "llvm/Support/raw_ostream.h"
66
#include <algorithm>
67
#include <cassert>
68
#include <memory>
69
#include <string>
70
#include <utility>
71
#include <vector>
72
73
using namespace clang;
74
75
LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
76
77
7.80k
ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
78
79
Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
80
                           DiagnosticsEngine &diags, LangOptions &opts,
81
                           SourceManager &SM, HeaderSearch &Headers,
82
                           ModuleLoader &TheModuleLoader,
83
                           IdentifierInfoLookup *IILookup, bool OwnsHeaders,
84
                           TranslationUnitKind TUKind)
85
    : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86
      FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87
      ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88
      TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
89
      // As the language options may have not been loaded yet (when
90
      // deserializing an ASTUnit), adding keywords to the identifier table is
91
      // deferred to Preprocessor::Initialize().
92
      Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
93
      TUKind(TUKind), SkipMainFilePreamble(0, true),
94
59.4k
      CurSubmoduleState(&NullSubmoduleState) {
95
59.4k
  OwnsHeaderSearch = OwnsHeaders;
96
59.4k
97
59.4k
  // Default to discarding comments.
98
59.4k
  KeepComments = false;
99
59.4k
  KeepMacroComments = false;
100
59.4k
  SuppressIncludeNotFoundError = false;
101
59.4k
102
59.4k
  // Macro expansion is enabled.
103
59.4k
  DisableMacroExpansion = false;
104
59.4k
  MacroExpansionInDirectivesOverride = false;
105
59.4k
  InMacroArgs = false;
106
59.4k
  ArgMacro = nullptr;
107
59.4k
  InMacroArgPreExpansion = false;
108
59.4k
  NumCachedTokenLexers = 0;
109
59.4k
  PragmasEnabled = true;
110
59.4k
  ParsingIfOrElifDirective = false;
111
59.4k
  PreprocessedOutput = false;
112
59.4k
113
59.4k
  // We haven't read anything from the external source.
114
59.4k
  ReadMacrosFromExternalSource = false;
115
59.4k
116
59.4k
  BuiltinInfo = std::make_unique<Builtin::Context>();
117
59.4k
118
59.4k
  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
119
59.4k
  // a macro. They get unpoisoned where it is allowed.
120
59.4k
  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
121
59.4k
  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
122
59.4k
  if (getLangOpts().CPlusPlus2a) {
123
385
    (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
124
385
    SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
125
59.0k
  } else {
126
59.0k
    Ident__VA_OPT__ = nullptr;
127
59.0k
  }
128
59.4k
129
59.4k
  // Initialize the pragma handlers.
130
59.4k
  RegisterBuiltinPragmas();
131
59.4k
132
59.4k
  // Initialize builtin macros like __LINE__ and friends.
133
59.4k
  RegisterBuiltinMacros();
134
59.4k
135
59.4k
  if(LangOpts.Borland) {
136
6
    Ident__exception_info        = getIdentifierInfo("_exception_info");
137
6
    Ident___exception_info       = getIdentifierInfo("__exception_info");
138
6
    Ident_GetExceptionInfo       = getIdentifierInfo("GetExceptionInformation");
139
6
    Ident__exception_code        = getIdentifierInfo("_exception_code");
140
6
    Ident___exception_code       = getIdentifierInfo("__exception_code");
141
6
    Ident_GetExceptionCode       = getIdentifierInfo("GetExceptionCode");
142
6
    Ident__abnormal_termination  = getIdentifierInfo("_abnormal_termination");
143
6
    Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
144
6
    Ident_AbnormalTermination    = getIdentifierInfo("AbnormalTermination");
145
59.4k
  } else {
146
59.4k
    Ident__exception_info = Ident__exception_code = nullptr;
147
59.4k
    Ident__abnormal_termination = Ident___exception_info = nullptr;
148
59.4k
    Ident___exception_code = Ident___abnormal_termination = nullptr;
149
59.4k
    Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
150
59.4k
    Ident_AbnormalTermination = nullptr;
151
59.4k
  }
152
59.4k
153
59.4k
  // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
154
59.4k
  if (usingPCHWithPragmaHdrStop())
155
8
    SkippingUntilPragmaHdrStop = true;
156
59.4k
157
59.4k
  // If using a PCH with a through header, start skipping tokens.
158
59.4k
  if (!this->PPOpts->PCHThroughHeader.empty() &&
159
59.4k
      
!this->PPOpts->ImplicitPCHInclude.empty()32
)
160
17
    SkippingUntilPCHThroughHeader = true;
161
59.4k
162
59.4k
  if (this->PPOpts->GeneratePreamble)
163
89
    PreambleConditionalStack.startRecording();
164
59.4k
165
59.4k
  ExcludedConditionalDirectiveSkipMappings =
166
59.4k
      this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
167
59.4k
  if (ExcludedConditionalDirectiveSkipMappings)
168
51
    ExcludedConditionalDirectiveSkipMappings->clear();
169
59.4k
170
59.4k
  MaxTokens = LangOpts.MaxTokens;
171
59.4k
}
172
173
51.7k
Preprocessor::~Preprocessor() {
174
51.7k
  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
175
51.7k
176
51.7k
  IncludeMacroStack.clear();
177
51.7k
178
51.7k
  // Destroy any macro definitions.
179
24.5M
  while (MacroInfoChain *I = MIChainHead) {
180
24.5M
    MIChainHead = I->Next;
181
24.5M
    I->~MacroInfoChain();
182
24.5M
  }
183
51.7k
184
51.7k
  // Free any cached macro expanders.
185
51.7k
  // This populates MacroArgCache, so all TokenLexers need to be destroyed
186
51.7k
  // before the code below that frees up the MacroArgCache list.
187
51.7k
  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
188
51.7k
  CurTokenLexer.reset();
189
51.7k
190
51.7k
  // Free any cached MacroArgs.
191
71.0k
  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
192
19.2k
    ArgList = ArgList->deallocate();
193
51.7k
194
51.7k
  // Delete the header search info, if we own it.
195
51.7k
  if (OwnsHeaderSearch)
196
51.6k
    delete &HeaderInfo;
197
51.7k
}
198
199
void Preprocessor::Initialize(const TargetInfo &Target,
200
59.4k
                              const TargetInfo *AuxTarget) {
201
59.4k
  assert((!this->Target || this->Target == &Target) &&
202
59.4k
         "Invalid override of target information");
203
59.4k
  this->Target = &Target;
204
59.4k
205
59.4k
  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
206
59.4k
         "Invalid override of aux target information.");
207
59.4k
  this->AuxTarget = AuxTarget;
208
59.4k
209
59.4k
  // Initialize information about built-ins.
210
59.4k
  BuiltinInfo->InitializeTarget(Target, AuxTarget);
211
59.4k
  HeaderInfo.setTarget(Target);
212
59.4k
213
59.4k
  // Populate the identifier table with info about keywords for the current language.
214
59.4k
  Identifiers.AddKeywords(LangOpts);
215
59.4k
}
216
217
2
void Preprocessor::InitializeForModelFile() {
218
2
  NumEnteredSourceFiles = 0;
219
2
220
2
  // Reset pragmas
221
2
  PragmaHandlersBackup = std::move(PragmaHandlers);
222
2
  PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
223
2
  RegisterBuiltinPragmas();
224
2
225
2
  // Reset PredefinesFileID
226
2
  PredefinesFileID = FileID();
227
2
}
228
229
2
void Preprocessor::FinalizeForModelFile() {
230
2
  NumEnteredSourceFiles = 1;
231
2
232
2
  PragmaHandlers = std::move(PragmaHandlersBackup);
233
2
}
234
235
7
void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
236
7
  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
237
7
               << getSpelling(Tok) << "'";
238
7
239
7
  if (!DumpFlags) 
return0
;
240
7
241
7
  llvm::errs() << "\t";
242
7
  if (Tok.isAtStartOfLine())
243
3
    llvm::errs() << " [StartOfLine]";
244
7
  if (Tok.hasLeadingSpace())
245
0
    llvm::errs() << " [LeadingSpace]";
246
7
  if (Tok.isExpandDisabled())
247
0
    llvm::errs() << " [ExpandDisabled]";
248
7
  if (Tok.needsCleaning()) {
249
0
    const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
250
0
    llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
251
0
                 << "']";
252
0
  }
253
7
254
7
  llvm::errs() << "\tLoc=<";
255
7
  DumpLocation(Tok.getLocation());
256
7
  llvm::errs() << ">";
257
7
}
258
259
7
void Preprocessor::DumpLocation(SourceLocation Loc) const {
260
7
  Loc.print(llvm::errs(), SourceMgr);
261
7
}
262
263
0
void Preprocessor::DumpMacro(const MacroInfo &MI) const {
264
0
  llvm::errs() << "MACRO: ";
265
0
  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
266
0
    DumpToken(MI.getReplacementToken(i));
267
0
    llvm::errs() << "  ";
268
0
  }
269
0
  llvm::errs() << "\n";
270
0
}
271
272
3
void Preprocessor::PrintStats() {
273
3
  llvm::errs() << "\n*** Preprocessor Stats:\n";
274
3
  llvm::errs() << NumDirectives << " directives found:\n";
275
3
  llvm::errs() << "  " << NumDefined << " #define.\n";
276
3
  llvm::errs() << "  " << NumUndefined << " #undef.\n";
277
3
  llvm::errs() << "  #include/#include_next/#import:\n";
278
3
  llvm::errs() << "    " << NumEnteredSourceFiles << " source files entered.\n";
279
3
  llvm::errs() << "    " << MaxIncludeStackDepth << " max include stack depth\n";
280
3
  llvm::errs() << "  " << NumIf << " #if/#ifndef/#ifdef.\n";
281
3
  llvm::errs() << "  " << NumElse << " #else/#elif.\n";
282
3
  llvm::errs() << "  " << NumEndif << " #endif.\n";
283
3
  llvm::errs() << "  " << NumPragma << " #pragma.\n";
284
3
  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
285
3
286
3
  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
287
3
             << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
288
3
             << NumFastMacroExpanded << " on the fast path.\n";
289
3
  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
290
3
             << " token paste (##) operations performed, "
291
3
             << NumFastTokenPaste << " on the fast path.\n";
292
3
293
3
  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
294
3
295
3
  llvm::errs() << "\n  BumpPtr: " << BP.getTotalMemory();
296
3
  llvm::errs() << "\n  Macro Expanded Tokens: "
297
3
               << llvm::capacity_in_bytes(MacroExpandedTokens);
298
3
  llvm::errs() << "\n  Predefines Buffer: " << Predefines.capacity();
299
3
  // FIXME: List information for all submodules.
300
3
  llvm::errs() << "\n  Macros: "
301
3
               << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
302
3
  llvm::errs() << "\n  #pragma push_macro Info: "
303
3
               << llvm::capacity_in_bytes(PragmaPushMacroInfo);
304
3
  llvm::errs() << "\n  Poison Reasons: "
305
3
               << llvm::capacity_in_bytes(PoisonReasons);
306
3
  llvm::errs() << "\n  Comment Handlers: "
307
3
               << llvm::capacity_in_bytes(CommentHandlers) << "\n";
308
3
}
309
310
Preprocessor::macro_iterator
311
2.23k
Preprocessor::macro_begin(bool IncludeExternalMacros) const {
312
2.23k
  if (IncludeExternalMacros && 
ExternalSource2.23k
&&
313
2.23k
      
!ReadMacrosFromExternalSource412
) {
314
184
    ReadMacrosFromExternalSource = true;
315
184
    ExternalSource->ReadDefinedMacros();
316
184
  }
317
2.23k
318
2.23k
  // Make sure we cover all macros in visible modules.
319
2.23k
  for (const ModuleMacro &Macro : ModuleMacros)
320
4.70M
    CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
321
2.23k
322
2.23k
  return CurSubmoduleState->Macros.begin();
323
2.23k
}
324
325
4
size_t Preprocessor::getTotalMemory() const {
326
4
  return BP.getTotalMemory()
327
4
    + llvm::capacity_in_bytes(MacroExpandedTokens)
328
4
    + Predefines.capacity() /* Predefines buffer. */
329
4
    // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
330
4
    // and ModuleMacros.
331
4
    + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
332
4
    + llvm::capacity_in_bytes(PragmaPushMacroInfo)
333
4
    + llvm::capacity_in_bytes(PoisonReasons)
334
4
    + llvm::capacity_in_bytes(CommentHandlers);
335
4
}
336
337
Preprocessor::macro_iterator
338
2.23k
Preprocessor::macro_end(bool IncludeExternalMacros) const {
339
2.23k
  if (IncludeExternalMacros && 
ExternalSource2.23k
&&
340
2.23k
      
!ReadMacrosFromExternalSource412
) {
341
0
    ReadMacrosFromExternalSource = true;
342
0
    ExternalSource->ReadDefinedMacros();
343
0
  }
344
2.23k
345
2.23k
  return CurSubmoduleState->Macros.end();
346
2.23k
}
347
348
/// Compares macro tokens with a specified token value sequence.
349
static bool MacroDefinitionEquals(const MacroInfo *MI,
350
114k
                                  ArrayRef<TokenValue> Tokens) {
351
114k
  return Tokens.size() == MI->getNumTokens() &&
352
114k
      
std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin())12.0k
;
353
114k
}
354
355
StringRef Preprocessor::getLastMacroWithSpelling(
356
                                    SourceLocation Loc,
357
303
                                    ArrayRef<TokenValue> Tokens) const {
358
303
  SourceLocation BestLocation;
359
303
  StringRef BestSpelling;
360
303
  for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
361
115k
       I != E; 
++I115k
) {
362
115k
    const MacroDirective::DefInfo
363
115k
      Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
364
115k
    if (!Def || 
!Def.getMacroInfo()114k
)
365
284
      continue;
366
114k
    if (!Def.getMacroInfo()->isObjectLike())
367
25
      continue;
368
114k
    if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
369
114k
      continue;
370
73
    SourceLocation Location = Def.getLocation();
371
73
    // Choose the macro defined latest.
372
73
    if (BestLocation.isInvalid() ||
373
73
        
(13
Location.isValid()13
&&
374
68
         
SourceMgr.isBeforeInTranslationUnit(BestLocation, Location)13
)) {
375
68
      BestLocation = Location;
376
68
      BestSpelling = I->first->getName();
377
68
    }
378
73
  }
379
303
  return BestSpelling;
380
303
}
381
382
3.19M
void Preprocessor::recomputeCurLexerKind() {
383
3.19M
  if (CurLexer)
384
3.67k
    CurLexerKind = CLK_Lexer;
385
3.18M
  else if (CurTokenLexer)
386
9
    CurLexerKind = CLK_TokenLexer;
387
3.18M
  else
388
3.18M
    CurLexerKind = CLK_CachingLexer;
389
3.19M
}
390
391
bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
392
                                          unsigned CompleteLine,
393
1.20k
                                          unsigned CompleteColumn) {
394
1.20k
  assert(File);
395
1.20k
  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
396
1.20k
  assert(!CodeCompletionFile && "Already set");
397
1.20k
398
1.20k
  using llvm::MemoryBuffer;
399
1.20k
400
1.20k
  // Load the actual file's contents.
401
1.20k
  bool Invalid = false;
402
1.20k
  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
403
1.20k
  if (Invalid)
404
0
    return true;
405
1.20k
406
1.20k
  // Find the byte position of the truncation point.
407
1.20k
  const char *Position = Buffer->getBufferStart();
408
34.5k
  for (unsigned Line = 1; Line < CompleteLine; 
++Line33.3k
) {
409
835k
    for (; *Position; 
++Position801k
) {
410
835k
      if (*Position != '\r' && *Position != '\n')
411
801k
        continue;
412
33.3k
413
33.3k
      // Eat \r\n or \n\r as a single line.
414
33.3k
      if ((Position[1] == '\r' || Position[1] == '\n') &&
415
33.3k
          
Position[0] != Position[1]4.87k
)
416
0
        ++Position;
417
33.3k
      ++Position;
418
33.3k
      break;
419
33.3k
    }
420
33.3k
  }
421
1.20k
422
1.20k
  Position += CompleteColumn - 1;
423
1.20k
424
1.20k
  // If pointing inside the preamble, adjust the position at the beginning of
425
1.20k
  // the file after the preamble.
426
1.20k
  if (SkipMainFilePreamble.first &&
427
1.20k
      
SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File75
) {
428
75
    if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
429
5
      Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
430
75
  }
431
1.20k
432
1.20k
  if (Position > Buffer->getBufferEnd())
433
1
    Position = Buffer->getBufferEnd();
434
1.20k
435
1.20k
  CodeCompletionFile = File;
436
1.20k
  CodeCompletionOffset = Position - Buffer->getBufferStart();
437
1.20k
438
1.20k
  auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
439
1.20k
      Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
440
1.20k
  char *NewBuf = NewBuffer->getBufferStart();
441
1.20k
  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
442
1.20k
  *NewPos = '\0';
443
1.20k
  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
444
1.20k
  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
445
1.20k
446
1.20k
  return false;
447
1.20k
}
448
449
void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
450
11
                                            bool IsAngled) {
451
11
  if (CodeComplete)
452
11
    CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
453
11
  setCodeCompletionReached();
454
11
}
455
456
50
void Preprocessor::CodeCompleteNaturalLanguage() {
457
50
  if (CodeComplete)
458
50
    CodeComplete->CodeCompleteNaturalLanguage();
459
50
  setCodeCompletionReached();
460
50
}
461
462
/// getSpelling - This method is used to get the spelling of a token into a
463
/// SmallVector. Note that the returned StringRef may not point to the
464
/// supplied buffer if a copy can be avoided.
465
StringRef Preprocessor::getSpelling(const Token &Tok,
466
                                          SmallVectorImpl<char> &Buffer,
467
12.0M
                                          bool *Invalid) const {
468
12.0M
  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
469
12.0M
  if (Tok.isNot(tok::raw_identifier) && 
!Tok.hasUCN()12.0M
) {
470
12.0M
    // Try the fast path.
471
12.0M
    if (const IdentifierInfo *II = Tok.getIdentifierInfo())
472
109k
      return II->getName();
473
11.9M
  }
474
11.9M
475
11.9M
  // Resize the buffer if we need to copy into it.
476
11.9M
  if (Tok.needsCleaning())
477
8.79k
    Buffer.resize(Tok.getLength());
478
11.9M
479
11.9M
  const char *Ptr = Buffer.data();
480
11.9M
  unsigned Len = getSpelling(Tok, Ptr, Invalid);
481
11.9M
  return StringRef(Ptr, Len);
482
11.9M
}
483
484
/// CreateString - Plop the specified string into a scratch buffer and return a
485
/// location for it.  If specified, the source location provides a source
486
/// location for the token.
487
void Preprocessor::CreateString(StringRef Str, Token &Tok,
488
                                SourceLocation ExpansionLocStart,
489
16.5M
                                SourceLocation ExpansionLocEnd) {
490
16.5M
  Tok.setLength(Str.size());
491
16.5M
492
16.5M
  const char *DestPtr;
493
16.5M
  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
494
16.5M
495
16.5M
  if (ExpansionLocStart.isValid())
496
1.63M
    Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
497
1.63M
                                       ExpansionLocEnd, Str.size());
498
16.5M
  Tok.setLocation(Loc);
499
16.5M
500
16.5M
  // If this is a raw identifier or a literal token, set the pointer data.
501
16.5M
  if (Tok.is(tok::raw_identifier))
502
41.5k
    Tok.setRawIdentifierData(DestPtr);
503
16.5M
  else if (Tok.isLiteral())
504
15.9M
    Tok.setLiteralData(DestPtr);
505
16.5M
}
506
507
18.8k
SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
508
18.8k
  auto &SM = getSourceManager();
509
18.8k
  SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
510
18.8k
  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
511
18.8k
  bool Invalid = false;
512
18.8k
  StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
513
18.8k
  if (Invalid)
514
0
    return SourceLocation();
515
18.8k
516
18.8k
  // FIXME: We could consider re-using spelling for tokens we see repeatedly.
517
18.8k
  const char *DestPtr;
518
18.8k
  SourceLocation Spelling =
519
18.8k
      ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
520
18.8k
  return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length));
521
18.8k
}
522
523
16.8M
Module *Preprocessor::getCurrentModule() {
524
16.8M
  if (!getLangOpts().isCompilingModule())
525
14.8M
    return nullptr;
526
1.95M
527
1.95M
  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
528
1.95M
}
529
530
//===----------------------------------------------------------------------===//
531
// Preprocessor Initialization Methods
532
//===----------------------------------------------------------------------===//
533
534
/// EnterMainSourceFile - Enter the specified FileID as the main source file,
535
/// which implicitly adds the builtin defines etc.
536
59.2k
void Preprocessor::EnterMainSourceFile() {
537
59.2k
  // We do not allow the preprocessor to reenter the main file.  Doing so will
538
59.2k
  // cause FileID's to accumulate information from both runs (e.g. #line
539
59.2k
  // information) and predefined macros aren't guaranteed to be set properly.
540
59.2k
  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
541
59.2k
  FileID MainFileID = SourceMgr.getMainFileID();
542
59.2k
543
59.2k
  // If MainFileID is loaded it means we loaded an AST file, no need to enter
544
59.2k
  // a main file.
545
59.2k
  if (!SourceMgr.isLoadedFileID(MainFileID)) {
546
59.2k
    // Enter the main file source buffer.
547
59.2k
    EnterSourceFile(MainFileID, nullptr, SourceLocation());
548
59.2k
549
59.2k
    // If we've been asked to skip bytes in the main file (e.g., as part of a
550
59.2k
    // precompiled preamble), do so now.
551
59.2k
    if (SkipMainFilePreamble.first > 0)
552
417
      CurLexer->SetByteOffset(SkipMainFilePreamble.first,
553
417
                              SkipMainFilePreamble.second);
554
59.2k
555
59.2k
    // Tell the header info that the main file was entered.  If the file is later
556
59.2k
    // #imported, it won't be re-entered.
557
59.2k
    if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
558
49.8k
      HeaderInfo.IncrementIncludeCount(FE);
559
59.2k
  }
560
59.2k
561
59.2k
  // Preprocess Predefines to populate the initial preprocessor state.
562
59.2k
  std::unique_ptr<llvm::MemoryBuffer> SB =
563
59.2k
    llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
564
59.2k
  assert(SB && "Cannot create predefined source buffer");
565
59.2k
  FileID FID = SourceMgr.createFileID(std::move(SB));
566
59.2k
  assert(FID.isValid() && "Could not create FileID for predefines?");
567
59.2k
  setPredefinesFileID(FID);
568
59.2k
569
59.2k
  // Start parsing the predefines.
570
59.2k
  EnterSourceFile(FID, nullptr, SourceLocation());
571
59.2k
572
59.2k
  if (!PPOpts->PCHThroughHeader.empty()) {
573
30
    // Lookup and save the FileID for the through header. If it isn't found
574
30
    // in the search path, it's a fatal error.
575
30
    const DirectoryLookup *CurDir;
576
30
    Optional<FileEntryRef> File = LookupFile(
577
30
        SourceLocation(), PPOpts->PCHThroughHeader,
578
30
        /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, CurDir,
579
30
        /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
580
30
        /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
581
30
        /*IsFrameworkFound=*/nullptr);
582
30
    if (!File) {
583
2
      Diag(SourceLocation(), diag::err_pp_through_header_not_found)
584
2
          << PPOpts->PCHThroughHeader;
585
2
      return;
586
2
    }
587
28
    setPCHThroughHeaderFileID(
588
28
        SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
589
28
  }
590
59.2k
591
59.2k
  // Skip tokens from the Predefines and if needed the main file.
592
59.2k
  
if (59.2k
(59.2k
usingPCHWithThroughHeader()59.2k
&&
SkippingUntilPCHThroughHeader15
) ||
593
59.2k
      
(59.2k
usingPCHWithPragmaHdrStop()59.2k
&&
SkippingUntilPragmaHdrStop8
))
594
23
    SkipTokensWhileUsingPCH();
595
59.2k
}
596
597
28
void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
598
28
  assert(PCHThroughHeaderFileID.isInvalid() &&
599
28
         "PCHThroughHeaderFileID already set!");
600
28
  PCHThroughHeaderFileID = FID;
601
28
}
602
603
43
bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
604
43
  assert(PCHThroughHeaderFileID.isValid() &&
605
43
         "Invalid PCH through header FileID");
606
43
  return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID);
607
43
}
608
609
1.54M
bool Preprocessor::creatingPCHWithThroughHeader() {
610
1.54M
  return TUKind == TU_Prefix && 
!PPOpts->PCHThroughHeader.empty()5.89k
&&
611
1.54M
         
PCHThroughHeaderFileID.isValid()40
;
612
1.54M
}
613
614
2.23M
bool Preprocessor::usingPCHWithThroughHeader() {
615
2.23M
  return TUKind != TU_Prefix && 
!PPOpts->PCHThroughHeader.empty()2.23M
&&
616
2.23M
         
PCHThroughHeaderFileID.isValid()43
;
617
2.23M
}
618
619
7
bool Preprocessor::creatingPCHWithPragmaHdrStop() {
620
7
  return TUKind == TU_Prefix && 
PPOpts->PCHWithHdrStop2
;
621
7
}
622
623
118k
bool Preprocessor::usingPCHWithPragmaHdrStop() {
624
118k
  return TUKind != TU_Prefix && 
PPOpts->PCHWithHdrStop113k
;
625
118k
}
626
627
/// Skip tokens until after the #include of the through header or
628
/// until after a #pragma hdrstop is seen. Tokens in the predefines file
629
/// and the main file may be skipped. If the end of the predefines file
630
/// is reached, skipping continues into the main file. If the end of the
631
/// main file is reached, it's a fatal error.
632
23
void Preprocessor::SkipTokensWhileUsingPCH() {
633
23
  bool ReachedMainFileEOF = false;
634
23
  bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
635
23
  bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
636
23
  Token Tok;
637
197
  while (true) {
638
197
    bool InPredefines =
639
197
        (CurLexer && 
CurLexer->getFileID() == getPredefinesFileID()193
);
640
197
    switch (CurLexerKind) {
641
193
    case CLK_Lexer:
642
193
      CurLexer->Lex(Tok);
643
193
     break;
644
4
    case CLK_TokenLexer:
645
4
      CurTokenLexer->Lex(Tok);
646
4
      break;
647
0
    case CLK_CachingLexer:
648
0
      CachingLex(Tok);
649
0
      break;
650
0
    case CLK_LexAfterModuleImport:
651
0
      LexAfterModuleImport(Tok);
652
0
      break;
653
197
    }
654
197
    if (Tok.is(tok::eof) && 
!InPredefines4
) {
655
4
      ReachedMainFileEOF = true;
656
4
      break;
657
4
    }
658
193
    if (UsingPCHThroughHeader && 
!SkippingUntilPCHThroughHeader61
)
659
14
      break;
660
179
    if (UsingPragmaHdrStop && 
!SkippingUntilPragmaHdrStop132
)
661
5
      break;
662
179
  }
663
23
  if (ReachedMainFileEOF) {
664
4
    if (UsingPCHThroughHeader)
665
1
      Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
666
1
          << PPOpts->PCHThroughHeader << 1;
667
3
    else if (!PPOpts->PCHWithHdrStopCreate)
668
1
      Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
669
4
  }
670
23
}
671
672
59.2k
void Preprocessor::replayPreambleConditionalStack() {
673
59.2k
  // Restore the conditional stack from the preamble, if there is one.
674
59.2k
  if (PreambleConditionalStack.isReplaying()) {
675
28
    assert(CurPPLexer &&
676
28
           "CurPPLexer is null when calling replayPreambleConditionalStack.");
677
28
    CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
678
28
    PreambleConditionalStack.doneReplaying();
679
28
    if (PreambleConditionalStack.reachedEOFWhileSkipping())
680
18
      SkipExcludedConditionalBlock(
681
18
          PreambleConditionalStack.SkipInfo->HashTokenLoc,
682
18
          PreambleConditionalStack.SkipInfo->IfTokenLoc,
683
18
          PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
684
18
          PreambleConditionalStack.SkipInfo->FoundElse,
685
18
          PreambleConditionalStack.SkipInfo->ElseLoc);
686
28
  }
687
59.2k
}
688
689
46.5k
void Preprocessor::EndSourceFile() {
690
46.5k
  // Notify the client that we reached the end of the source file.
691
46.5k
  if (Callbacks)
692
45.2k
    Callbacks->EndOfMainFile();
693
46.5k
}
694
695
//===----------------------------------------------------------------------===//
696
// Lexer Event Handling.
697
//===----------------------------------------------------------------------===//
698
699
/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
700
/// identifier information for the token and install it into the token,
701
/// updating the token kind accordingly.
702
384M
IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
703
384M
  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
704
384M
705
384M
  // Look up this token, see if it is a macro, or if it is a language keyword.
706
384M
  IdentifierInfo *II;
707
384M
  if (!Identifier.needsCleaning() && 
!Identifier.hasUCN()384M
) {
708
384M
    // No cleaning needed, just use the characters from the lexed buffer.
709
384M
    II = getIdentifierInfo(Identifier.getRawIdentifier());
710
384M
  } else {
711
9.27k
    // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
712
9.27k
    SmallString<64> IdentifierBuffer;
713
9.27k
    StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
714
9.27k
715
9.27k
    if (Identifier.hasUCN()) {
716
155
      SmallString<64> UCNIdentifierBuffer;
717
155
      expandUCNs(UCNIdentifierBuffer, CleanedStr);
718
155
      II = getIdentifierInfo(UCNIdentifierBuffer);
719
9.12k
    } else {
720
9.12k
      II = getIdentifierInfo(CleanedStr);
721
9.12k
    }
722
9.27k
  }
723
384M
724
384M
  // Update the token info (identifier info and appropriate token kind).
725
384M
  Identifier.setIdentifierInfo(II);
726
384M
  if (getLangOpts().MSVCCompat && 
II->isCPlusPlusOperatorKeyword()3.02M
&&
727
384M
      
getSourceManager().isInSystemHeader(Identifier.getLocation())6
)
728
3
    Identifier.setKind(tok::identifier);
729
384M
  else
730
384M
    Identifier.setKind(II->getTokenID());
731
384M
732
384M
  return II;
733
384M
}
734
735
59.9k
void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
736
59.9k
  PoisonReasons[II] = DiagID;
737
59.9k
}
738
739
0
void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
740
0
  assert(Ident__exception_code && Ident__exception_info);
741
0
  assert(Ident___exception_code && Ident___exception_info);
742
0
  Ident__exception_code->setIsPoisoned(Poison);
743
0
  Ident___exception_code->setIsPoisoned(Poison);
744
0
  Ident_GetExceptionCode->setIsPoisoned(Poison);
745
0
  Ident__exception_info->setIsPoisoned(Poison);
746
0
  Ident___exception_info->setIsPoisoned(Poison);
747
0
  Ident_GetExceptionInfo->setIsPoisoned(Poison);
748
0
  Ident__abnormal_termination->setIsPoisoned(Poison);
749
0
  Ident___abnormal_termination->setIsPoisoned(Poison);
750
0
  Ident_AbnormalTermination->setIsPoisoned(Poison);
751
0
}
752
753
16.6k
void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
754
16.6k
  assert(Identifier.getIdentifierInfo() &&
755
16.6k
         "Can't handle identifiers without identifier info!");
756
16.6k
  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
757
16.6k
    PoisonReasons.find(Identifier.getIdentifierInfo());
758
16.6k
  if(it == PoisonReasons.end())
759
4
    Diag(Identifier, diag::err_pp_used_poisoned_id);
760
16.6k
  else
761
16.6k
    Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
762
16.6k
}
763
764
/// Returns a diagnostic message kind for reporting a future keyword as
765
/// appropriate for the identifier and specified language.
766
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II,
767
78
                                          const LangOptions &LangOpts) {
768
78
  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
769
78
770
78
  if (LangOpts.CPlusPlus)
771
78
    return llvm::StringSwitch<diag::kind>(II.getName())
772
78
#define CXX11_KEYWORD(NAME, FLAGS)                                             \
773
780
        .Case(#NAME, diag::warn_cxx11_keyword)
774
78
#define CXX2A_KEYWORD(NAME, FLAGS)                                             \
775
624
        .Case(#NAME, diag::warn_cxx2a_keyword)
776
78
#include "clang/Basic/TokenKinds.def"
777
0
        ;
778
0
779
0
  llvm_unreachable(
780
0
      "Keyword not known to come from a newer Standard or proposed Standard");
781
0
}
782
783
60.6k
void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
784
60.6k
  assert(II.isOutOfDate() && "not out of date");
785
60.6k
  getExternalSource()->updateOutOfDateIdentifier(II);
786
60.6k
}
787
788
/// HandleIdentifier - This callback is invoked when the lexer reads an
789
/// identifier.  This callback looks up the identifier in the map and/or
790
/// potentially macro expands it or turns it into a named token (like 'for').
791
///
792
/// Note that callers of this method are guarded by checking the
793
/// IdentifierInfo's 'isHandleIdentifierCase' bit.  If this method changes, the
794
/// IdentifierInfo methods that compute these properties will need to change to
795
/// match.
796
106M
bool Preprocessor::HandleIdentifier(Token &Identifier) {
797
106M
  assert(Identifier.getIdentifierInfo() &&
798
106M
         "Can't handle identifiers without identifier info!");
799
106M
800
106M
  IdentifierInfo &II = *Identifier.getIdentifierInfo();
801
106M
802
106M
  // If the information about this identifier is out of date, update it from
803
106M
  // the external source.
804
106M
  // We have to treat __VA_ARGS__ in a special way, since it gets
805
106M
  // serialized with isPoisoned = true, but our preprocessor may have
806
106M
  // unpoisoned it if we're defining a C99 macro.
807
106M
  if (II.isOutOfDate()) {
808
59.5k
    bool CurrentIsPoisoned = false;
809
59.5k
    const bool IsSpecialVariadicMacro =
810
59.5k
        &II == Ident__VA_ARGS__ || 
&II == Ident__VA_OPT__59.4k
;
811
59.5k
    if (IsSpecialVariadicMacro)
812
121
      CurrentIsPoisoned = II.isPoisoned();
813
59.5k
814
59.5k
    updateOutOfDateIdentifier(II);
815
59.5k
    Identifier.setKind(II.getTokenID());
816
59.5k
817
59.5k
    if (IsSpecialVariadicMacro)
818
121
      II.setIsPoisoned(CurrentIsPoisoned);
819
59.5k
  }
820
106M
821
106M
  // If this identifier was poisoned, and if it was not produced from a macro
822
106M
  // expansion, emit an error.
823
106M
  if (II.isPoisoned() && 
CurPPLexer16.6k
) {
824
16.6k
    HandlePoisonedIdentifier(Identifier);
825
16.6k
  }
826
106M
827
106M
  // If this is a macro to be expanded, do it.
828
106M
  if (MacroDefinition MD = getMacroDefinition(&II)) {
829
105M
    auto *MI = MD.getMacroInfo();
830
105M
    assert(MI && "macro definition with no macro info?");
831
105M
    if (!DisableMacroExpansion) {
832
72.5M
      if (!Identifier.isExpandDisabled() && 
MI->isEnabled()72.5M
) {
833
72.5M
        // C99 6.10.3p10: If the preprocessing token immediately after the
834
72.5M
        // macro name isn't a '(', this macro should not be expanded.
835
72.5M
        if (!MI->isFunctionLike() || 
isNextPPTokenLParen()40.5M
)
836
68.3M
          return HandleMacroExpandedIdentifier(Identifier, MD);
837
2.56k
      } else {
838
2.56k
        // C99 6.10.3.4p2 says that a disabled macro may never again be
839
2.56k
        // expanded, even if it's in a context where it could be expanded in the
840
2.56k
        // future.
841
2.56k
        Identifier.setFlag(Token::DisableExpand);
842
2.56k
        if (MI->isObjectLike() || 
isNextPPTokenLParen()588
)
843
2.13k
          Diag(Identifier, diag::pp_disabled_macro_expansion);
844
2.56k
      }
845
72.5M
    }
846
105M
  }
847
106M
848
106M
  // If this identifier is a keyword in a newer Standard or proposed Standard,
849
106M
  // produce a warning. Don't warn if we're not considering macro expansion,
850
106M
  // since this identifier might be the name of a macro.
851
106M
  // FIXME: This warning is disabled in cases where it shouldn't be, like
852
106M
  //   "#define constexpr constexpr", "int constexpr;"
853
106M
  
if (37.8M
II.isFutureCompatKeyword()37.8M
&&
!DisableMacroExpansion906
) {
854
78
    Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
855
78
        << II.getName();
856
78
    // Don't diagnose this keyword again in this translation unit.
857
78
    II.setIsFutureCompatKeyword(false);
858
78
  }
859
37.8M
860
37.8M
  // If this is an extension token, diagnose its use.
861
37.8M
  // We avoid diagnosing tokens that originate from macro definitions.
862
37.8M
  // FIXME: This warning is disabled in cases where it shouldn't be,
863
37.8M
  // like "#define TY typeof", "TY(1) x".
864
37.8M
  if (II.isExtensionToken() && 
!DisableMacroExpansion6.90k
)
865
5.37k
    Diag(Identifier, diag::ext_token_used);
866
37.8M
867
37.8M
  // If this is the 'import' contextual keyword following an '@', note
868
37.8M
  // that the next token indicates a module name.
869
37.8M
  //
870
37.8M
  // Note that we do not treat 'import' as a contextual
871
37.8M
  // keyword when we're in a caching lexer, because caching lexers only get
872
37.8M
  // used in contexts where import declarations are disallowed.
873
37.8M
  //
874
37.8M
  // Likewise if this is the C++ Modules TS import keyword.
875
37.8M
  if (((LastTokenWasAt && 
II.isModulesImport()2.95k
) ||
876
37.8M
       
Identifier.is(tok::kw_import)37.8M
) &&
877
37.8M
      
!InMacroArgs1.57k
&&
!DisableMacroExpansion1.57k
&&
878
37.8M
      
(1.56k
getLangOpts().Modules1.56k
||
getLangOpts().DebuggerSupport35
) &&
879
37.8M
      
CurLexerKind != CLK_CachingLexer1.55k
) {
880
1.55k
    ModuleImportLoc = Identifier.getLocation();
881
1.55k
    ModuleImportPath.clear();
882
1.55k
    ModuleImportExpectsIdentifier = true;
883
1.55k
    CurLexerKind = CLK_LexAfterModuleImport;
884
1.55k
  }
885
37.8M
  return true;
886
106M
}
887
888
1.62G
void Preprocessor::Lex(Token &Result) {
889
1.62G
  ++LexLevel;
890
1.62G
891
1.62G
  // We loop here until a lex function returns a token; this avoids recursion.
892
1.62G
  bool ReturnedToken;
893
1.78G
  do {
894
1.78G
    switch (CurLexerKind) {
895
1.03G
    case CLK_Lexer:
896
1.03G
      ReturnedToken = CurLexer->Lex(Result);
897
1.03G
      break;
898
647M
    case CLK_TokenLexer:
899
647M
      ReturnedToken = CurTokenLexer->Lex(Result);
900
647M
      break;
901
107M
    case CLK_CachingLexer:
902
107M
      CachingLex(Result);
903
107M
      ReturnedToken = true;
904
107M
      break;
905
3.68k
    case CLK_LexAfterModuleImport:
906
3.68k
      ReturnedToken = LexAfterModuleImport(Result);
907
3.68k
      break;
908
1.78G
    }
909
1.78G
  } while (!ReturnedToken);
910
1.62G
911
1.62G
  
if (1.62G
Result.is(tok::unknown)1.62G
&&
TheModuleLoader.HadFatalFailure12.7k
)
912
0
    return;
913
1.62G
914
1.62G
  if (Result.is(tok::code_completion) && 
Result.getIdentifierInfo()3.24k
) {
915
85
    // Remember the identifier before code completion token.
916
85
    setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
917
85
    setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc());
918
85
    // Set IdenfitierInfo to null to avoid confusing code that handles both
919
85
    // identifiers and completion tokens.
920
85
    Result.setIdentifierInfo(nullptr);
921
85
  }
922
1.62G
923
1.62G
  // Update ImportSeqState to track our position within a C++20 import-seq
924
1.62G
  // if this token is being produced as a result of phase 4 of translation.
925
1.62G
  if (getLangOpts().CPlusPlusModules && 
LexLevel == 1965k
&&
926
1.62G
      
!Result.getFlag(Token::IsReinjected)293k
) {
927
216k
    switch (Result.getKind()) {
928
29.5k
    case tok::l_paren: case tok::l_square: case tok::l_brace:
929
29.5k
      ImportSeqState.handleOpenBracket();
930
29.5k
      break;
931
29.5k
    
case tok::r_paren: 21.4k
case tok::r_square:
932
21.4k
      ImportSeqState.handleCloseBracket();
933
21.4k
      break;
934
21.4k
    case tok::r_brace:
935
8.05k
      ImportSeqState.handleCloseBrace();
936
8.05k
      break;
937
21.4k
    case tok::semi:
938
18.2k
      ImportSeqState.handleSemi();
939
18.2k
      break;
940
21.4k
    case tok::header_name:
941
24
    case tok::annot_header_unit:
942
24
      ImportSeqState.handleHeaderName();
943
24
      break;
944
86
    case tok::kw_export:
945
86
      ImportSeqState.handleExport();
946
86
      break;
947
43.9k
    case tok::identifier:
948
43.9k
      if (Result.getIdentifierInfo()->isModulesImport()) {
949
50
        ImportSeqState.handleImport();
950
50
        if (ImportSeqState.afterImportSeq()) {
951
32
          ModuleImportLoc = Result.getLocation();
952
32
          ModuleImportPath.clear();
953
32
          ModuleImportExpectsIdentifier = true;
954
32
          CurLexerKind = CLK_LexAfterModuleImport;
955
32
        }
956
50
        break;
957
50
      }
958
43.9k
      LLVM_FALLTHROUGH;
959
139k
    default:
960
139k
      ImportSeqState.handleMisc();
961
139k
      break;
962
1.62G
    }
963
1.62G
  }
964
1.62G
965
1.62G
  LastTokenWasAt = Result.is(tok::at);
966
1.62G
  --LexLevel;
967
1.62G
968
1.62G
  if (LexLevel == 0 && 
!Result.getFlag(Token::IsReinjected)748M
) {
969
673M
    ++TokenCount;
970
673M
    if (OnToken)
971
703
      OnToken(Result);
972
673M
  }
973
1.62G
}
974
975
/// Lex a header-name token (including one formed from header-name-tokens if
976
/// \p AllowConcatenation is \c true).
977
///
978
/// \param FilenameTok Filled in with the next token. On success, this will
979
///        be either a header_name token. On failure, it will be whatever other
980
///        token was found instead.
981
/// \param AllowMacroExpansion If \c true, allow the header name to be formed
982
///        by macro expansion (concatenating tokens as necessary if the first
983
///        token is a '<').
984
/// \return \c true if we reached EOD or EOF while looking for a > token in
985
///         a concatenated header name and diagnosed it. \c false otherwise.
986
2.20M
bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
987
2.20M
  // Lex using header-name tokenization rules if tokens are being lexed from
988
2.20M
  // a file. Just grab a token normally if we're in a macro expansion.
989
2.20M
  if (CurPPLexer)
990
2.20M
    CurPPLexer->LexIncludeFilename(FilenameTok);
991
21
  else
992
21
    Lex(FilenameTok);
993
2.20M
994
2.20M
  // This could be a <foo/bar.h> file coming from a macro expansion.  In this
995
2.20M
  // case, glue the tokens together into an angle_string_literal token.
996
2.20M
  SmallString<128> FilenameBuffer;
997
2.20M
  if (FilenameTok.is(tok::less) && 
AllowMacroExpansion23
) {
998
23
    bool StartOfLine = FilenameTok.isAtStartOfLine();
999
23
    bool LeadingSpace = FilenameTok.hasLeadingSpace();
1000
23
    bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1001
23
1002
23
    SourceLocation Start = FilenameTok.getLocation();
1003
23
    SourceLocation End;
1004
23
    FilenameBuffer.push_back('<');
1005
23
1006
23
    // Consume tokens until we find a '>'.
1007
23
    // FIXME: A header-name could be formed starting or ending with an
1008
23
    // alternative token. It's not clear whether that's ill-formed in all
1009
23
    // cases.
1010
109
    while (FilenameTok.isNot(tok::greater)) {
1011
91
      Lex(FilenameTok);
1012
91
      if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
1013
5
        Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1014
5
        Diag(Start, diag::note_matching) << tok::less;
1015
5
        return true;
1016
5
      }
1017
86
1018
86
      End = FilenameTok.getLocation();
1019
86
1020
86
      // FIXME: Provide code completion for #includes.
1021
86
      if (FilenameTok.is(tok::code_completion)) {
1022
0
        setCodeCompletionReached();
1023
0
        Lex(FilenameTok);
1024
0
        continue;
1025
0
      }
1026
86
1027
86
      // Append the spelling of this token to the buffer. If there was a space
1028
86
      // before it, add it now.
1029
86
      if (FilenameTok.hasLeadingSpace())
1030
4
        FilenameBuffer.push_back(' ');
1031
86
1032
86
      // Get the spelling of the token, directly into FilenameBuffer if
1033
86
      // possible.
1034
86
      size_t PreAppendSize = FilenameBuffer.size();
1035
86
      FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
1036
86
1037
86
      const char *BufPtr = &FilenameBuffer[PreAppendSize];
1038
86
      unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
1039
86
1040
86
      // If the token was spelled somewhere else, copy it into FilenameBuffer.
1041
86
      if (BufPtr != &FilenameBuffer[PreAppendSize])
1042
86
        memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1043
86
1044
86
      // Resize FilenameBuffer to the correct size.
1045
86
      if (FilenameTok.getLength() != ActualLen)
1046
0
        FilenameBuffer.resize(PreAppendSize + ActualLen);
1047
86
    }
1048
23
1049
23
    FilenameTok.startToken();
1050
18
    FilenameTok.setKind(tok::header_name);
1051
18
    FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine);
1052
18
    FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace);
1053
18
    FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro);
1054
18
    CreateString(FilenameBuffer, FilenameTok, Start, End);
1055
2.20M
  } else if (FilenameTok.is(tok::string_literal) && 
AllowMacroExpansion69
) {
1056
68
    // Convert a string-literal token of the form " h-char-sequence "
1057
68
    // (produced by macro expansion) into a header-name token.
1058
68
    //
1059
68
    // The rules for header-names don't quite match the rules for
1060
68
    // string-literals, but all the places where they differ result in
1061
68
    // undefined behavior, so we can and do treat them the same.
1062
68
    //
1063
68
    // A string-literal with a prefix or suffix is not translated into a
1064
68
    // header-name. This could theoretically be observable via the C++20
1065
68
    // context-sensitive header-name formation rules.
1066
68
    StringRef Str = getSpelling(FilenameTok, FilenameBuffer);
1067
68
    if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1068
68
      FilenameTok.setKind(tok::header_name);
1069
68
  }
1070
2.20M
1071
2.20M
  
return false2.20M
;
1072
2.20M
}
1073
1074
/// Collect the tokens of a C++20 pp-import-suffix.
1075
39
void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
1076
39
  // FIXME: For error recovery, consider recognizing attribute syntax here
1077
39
  // and terminating / diagnosing a missing semicolon if we find anything
1078
39
  // else? (Can we leave that to the parser?)
1079
39
  unsigned BracketDepth = 0;
1080
104
  while (true) {
1081
104
    Toks.emplace_back();
1082
104
    Lex(Toks.back());
1083
104
1084
104
    switch (Toks.back().getKind()) {
1085
16
    case tok::l_paren: case tok::l_square: case tok::l_brace:
1086
16
      ++BracketDepth;
1087
16
      break;
1088
16
1089
28
    case tok::r_paren: case tok::r_square: case tok::r_brace:
1090
28
      if (BracketDepth == 0)
1091
12
        return;
1092
16
      --BracketDepth;
1093
16
      break;
1094
16
1095
26
    case tok::semi:
1096
26
      if (BracketDepth == 0)
1097
26
        return;
1098
0
    break;
1099
0
1100
1
    case tok::eof:
1101
1
      return;
1102
0
1103
33
    default:
1104
33
      break;
1105
104
    }
1106
104
  }
1107
39
}
1108
1109
1110
/// Lex a token following the 'import' contextual keyword.
1111
///
1112
///     pp-import: [C++20]
1113
///           import header-name pp-import-suffix[opt] ;
1114
///           import header-name-tokens pp-import-suffix[opt] ;
1115
/// [ObjC]    @ import module-name ;
1116
/// [Clang]   import module-name ;
1117
///
1118
///     header-name-tokens:
1119
///           string-literal
1120
///           < [any sequence of preprocessing-tokens other than >] >
1121
///
1122
///     module-name:
1123
///           module-name-qualifier[opt] identifier
1124
///
1125
///     module-name-qualifier
1126
///           module-name-qualifier[opt] identifier .
1127
///
1128
/// We respond to a pp-import by importing macros from the named module.
1129
3.68k
bool Preprocessor::LexAfterModuleImport(Token &Result) {
1130
3.68k
  // Figure out what kind of lexer we actually have.
1131
3.68k
  recomputeCurLexerKind();
1132
3.68k
1133
3.68k
  // Lex the next token. The header-name lexing rules are used at the start of
1134
3.68k
  // a pp-import.
1135
3.68k
  //
1136
3.68k
  // For now, we only support header-name imports in C++20 mode.
1137
3.68k
  // FIXME: Should we allow this in all language modes that support an import
1138
3.68k
  // declaration as an extension?
1139
3.68k
  if (ModuleImportPath.empty() && 
getLangOpts().CPlusPlusModules1.58k
) {
1140
32
    if (LexHeaderName(Result))
1141
0
      return true;
1142
3.65k
  } else {
1143
3.65k
    Lex(Result);
1144
3.65k
  }
1145
3.68k
1146
3.68k
  // Allocate a holding buffer for a sequence of tokens and introduce it into
1147
3.68k
  // the token stream.
1148
3.68k
  auto EnterTokens = [this](ArrayRef<Token> Toks) {
1149
39
    auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1150
39
    std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1151
39
    EnterTokenStream(std::move(ToksCopy), Toks.size(),
1152
39
                     /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1153
39
  };
1154
3.68k
1155
3.68k
  // Check for a header-name.
1156
3.68k
  SmallVector<Token, 32> Suffix;
1157
3.68k
  if (Result.is(tok::header_name)) {
1158
24
    // Enter the header-name token into the token stream; a Lex action cannot
1159
24
    // both return a token and cache tokens (doing so would corrupt the token
1160
24
    // cache if the call to Lex comes from CachingLex / PeekAhead).
1161
24
    Suffix.push_back(Result);
1162
24
1163
24
    // Consume the pp-import-suffix and expand any macros in it now. We'll add
1164
24
    // it back into the token stream later.
1165
24
    CollectPpImportSuffix(Suffix);
1166
24
    if (Suffix.back().isNot(tok::semi)) {
1167
0
      // This is not a pp-import after all.
1168
0
      EnterTokens(Suffix);
1169
0
      return false;
1170
0
    }
1171
24
1172
24
    // C++2a [cpp.module]p1:
1173
24
    //   The ';' preprocessing-token terminating a pp-import shall not have
1174
24
    //   been produced by macro replacement.
1175
24
    SourceLocation SemiLoc = Suffix.back().getLocation();
1176
24
    if (SemiLoc.isMacroID())
1177
1
      Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1178
24
1179
24
    // Reconstitute the import token.
1180
24
    Token ImportTok;
1181
24
    ImportTok.startToken();
1182
24
    ImportTok.setKind(tok::kw_import);
1183
24
    ImportTok.setLocation(ModuleImportLoc);
1184
24
    ImportTok.setIdentifierInfo(getIdentifierInfo("import"));
1185
24
    ImportTok.setLength(6);
1186
24
1187
24
    auto Action = HandleHeaderIncludeOrImport(
1188
24
        /*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc);
1189
24
    switch (Action.Kind) {
1190
15
    case ImportAction::None:
1191
15
      break;
1192
0
1193
0
    case ImportAction::ModuleBegin:
1194
0
      // Let the parser know we're textually entering the module.
1195
0
      Suffix.emplace_back();
1196
0
      Suffix.back().startToken();
1197
0
      Suffix.back().setKind(tok::annot_module_begin);
1198
0
      Suffix.back().setLocation(SemiLoc);
1199
0
      Suffix.back().setAnnotationEndLoc(SemiLoc);
1200
0
      Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1201
0
      LLVM_FALLTHROUGH;
1202
0
1203
9
    case ImportAction::ModuleImport:
1204
9
    case ImportAction::SkippedModuleImport:
1205
9
      // We chose to import (or textually enter) the file. Convert the
1206
9
      // header-name token into a header unit annotation token.
1207
9
      Suffix[0].setKind(tok::annot_header_unit);
1208
9
      Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1209
9
      Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1210
9
      // FIXME: Call the moduleImport callback?
1211
9
      break;
1212
9
    case ImportAction::Failure:
1213
0
      assert(TheModuleLoader.HadFatalFailure &&
1214
0
             "This should be an early exit only to a fatal error");
1215
0
      Result.setKind(tok::eof);
1216
0
      CurLexer->cutOffLexing();
1217
0
      EnterTokens(Suffix);
1218
0
      return true;
1219
24
    }
1220
24
1221
24
    EnterTokens(Suffix);
1222
24
    return false;
1223
24
  }
1224
3.65k
1225
3.65k
  // The token sequence
1226
3.65k
  //
1227
3.65k
  //   import identifier (. identifier)*
1228
3.65k
  //
1229
3.65k
  // indicates a module import directive. We already saw the 'import'
1230
3.65k
  // contextual keyword, so now we're looking for the identifiers.
1231
3.65k
  if (ModuleImportExpectsIdentifier && 
Result.getKind() == tok::identifier1.83k
) {
1232
1.82k
    // We expected to see an identifier here, and we did; continue handling
1233
1.82k
    // identifiers.
1234
1.82k
    ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
1235
1.82k
                                              Result.getLocation()));
1236
1.82k
    ModuleImportExpectsIdentifier = false;
1237
1.82k
    CurLexerKind = CLK_LexAfterModuleImport;
1238
1.82k
    return true;
1239
1.82k
  }
1240
1.83k
1241
1.83k
  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1242
1.83k
  // see the next identifier. (We can also see a '[[' that begins an
1243
1.83k
  // attribute-specifier-seq here under the C++ Modules TS.)
1244
1.83k
  if (!ModuleImportExpectsIdentifier && 
Result.getKind() == tok::period1.82k
) {
1245
270
    ModuleImportExpectsIdentifier = true;
1246
270
    CurLexerKind = CLK_LexAfterModuleImport;
1247
270
    return true;
1248
270
  }
1249
1.56k
1250
1.56k
  // If we didn't recognize a module name at all, this is not a (valid) import.
1251
1.56k
  if (ModuleImportPath.empty() || 
Result.is(tok::eof)1.55k
)
1252
8
    return true;
1253
1.55k
1254
1.55k
  // Consume the pp-import-suffix and expand any macros in it now, if we're not
1255
1.55k
  // at the semicolon already.
1256
1.55k
  SourceLocation SemiLoc = Result.getLocation();
1257
1.55k
  if (Result.isNot(tok::semi)) {
1258
15
    Suffix.push_back(Result);
1259
15
    CollectPpImportSuffix(Suffix);
1260
15
    if (Suffix.back().isNot(tok::semi)) {
1261
13
      // This is not an import after all.
1262
13
      EnterTokens(Suffix);
1263
13
      return false;
1264
13
    }
1265
2
    SemiLoc = Suffix.back().getLocation();
1266
2
  }
1267
1.55k
1268
1.55k
  // Under the Modules TS, the dot is just part of the module name, and not
1269
1.55k
  // a real hierarchy separator. Flatten such module names now.
1270
1.55k
  //
1271
1.55k
  // FIXME: Is this the right level to be performing this transformation?
1272
1.55k
  std::string FlatModuleName;
1273
1.54k
  if (getLangOpts().ModulesTS || 
getLangOpts().CPlusPlusModules1.48k
) {
1274
72
    for (auto &Piece : ModuleImportPath) {
1275
72
      if (!FlatModuleName.empty())
1276
7
        FlatModuleName += ".";
1277
72
      FlatModuleName += Piece.first->getName();
1278
72
    }
1279
65
    SourceLocation FirstPathLoc = ModuleImportPath[0].second;
1280
65
    ModuleImportPath.clear();
1281
65
    ModuleImportPath.push_back(
1282
65
        std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
1283
65
  }
1284
1.54k
1285
1.54k
  Module *Imported = nullptr;
1286
1.54k
  if (getLangOpts().Modules) {
1287
1.51k
    Imported = TheModuleLoader.loadModule(ModuleImportLoc,
1288
1.51k
                                          ModuleImportPath,
1289
1.51k
                                          Module::Hidden,
1290
1.51k
                                          /*IsInclusionDirective=*/false);
1291
1.51k
    if (Imported)
1292
1.42k
      makeModuleVisible(Imported, SemiLoc);
1293
1.51k
  }
1294
1.54k
  if (Callbacks)
1295
1.54k
    Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1296
1.54k
1297
1.54k
  if (!Suffix.empty()) {
1298
2
    EnterTokens(Suffix);
1299
2
    return false;
1300
2
  }
1301
1.54k
  return true;
1302
1.54k
}
1303
1304
103k
void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
1305
103k
  CurSubmoduleState->VisibleModules.setVisible(
1306
569k
      M, Loc, [](Module *) {},
1307
103k
      [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1308
1
        // FIXME: Include the path in the diagnostic.
1309
1
        // FIXME: Include the import location for the conflicting module.
1310
1
        Diag(ModuleImportLoc, diag::warn_module_conflict)
1311
1
            << Path[0]->getFullModuleName()
1312
1
            << Conflict->getFullModuleName()
1313
1
            << Message;
1314
1
      });
1315
103k
1316
103k
  // Add this module to the imports list of the currently-built submodule.
1317
103k
  if (!BuildingSubmoduleStack.empty() && 
M != BuildingSubmoduleStack.back().M79.4k
)
1318
77.6k
    BuildingSubmoduleStack.back().M->Imports.insert(M);
1319
103k
}
1320
1321
bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1322
                                          const char *DiagnosticTag,
1323
15.5k
                                          bool AllowMacroExpansion) {
1324
15.5k
  // We need at least one string literal.
1325
15.5k
  if (Result.isNot(tok::string_literal)) {
1326
15
    Diag(Result, diag::err_expected_string_literal)
1327
15
      << /*Source='in...'*/0 << DiagnosticTag;
1328
15
    return false;
1329
15
  }
1330
15.5k
1331
15.5k
  // Lex string literal tokens, optionally with macro expansion.
1332
15.5k
  SmallVector<Token, 4> StrToks;
1333
15.5k
  do {
1334
15.5k
    StrToks.push_back(Result);
1335
15.5k
1336
15.5k
    if (Result.hasUDSuffix())
1337
4
      Diag(Result, diag::err_invalid_string_udl);
1338
15.5k
1339
15.5k
    if (AllowMacroExpansion)
1340
139
      Lex(Result);
1341
15.4k
    else
1342
15.4k
      LexUnexpandedToken(Result);
1343
15.5k
  } while (Result.is(tok::string_literal));
1344
15.5k
1345
15.5k
  // Concatenate and parse the strings.
1346
15.5k
  StringLiteralParser Literal(StrToks, *this);
1347
15.5k
  assert(Literal.isAscii() && "Didn't allow wide strings in");
1348
15.5k
1349
15.5k
  if (Literal.hadError)
1350
0
    return false;
1351
15.5k
1352
15.5k
  if (Literal.Pascal) {
1353
0
    Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1354
0
      << /*Source='in...'*/0 << DiagnosticTag;
1355
0
    return false;
1356
0
  }
1357
15.5k
1358
15.5k
  String = std::string(Literal.GetString());
1359
15.5k
  return true;
1360
15.5k
}
1361
1362
101
bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1363
101
  assert(Tok.is(tok::numeric_constant));
1364
101
  SmallString<8> IntegerBuffer;
1365
101
  bool NumberInvalid = false;
1366
101
  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1367
101
  if (NumberInvalid)
1368
0
    return false;
1369
101
  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
1370
101
  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1371
1
    return false;
1372
100
  llvm::APInt APVal(64, 0);
1373
100
  if (Literal.GetIntegerValue(APVal))
1374
0
    return false;
1375
100
  Lex(Tok);
1376
100
  Value = APVal.getLimitedValue();
1377
100
  return true;
1378
100
}
1379
1380
68.5k
void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1381
68.5k
  assert(Handler && "NULL comment handler");
1382
68.5k
  assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1383
68.5k
         "Comment handler already registered");
1384
68.5k
  CommentHandlers.push_back(Handler);
1385
68.5k
}
1386
1387
68.4k
void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1388
68.4k
  std::vector<CommentHandler *>::iterator Pos =
1389
68.4k
      llvm::find(CommentHandlers, Handler);
1390
68.4k
  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1391
68.4k
  CommentHandlers.erase(Pos);
1392
68.4k
}
1393
1394
54.4M
bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1395
54.4M
  bool AnyPendingTokens = false;
1396
54.4M
  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1397
54.4M
       HEnd = CommentHandlers.end();
1398
96.8M
       H != HEnd; 
++H42.3M
) {
1399
42.3M
    if ((*H)->HandleComment(*this, Comment))
1400
0
      AnyPendingTokens = true;
1401
42.3M
  }
1402
54.4M
  if (!AnyPendingTokens || 
getCommentRetentionState()0
)
1403
54.4M
    return false;
1404
0
  Lex(result);
1405
0
  return true;
1406
0
}
1407
1408
58.2k
ModuleLoader::~ModuleLoader() = default;
1409
1410
68.3k
CommentHandler::~CommentHandler() = default;
1411
1412
54.4k
CodeCompletionHandler::~CodeCompletionHandler() = default;
1413
1414
2.05k
void Preprocessor::createPreprocessingRecord() {
1415
2.05k
  if (Record)
1416
0
    return;
1417
2.05k
1418
2.05k
  Record = new PreprocessingRecord(getSourceManager());
1419
2.05k
  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
1420
2.05k
}