Coverage Report

Created: 2023-11-11 10:31

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/PPLexerChange.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements pieces of the Preprocessor interface that manage the
10
// current lexer stack.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "clang/Basic/FileManager.h"
15
#include "clang/Basic/SourceManager.h"
16
#include "clang/Lex/HeaderSearch.h"
17
#include "clang/Lex/LexDiagnostic.h"
18
#include "clang/Lex/MacroInfo.h"
19
#include "clang/Lex/Preprocessor.h"
20
#include "clang/Lex/PreprocessorOptions.h"
21
#include "llvm/ADT/StringSwitch.h"
22
#include "llvm/Support/FileSystem.h"
23
#include "llvm/Support/MemoryBufferRef.h"
24
#include "llvm/Support/Path.h"
25
#include <optional>
26
27
using namespace clang;
28
29
//===----------------------------------------------------------------------===//
30
// Miscellaneous Methods.
31
//===----------------------------------------------------------------------===//
32
33
/// isInPrimaryFile - Return true if we're in the top-level file, not in a
34
/// \#include.  This looks through macro expansions and active _Pragma lexers.
35
391k
bool Preprocessor::isInPrimaryFile() const {
36
391k
  if (IsFileLexer())
37
391k
    return IncludeMacroStack.empty();
38
39
  // If there are any stacked lexers, we're in a #include.
40
2
  assert(IsFileLexer(IncludeMacroStack[0]) &&
41
2
         "Top level include stack isn't our primary lexer?");
42
2
  return llvm::none_of(
43
2
      llvm::drop_begin(IncludeMacroStack),
44
2
      [&](const IncludeStackInfo &ISI) -> bool 
{ return IsFileLexer(ISI); }1
);
45
2
}
46
47
/// getCurrentLexer - Return the current file lexer being lexed from.  Note
48
/// that this ignores any potentially active macro expansions and _Pragma
49
/// expansions going on at the time.
50
4.21M
PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
51
4.21M
  if (IsFileLexer())
52
4.21M
    return CurPPLexer;
53
54
  // Look for a stacked lexer.
55
15
  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
56
15
    if (IsFileLexer(ISI))
57
14
      return ISI.ThePPLexer;
58
15
  }
59
1
  return nullptr;
60
15
}
61
62
63
//===----------------------------------------------------------------------===//
64
// Methods for Entering and Callbacks for leaving various contexts
65
//===----------------------------------------------------------------------===//
66
67
/// EnterSourceFile - Add a source file to the top of the include stack and
68
/// start lexing tokens from it instead of the current buffer.
69
bool Preprocessor::EnterSourceFile(FileID FID, ConstSearchDirIterator CurDir,
70
                                   SourceLocation Loc,
71
1.06M
                                   bool IsFirstIncludeOfFile) {
72
1.06M
  assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
73
1.06M
  ++NumEnteredSourceFiles;
74
75
1.06M
  if (MaxIncludeStackDepth < IncludeMacroStack.size())
76
31.8k
    MaxIncludeStackDepth = IncludeMacroStack.size();
77
78
  // Get the MemoryBuffer for this FID, if it fails, we fail.
79
1.06M
  std::optional<llvm::MemoryBufferRef> InputFile =
80
1.06M
      getSourceManager().getBufferOrNone(FID, Loc);
81
1.06M
  if (!InputFile) {
82
2
    SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
83
2
    Diag(Loc, diag::err_pp_error_opening_file)
84
2
        << std::string(SourceMgr.getBufferName(FileStart)) << "";
85
2
    return true;
86
2
  }
87
88
1.06M
  if (isCodeCompletionEnabled() &&
89
1.06M
      
SourceMgr.getFileEntryForID(FID) == CodeCompletionFile2.65k
) {
90
1.29k
    CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
91
1.29k
    CodeCompletionLoc =
92
1.29k
        CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
93
1.29k
  }
94
95
1.06M
  Lexer *TheLexer = new Lexer(FID, *InputFile, *this, IsFirstIncludeOfFile);
96
1.06M
  if (getPreprocessorOpts().DependencyDirectivesForFile &&
97
1.06M
      
FID != PredefinesFileID984
) {
98
633
    if (OptionalFileEntryRef File = SourceMgr.getFileEntryRefForID(FID)) {
99
495
      if (std::optional<ArrayRef<dependency_directives_scan::Directive>>
100
495
              DepDirectives =
101
495
                  getPreprocessorOpts().DependencyDirectivesForFile(*File)) {
102
474
        TheLexer->DepDirectives = *DepDirectives;
103
474
      }
104
495
    }
105
633
  }
106
107
1.06M
  EnterSourceFileWithLexer(TheLexer, CurDir);
108
1.06M
  return false;
109
1.06M
}
110
111
/// EnterSourceFileWithLexer - Add a source file to the top of the include stack
112
///  and start lexing tokens from it instead of the current buffer.
113
void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
114
1.78M
                                            ConstSearchDirIterator CurDir) {
115
1.78M
  PreprocessorLexer *PrevPPLexer = CurPPLexer;
116
117
  // Add the current lexer to the include stack.
118
1.78M
  if (CurPPLexer || 
CurTokenLexer809k
)
119
1.69M
    PushIncludeMacroStack();
120
121
1.78M
  CurLexer.reset(TheLexer);
122
1.78M
  CurPPLexer = TheLexer;
123
1.78M
  CurDirLookup = CurDir;
124
1.78M
  CurLexerSubmodule = nullptr;
125
1.78M
  if (CurLexerCallback != CLK_LexAfterModuleImport)
126
1.78M
    CurLexerCallback = TheLexer->isDependencyDirectivesLexer()
127
1.78M
                           ? 
CLK_DependencyDirectivesLexer473
128
1.78M
                           : 
CLK_Lexer1.78M
;
129
130
  // Notify the client, if desired, that we are in a new source file.
131
1.78M
  if (Callbacks && 
!CurLexer->Is_PragmaLexer1.78M
) {
132
1.06M
    SrcMgr::CharacteristicKind FileType =
133
1.06M
       SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
134
135
1.06M
    FileID PrevFID;
136
1.06M
    SourceLocation EnterLoc;
137
1.06M
    if (PrevPPLexer) {
138
972k
      PrevFID = PrevPPLexer->getFileID();
139
972k
      EnterLoc = PrevPPLexer->getSourceLocation();
140
972k
    }
141
1.06M
    Callbacks->FileChanged(CurLexer->getFileLoc(), PPCallbacks::EnterFile,
142
1.06M
                           FileType, PrevFID);
143
1.06M
    Callbacks->LexedFileChanged(CurLexer->getFileID(),
144
1.06M
                                PPCallbacks::LexedFileChangeReason::EnterFile,
145
1.06M
                                FileType, PrevFID, EnterLoc);
146
1.06M
  }
147
1.78M
}
148
149
/// EnterMacro - Add a Macro to the top of the include stack and start lexing
150
/// tokens from it instead of the current buffer.
151
void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
152
76.8M
                              MacroInfo *Macro, MacroArgs *Args) {
153
76.8M
  std::unique_ptr<TokenLexer> TokLexer;
154
76.8M
  if (NumCachedTokenLexers == 0) {
155
60.0k
    TokLexer = std::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
156
76.8M
  } else {
157
76.8M
    TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
158
76.8M
    TokLexer->Init(Tok, ILEnd, Macro, Args);
159
76.8M
  }
160
161
76.8M
  PushIncludeMacroStack();
162
76.8M
  CurDirLookup = nullptr;
163
76.8M
  CurTokenLexer = std::move(TokLexer);
164
76.8M
  if (CurLexerCallback != CLK_LexAfterModuleImport)
165
76.8M
    CurLexerCallback = CLK_TokenLexer;
166
76.8M
}
167
168
/// EnterTokenStream - Add a "macro" context to the top of the include stack,
169
/// which will cause the lexer to start returning the specified tokens.
170
///
171
/// If DisableMacroExpansion is true, tokens lexed from the token stream will
172
/// not be subject to further macro expansion.  Otherwise, these tokens will
173
/// be re-macro-expanded when/if expansion is enabled.
174
///
175
/// If OwnsTokens is false, this method assumes that the specified stream of
176
/// tokens has a permanent owner somewhere, so they do not need to be copied.
177
/// If it is true, it assumes the array of tokens is allocated with new[] and
178
/// must be freed.
179
///
180
void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
181
                                    bool DisableMacroExpansion, bool OwnsTokens,
182
13.2M
                                    bool IsReinject) {
183
13.2M
  if (CurLexerCallback == CLK_CachingLexer) {
184
5.13k
    if (CachedLexPos < CachedTokens.size()) {
185
4.10k
      assert(IsReinject && "new tokens in the middle of cached stream");
186
      // We're entering tokens into the middle of our cached token stream. We
187
      // can't represent that, so just insert the tokens into the buffer.
188
4.10k
      CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
189
4.10k
                          Toks, Toks + NumToks);
190
4.10k
      if (OwnsTokens)
191
0
        delete [] Toks;
192
4.10k
      return;
193
4.10k
    }
194
195
    // New tokens are at the end of the cached token sequnece; insert the
196
    // token stream underneath the caching lexer.
197
1.02k
    ExitCachingLexMode();
198
1.02k
    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
199
1.02k
                     IsReinject);
200
1.02k
    EnterCachingLexMode();
201
1.02k
    return;
202
5.13k
  }
203
204
  // Create a macro expander to expand from the specified token stream.
205
13.2M
  std::unique_ptr<TokenLexer> TokLexer;
206
13.2M
  if (NumCachedTokenLexers == 0) {
207
1.34M
    TokLexer = std::make_unique<TokenLexer>(
208
1.34M
        Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
209
11.9M
  } else {
210
11.9M
    TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
211
11.9M
    TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
212
11.9M
                   IsReinject);
213
11.9M
  }
214
215
  // Save our current state.
216
13.2M
  PushIncludeMacroStack();
217
13.2M
  CurDirLookup = nullptr;
218
13.2M
  CurTokenLexer = std::move(TokLexer);
219
13.2M
  if (CurLexerCallback != CLK_LexAfterModuleImport)
220
13.2M
    CurLexerCallback = CLK_TokenLexer;
221
13.2M
}
222
223
/// Compute the relative path that names the given file relative to
224
/// the given directory.
225
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
226
54
                                FileEntryRef File, SmallString<128> &Result) {
227
54
  Result.clear();
228
229
54
  StringRef FilePath = File.getDir().getName();
230
54
  StringRef Path = FilePath;
231
54
  while (!Path.empty()) {
232
54
    if (auto CurDir = FM.getDirectory(Path)) {
233
54
      if (*CurDir == Dir) {
234
54
        Result = FilePath.substr(Path.size());
235
54
        llvm::sys::path::append(Result,
236
54
                                llvm::sys::path::filename(File.getName()));
237
54
        return;
238
54
      }
239
54
    }
240
241
0
    Path = llvm::sys::path::parent_path(Path);
242
0
  }
243
244
0
  Result = File.getName();
245
0
}
246
247
81.9M
void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
248
81.9M
  if (CurTokenLexer) {
249
39.6M
    CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
250
39.6M
    return;
251
39.6M
  }
252
42.3M
  
if (42.3M
CurLexer42.3M
) {
253
42.3M
    CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
254
42.3M
    return;
255
42.3M
  }
256
  // FIXME: Handle other kinds of lexers?  It generally shouldn't matter,
257
  // but it might if they're empty?
258
42.3M
}
259
260
/// Determine the location to use as the end of the buffer for a lexer.
261
///
262
/// If the file ends with a newline, form the EOF token on the newline itself,
263
/// rather than "on the line following it", which doesn't exist.  This makes
264
/// diagnostics relating to the end of file include the last file that the user
265
/// actually typed, which is goodness.
266
117k
const char *Preprocessor::getCurLexerEndPos() {
267
117k
  const char *EndPos = CurLexer->BufferEnd;
268
117k
  if (EndPos != CurLexer->BufferStart &&
269
117k
      
(113k
EndPos[-1] == '\n'113k
||
EndPos[-1] == '\r'25.1k
)) {
270
88.1k
    --EndPos;
271
272
    // Handle \n\r and \r\n:
273
88.1k
    if (EndPos != CurLexer->BufferStart &&
274
88.1k
        
(88.0k
EndPos[-1] == '\n'88.0k
||
EndPos[-1] == '\r'80.4k
) &&
275
88.1k
        
EndPos[-1] != EndPos[0]7.71k
)
276
79
      --EndPos;
277
88.1k
  }
278
279
117k
  return EndPos;
280
117k
}
281
282
static void collectAllSubModulesWithUmbrellaHeader(
283
13.1k
    const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
284
13.1k
  if (Mod.getUmbrellaHeaderAsWritten())
285
420
    SubMods.push_back(&Mod);
286
13.1k
  for (auto *M : Mod.submodules())
287
9.48k
    collectAllSubModulesWithUmbrellaHeader(*M, SubMods);
288
13.1k
}
289
290
420
void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
291
420
  std::optional<Module::Header> UmbrellaHeader =
292
420
      Mod.getUmbrellaHeaderAsWritten();
293
420
  assert(UmbrellaHeader && "Module must use umbrella header");
294
420
  const FileID &File = SourceMgr.translateFile(UmbrellaHeader->Entry);
295
420
  SourceLocation ExpectedHeadersLoc = SourceMgr.getLocForEndOfFile(File);
296
420
  if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header,
297
420
                                 ExpectedHeadersLoc))
298
169
    return;
299
300
251
  ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
301
251
  OptionalDirectoryEntryRef Dir = Mod.getEffectiveUmbrellaDir();
302
251
  llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
303
251
  std::error_code EC;
304
251
  for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC),
305
251
       End;
306
981
       Entry != End && 
!EC730
;
Entry.increment(EC)730
) {
307
730
    using llvm::StringSwitch;
308
309
    // Check whether this entry has an extension typically associated with
310
    // headers.
311
730
    if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
312
730
             .Cases(".h", ".H", ".hh", ".hpp", true)
313
730
             .Default(false))
314
51
      continue;
315
316
679
    if (auto Header = getFileManager().getOptionalFileRef(Entry->path()))
317
679
      if (!getSourceManager().hasFileInfo(*Header)) {
318
135
        if (!ModMap.isHeaderInUnavailableModule(*Header)) {
319
          // Find the relative path that would access this header.
320
54
          SmallString<128> RelativePath;
321
54
          computeRelativePath(FileMgr, *Dir, *Header, RelativePath);
322
54
          Diag(ExpectedHeadersLoc, diag::warn_uncovered_module_header)
323
54
              << Mod.getFullModuleName() << RelativePath;
324
54
        }
325
135
      }
326
679
  }
327
251
}
328
329
/// HandleEndOfFile - This callback is invoked when the lexer hits the end of
330
/// the current file.  This either returns the EOF token or pops a level off
331
/// the include stack and keeps going.
332
80.5M
bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
333
80.5M
  assert(!CurTokenLexer &&
334
80.5M
         "Ending a file when currently in a macro!");
335
336
80.5M
  SourceLocation UnclosedSafeBufferOptOutLoc;
337
338
80.5M
  if (IncludeMacroStack.empty() &&
339
80.5M
      
isPPInSafeBufferOptOutRegion(UnclosedSafeBufferOptOutLoc)93.7k
) {
340
    // To warn if a "-Wunsafe-buffer-usage" opt-out region is still open by the
341
    // end of a file.
342
1
    Diag(UnclosedSafeBufferOptOutLoc,
343
1
         diag::err_pp_unclosed_pragma_unsafe_buffer_usage);
344
1
  }
345
  // If we have an unclosed module region from a pragma at the end of a
346
  // module, complain and close it now.
347
80.5M
  const bool LeavingSubmodule = CurLexer && 
CurLexerSubmodule1.78M
;
348
80.5M
  if ((LeavingSubmodule || 
IncludeMacroStack.empty()80.5M
) &&
349
80.5M
      
!BuildingSubmoduleStack.empty()117k
&&
350
80.5M
      
BuildingSubmoduleStack.back().IsPragma23.6k
) {
351
2
    Diag(BuildingSubmoduleStack.back().ImportLoc,
352
2
         diag::err_pp_module_begin_without_module_end);
353
2
    Module *M = LeaveSubmodule(/*ForPragma*/true);
354
355
2
    Result.startToken();
356
2
    const char *EndPos = getCurLexerEndPos();
357
2
    CurLexer->BufferPtr = EndPos;
358
2
    CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
359
2
    Result.setAnnotationEndLoc(Result.getLocation());
360
2
    Result.setAnnotationValue(M);
361
2
    return true;
362
2
  }
363
364
  // See if this file had a controlling macro.
365
80.5M
  if (CurPPLexer) {  // Not ending a macro, ignore it.
366
1.78M
    if (const IdentifierInfo *ControllingMacro =
367
1.78M
          CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
368
      // Okay, this has a controlling macro, remember in HeaderFileInfo.
369
657k
      if (OptionalFileEntryRef FE = CurPPLexer->getFileEntry()) {
370
657k
        HeaderInfo.SetFileControllingMacro(*FE, ControllingMacro);
371
657k
        if (MacroInfo *MI =
372
657k
              getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
373
656k
          MI->setUsedForHeaderGuard(true);
374
657k
        if (const IdentifierInfo *DefinedMacro =
375
657k
              CurPPLexer->MIOpt.GetDefinedMacro()) {
376
618k
          if (!isMacroDefined(ControllingMacro) &&
377
618k
              
DefinedMacro != ControllingMacro14
&&
378
618k
              
CurLexer->isFirstTimeLexingFile()14
) {
379
380
            // If the edit distance between the two macros is more than 50%,
381
            // DefinedMacro may not be header guard, or can be header guard of
382
            // another header file. Therefore, it maybe defining something
383
            // completely different. This can be observed in the wild when
384
            // handling feature macros or header guards in different files.
385
386
8
            const StringRef ControllingMacroName = ControllingMacro->getName();
387
8
            const StringRef DefinedMacroName = DefinedMacro->getName();
388
8
            const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
389
8
                                                  DefinedMacroName.size()) / 2;
390
8
            const unsigned ED = ControllingMacroName.edit_distance(
391
8
                DefinedMacroName, true, MaxHalfLength);
392
8
            if (ED <= MaxHalfLength) {
393
              // Emit a warning for a bad header guard.
394
6
              Diag(CurPPLexer->MIOpt.GetMacroLocation(),
395
6
                   diag::warn_header_guard)
396
6
                  << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
397
6
              Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
398
6
                   diag::note_header_guard)
399
6
                  << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
400
6
                  << ControllingMacro
401
6
                  << FixItHint::CreateReplacement(
402
6
                         CurPPLexer->MIOpt.GetDefinedLocation(),
403
6
                         ControllingMacro->getName());
404
6
            }
405
8
          }
406
618k
        }
407
657k
      }
408
657k
    }
409
1.78M
  }
410
411
  // Complain about reaching a true EOF within arc_cf_code_audited.
412
  // We don't want to complain about reaching the end of a macro
413
  // instantiation or a _Pragma.
414
80.5M
  if (PragmaARCCFCodeAuditedInfo.second.isValid() && 
!isEndOfMacro8.85M
&&
415
80.5M
      
!(2
CurLexer2
&&
CurLexer->Is_PragmaLexer2
)) {
416
2
    Diag(PragmaARCCFCodeAuditedInfo.second,
417
2
         diag::err_pp_eof_in_arc_cf_code_audited);
418
419
    // Recover by leaving immediately.
420
2
    PragmaARCCFCodeAuditedInfo = {nullptr, SourceLocation()};
421
2
  }
422
423
  // Complain about reaching a true EOF within assume_nonnull.
424
  // We don't want to complain about reaching the end of a macro
425
  // instantiation or a _Pragma.
426
80.5M
  if (PragmaAssumeNonNullLoc.isValid() &&
427
80.5M
      
!isEndOfMacro15.5M
&&
!(2
CurLexer2
&&
CurLexer->Is_PragmaLexer2
)) {
428
    // If we're at the end of generating a preamble, we should record the
429
    // unterminated \#pragma clang assume_nonnull so we can restore it later
430
    // when the preamble is loaded into the main file.
431
2
    if (isRecordingPreamble() && 
isInPrimaryFile()1
)
432
1
      PreambleRecordedPragmaAssumeNonNullLoc = PragmaAssumeNonNullLoc;
433
1
    else
434
1
      Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
435
    // Recover by leaving immediately.
436
2
    PragmaAssumeNonNullLoc = SourceLocation();
437
2
  }
438
439
80.5M
  bool LeavingPCHThroughHeader = false;
440
441
  // If this is a #include'd file, pop it off the include stack and continue
442
  // lexing the #includer file.
443
80.5M
  if (!IncludeMacroStack.empty()) {
444
445
    // If we lexed the code-completion file, act as if we reached EOF.
446
80.4M
    if (isCodeCompletionEnabled() && 
CurPPLexer2.00k
&&
447
80.4M
        SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
448
1.35k
            CodeCompletionFileLoc) {
449
1
      assert(CurLexer && "Got EOF but no current lexer set!");
450
1
      Result.startToken();
451
1
      CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
452
1
      CurLexer.reset();
453
454
1
      CurPPLexer = nullptr;
455
1
      recomputeCurLexerKind();
456
1
      return true;
457
1
    }
458
459
80.4M
    if (!isEndOfMacro && 
CurPPLexer974k
&&
460
80.4M
        
(974k
SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()974k
||
461
         // Predefines file doesn't have a valid include location.
462
974k
         
(94.0k
PredefinesFileID.isValid()94.0k
&&
463
974k
          
CurPPLexer->getFileID() == PredefinesFileID94.0k
))) {
464
      // Notify SourceManager to record the number of FileIDs that were created
465
      // during lexing of the #include'd file.
466
974k
      unsigned NumFIDs =
467
974k
          SourceMgr.local_sloc_entry_size() -
468
974k
          CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
469
974k
      SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
470
974k
    }
471
472
80.4M
    bool ExitedFromPredefinesFile = false;
473
80.4M
    FileID ExitedFID;
474
80.4M
    if (!isEndOfMacro && 
CurPPLexer974k
) {
475
974k
      ExitedFID = CurPPLexer->getFileID();
476
477
974k
      assert(PredefinesFileID.isValid() &&
478
974k
             "HandleEndOfFile is called before PredefinesFileId is set");
479
974k
      ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
480
974k
    }
481
482
80.4M
    if (LeavingSubmodule) {
483
      // We're done with this submodule.
484
23.6k
      Module *M = LeaveSubmodule(/*ForPragma*/false);
485
486
      // Notify the parser that we've left the module.
487
23.6k
      const char *EndPos = getCurLexerEndPos();
488
23.6k
      Result.startToken();
489
23.6k
      CurLexer->BufferPtr = EndPos;
490
23.6k
      CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
491
23.6k
      Result.setAnnotationEndLoc(Result.getLocation());
492
23.6k
      Result.setAnnotationValue(M);
493
23.6k
    }
494
495
80.4M
    bool FoundPCHThroughHeader = false;
496
80.4M
    if (CurPPLexer && 
creatingPCHWithThroughHeader()1.69M
&&
497
80.4M
        isPCHThroughHeader(
498
23
            SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
499
12
      FoundPCHThroughHeader = true;
500
501
    // We're done with the #included file.
502
80.4M
    RemoveTopOfLexerStack();
503
504
    // Propagate info about start-of-line/leading white-space/etc.
505
80.4M
    PropagateLineStartLeadingSpaceInfo(Result);
506
507
    // Notify the client, if desired, that we are in a new source file.
508
80.4M
    if (Callbacks && 
!isEndOfMacro80.4M
&&
CurPPLexer972k
) {
509
972k
      SourceLocation Loc = CurPPLexer->getSourceLocation();
510
972k
      SrcMgr::CharacteristicKind FileType =
511
972k
          SourceMgr.getFileCharacteristic(Loc);
512
972k
      Callbacks->FileChanged(Loc, PPCallbacks::ExitFile, FileType, ExitedFID);
513
972k
      Callbacks->LexedFileChanged(CurPPLexer->getFileID(),
514
972k
                                  PPCallbacks::LexedFileChangeReason::ExitFile,
515
972k
                                  FileType, ExitedFID, Loc);
516
972k
    }
517
518
    // Restore conditional stack as well as the recorded
519
    // \#pragma clang assume_nonnull from the preamble right after exiting
520
    // from the predefines file.
521
80.4M
    if (ExitedFromPredefinesFile) {
522
94.0k
      replayPreambleConditionalStack();
523
94.0k
      if (PreambleRecordedPragmaAssumeNonNullLoc.isValid())
524
5
        PragmaAssumeNonNullLoc = PreambleRecordedPragmaAssumeNonNullLoc;
525
94.0k
    }
526
527
80.4M
    if (!isEndOfMacro && 
CurPPLexer974k
&&
FoundPCHThroughHeader974k
&&
528
80.4M
        
(12
isInPrimaryFile()12
||
529
12
         
CurPPLexer->getFileID() == getPredefinesFileID()7
)) {
530
      // Leaving the through header. Continue directly to end of main file
531
      // processing.
532
12
      LeavingPCHThroughHeader = true;
533
80.4M
    } else {
534
      // Client should lex another token unless we generated an EOM.
535
80.4M
      return LeavingSubmodule;
536
80.4M
    }
537
80.4M
  }
538
  // If this is the end of the main file, form an EOF token.
539
93.7k
  assert(CurLexer && "Got EOF but no current lexer set!");
540
93.7k
  const char *EndPos = getCurLexerEndPos();
541
93.7k
  Result.startToken();
542
93.7k
  CurLexer->BufferPtr = EndPos;
543
544
93.7k
  if (getLangOpts().IncrementalExtensions) {
545
2.14k
    CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_repl_input_end);
546
2.14k
    Result.setAnnotationEndLoc(Result.getLocation());
547
2.14k
    Result.setAnnotationValue(nullptr);
548
91.6k
  } else {
549
91.6k
    CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
550
91.6k
  }
551
552
93.7k
  if (isCodeCompletionEnabled()) {
553
    // Inserting the code-completion point increases the source buffer by 1,
554
    // but the main FileID was created before inserting the point.
555
    // Compensate by reducing the EOF location by 1, otherwise the location
556
    // will point to the next FileID.
557
    // FIXME: This is hacky, the code-completion point should probably be
558
    // inserted before the main FileID is created.
559
121
    if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
560
121
      Result.setLocation(Result.getLocation().getLocWithOffset(-1));
561
121
  }
562
563
93.7k
  if (creatingPCHWithThroughHeader() && 
!LeavingPCHThroughHeader13
) {
564
    // Reached the end of the compilation without finding the through header.
565
1
    Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
566
1
        << PPOpts->PCHThroughHeader << 0;
567
1
  }
568
569
93.7k
  if (!isIncrementalProcessingEnabled())
570
    // We're done with lexing.
571
91.6k
    CurLexer.reset();
572
573
93.7k
  if (!isIncrementalProcessingEnabled())
574
91.6k
    CurPPLexer = nullptr;
575
576
93.7k
  if (TUKind == TU_Complete) {
577
    // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
578
    // collected all macro locations that we need to warn because they are not
579
    // used.
580
85.3k
    for (WarnUnusedMacroLocsTy::iterator
581
85.3k
           I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
582
85.3k
           I!=E; 
++I8
)
583
8
      Diag(*I, diag::pp_macro_not_used);
584
85.3k
  }
585
586
  // If we are building a module that has an umbrella header, make sure that
587
  // each of the headers within the directory, including all submodules, is
588
  // covered by the umbrella header was actually included by the umbrella
589
  // header.
590
93.7k
  if (Module *Mod = getCurrentModule()) {
591
3.69k
    llvm::SmallVector<const Module *, 4> AllMods;
592
3.69k
    collectAllSubModulesWithUmbrellaHeader(*Mod, AllMods);
593
3.69k
    for (auto *M : AllMods)
594
420
      diagnoseMissingHeaderInUmbrellaDir(*M);
595
3.69k
  }
596
597
93.7k
  return true;
598
93.7k
}
599
600
/// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
601
/// hits the end of its token stream.
602
78.7M
bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
603
78.7M
  assert(CurTokenLexer && !CurPPLexer &&
604
78.7M
         "Ending a macro when currently in a #include file!");
605
606
78.7M
  if (!MacroExpandingLexersStack.empty() &&
607
78.7M
      
MacroExpandingLexersStack.back().first == CurTokenLexer.get()34.2M
)
608
31.8M
    removeCachedMacroExpandedTokensOfLastLexer();
609
610
  // Delete or cache the now-dead macro expander.
611
78.7M
  if (NumCachedTokenLexers == TokenLexerCacheSize)
612
1.30M
    CurTokenLexer.reset();
613
77.4M
  else
614
77.4M
    TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
615
616
  // Handle this like a #include file being popped off the stack.
617
78.7M
  return HandleEndOfFile(Result, true);
618
78.7M
}
619
620
/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
621
/// lexer stack.  This should only be used in situations where the current
622
/// state of the top-of-stack lexer is unknown.
623
279M
void Preprocessor::RemoveTopOfLexerStack() {
624
279M
  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
625
626
279M
  if (CurTokenLexer) {
627
    // Delete or cache the now-dead macro expander.
628
11.3M
    if (NumCachedTokenLexers == TokenLexerCacheSize)
629
2.69k
      CurTokenLexer.reset();
630
11.3M
    else
631
11.3M
      TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
632
11.3M
  }
633
634
279M
  PopIncludeMacroStack();
635
279M
}
636
637
/// HandleMicrosoftCommentPaste - When the macro expander pastes together a
638
/// comment (/##/) in microsoft mode, this method handles updating the current
639
/// state, returning the token on the next source line.
640
4
void Preprocessor::HandleMicrosoftCommentPaste(Token &Tok) {
641
4
  assert(CurTokenLexer && !CurPPLexer &&
642
4
         "Pasted comment can only be formed from macro");
643
  // We handle this by scanning for the closest real lexer, switching it to
644
  // raw mode and preprocessor mode.  This will cause it to return \n as an
645
  // explicit EOD token.
646
4
  PreprocessorLexer *FoundLexer = nullptr;
647
4
  bool LexerWasInPPMode = false;
648
6
  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
649
6
    if (ISI.ThePPLexer == nullptr) 
continue2
; // Scan for a real lexer.
650
651
    // Once we find a real lexer, mark it as raw mode (disabling macro
652
    // expansions) and preprocessor mode (return EOD).  We know that the lexer
653
    // was *not* in raw mode before, because the macro that the comment came
654
    // from was expanded.  However, it could have already been in preprocessor
655
    // mode (#if COMMENT) in which case we have to return it to that mode and
656
    // return EOD.
657
4
    FoundLexer = ISI.ThePPLexer;
658
4
    FoundLexer->LexingRawMode = true;
659
4
    LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
660
4
    FoundLexer->ParsingPreprocessorDirective = true;
661
4
    break;
662
6
  }
663
664
  // Okay, we either found and switched over the lexer, or we didn't find a
665
  // lexer.  In either case, finish off the macro the comment came from, getting
666
  // the next token.
667
4
  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
668
669
  // Discarding comments as long as we don't have EOF or EOD.  This 'comments
670
  // out' the rest of the line, including any tokens that came from other macros
671
  // that were active, as in:
672
  //  #define submacro a COMMENT b
673
  //    submacro c
674
  // which should lex to 'a' only: 'b' and 'c' should be removed.
675
30
  while (Tok.isNot(tok::eod) && 
Tok.isNot(tok::eof)26
)
676
26
    Lex(Tok);
677
678
  // If we got an eod token, then we successfully found the end of the line.
679
4
  if (Tok.is(tok::eod)) {
680
4
    assert(FoundLexer && "Can't get end of line without an active lexer");
681
    // Restore the lexer back to normal mode instead of raw mode.
682
4
    FoundLexer->LexingRawMode = false;
683
684
    // If the lexer was already in preprocessor mode, just return the EOD token
685
    // to finish the preprocessor line.
686
4
    if (LexerWasInPPMode) 
return0
;
687
688
    // Otherwise, switch out of PP mode and return the next lexed token.
689
4
    FoundLexer->ParsingPreprocessorDirective = false;
690
4
    return Lex(Tok);
691
4
  }
692
693
  // If we got an EOF token, then we reached the end of the token stream but
694
  // didn't find an explicit \n.  This can only happen if there was no lexer
695
  // active (an active lexer would return EOD at EOF if there was no \n in
696
  // preprocessor directive mode), so just return EOF as our token.
697
0
  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
698
0
}
699
700
void Preprocessor::EnterSubmodule(Module *M, SourceLocation ImportLoc,
701
23.8k
                                  bool ForPragma) {
702
23.8k
  if (!getLangOpts().ModulesLocalVisibility) {
703
    // Just track that we entered this submodule.
704
23.4k
    BuildingSubmoduleStack.push_back(
705
23.4k
        BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
706
23.4k
                              PendingModuleMacroNames.size()));
707
23.4k
    if (Callbacks)
708
23.4k
      Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
709
23.4k
    return;
710
23.4k
  }
711
712
  // Resolve as much of the module definition as we can now, before we enter
713
  // one of its headers.
714
  // FIXME: Can we enable Complain here?
715
  // FIXME: Can we do this when local visibility is disabled?
716
333
  ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
717
333
  ModMap.resolveExports(M, /*Complain=*/false);
718
333
  ModMap.resolveUses(M, /*Complain=*/false);
719
333
  ModMap.resolveConflicts(M, /*Complain=*/false);
720
721
  // If this is the first time we've entered this module, set up its state.
722
333
  auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
723
333
  auto &State = R.first->second;
724
333
  bool FirstTime = R.second;
725
333
  if (FirstTime) {
726
    // Determine the set of starting macros for this submodule; take these
727
    // from the "null" module (the predefines buffer).
728
    //
729
    // FIXME: If we have local visibility but not modules enabled, the
730
    // NullSubmoduleState is polluted by #defines in the top-level source
731
    // file.
732
300
    auto &StartingMacros = NullSubmoduleState.Macros;
733
734
    // Restore to the starting state.
735
    // FIXME: Do this lazily, when each macro name is first referenced.
736
132k
    for (auto &Macro : StartingMacros) {
737
      // Skip uninteresting macros.
738
132k
      if (!Macro.second.getLatest() &&
739
132k
          
Macro.second.getOverriddenMacros().empty()4
)
740
4
        continue;
741
742
132k
      MacroState MS(Macro.second.getLatest());
743
132k
      MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
744
132k
      State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
745
132k
    }
746
300
  }
747
748
  // Track that we entered this module.
749
333
  BuildingSubmoduleStack.push_back(
750
333
      BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
751
333
                            PendingModuleMacroNames.size()));
752
753
333
  if (Callbacks)
754
333
    Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
755
756
  // Switch to this submodule as the current submodule.
757
333
  CurSubmoduleState = &State;
758
759
  // This module is visible to itself.
760
333
  if (FirstTime)
761
300
    makeModuleVisible(M, ImportLoc);
762
333
}
763
764
49.3M
bool Preprocessor::needModuleMacros() const {
765
  // If we're not within a submodule, we never need to create ModuleMacros.
766
49.3M
  if (BuildingSubmoduleStack.empty())
767
49.1M
    return false;
768
  // If we are tracking module macro visibility even for textually-included
769
  // headers, we need ModuleMacros.
770
267k
  if (getLangOpts().ModulesLocalVisibility)
771
598
    return true;
772
  // Otherwise, we only need module macros if we're actually compiling a module
773
  // interface.
774
266k
  return getLangOpts().isCompilingModule();
775
267k
}
776
777
23.8k
Module *Preprocessor::LeaveSubmodule(bool ForPragma) {
778
23.8k
  if (BuildingSubmoduleStack.empty() ||
779
23.8k
      
BuildingSubmoduleStack.back().IsPragma != ForPragma23.8k
) {
780
3
    assert(ForPragma && "non-pragma module enter/leave mismatch");
781
3
    return nullptr;
782
3
  }
783
784
23.8k
  auto &Info = BuildingSubmoduleStack.back();
785
786
23.8k
  Module *LeavingMod = Info.M;
787
23.8k
  SourceLocation ImportLoc = Info.ImportLoc;
788
789
23.8k
  if (!needModuleMacros() ||
790
23.8k
      
(23.6k
!getLangOpts().ModulesLocalVisibility23.6k
&&
791
23.6k
       
LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule23.3k
)) {
792
    // If we don't need module macros, or this is not a module for which we
793
    // are tracking macro visibility, don't build any, and preserve the list
794
    // of pending names for the surrounding submodule.
795
124
    BuildingSubmoduleStack.pop_back();
796
797
124
    if (Callbacks)
798
122
      Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
799
800
124
    makeModuleVisible(LeavingMod, ImportLoc);
801
124
    return LeavingMod;
802
124
  }
803
804
  // Create ModuleMacros for any macros defined in this submodule.
805
23.6k
  llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
806
23.6k
  for (unsigned I = Info.OuterPendingModuleMacroNames;
807
267k
       I != PendingModuleMacroNames.size(); 
++I243k
) {
808
243k
    auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
809
243k
    if (!VisitedMacros.insert(II).second)
810
5.21k
      continue;
811
812
238k
    auto MacroIt = CurSubmoduleState->Macros.find(II);
813
238k
    if (MacroIt == CurSubmoduleState->Macros.end())
814
0
      continue;
815
238k
    auto &Macro = MacroIt->second;
816
817
    // Find the starting point for the MacroDirective chain in this submodule.
818
238k
    MacroDirective *OldMD = nullptr;
819
238k
    auto *OldState = Info.OuterSubmoduleState;
820
238k
    if (getLangOpts().ModulesLocalVisibility)
821
238
      OldState = &NullSubmoduleState;
822
238k
    if (OldState && OldState != CurSubmoduleState) {
823
      // FIXME: It'd be better to start at the state from when we most recently
824
      // entered this submodule, but it doesn't really matter.
825
238
      auto &OldMacros = OldState->Macros;
826
238
      auto OldMacroIt = OldMacros.find(II);
827
238
      if (OldMacroIt == OldMacros.end())
828
238
        OldMD = nullptr;
829
0
      else
830
0
        OldMD = OldMacroIt->second.getLatest();
831
238
    }
832
833
    // This module may have exported a new macro. If so, create a ModuleMacro
834
    // representing that fact.
835
238k
    bool ExplicitlyPublic = false;
836
238k
    for (auto *MD = Macro.getLatest(); MD != OldMD; 
MD = MD->getPrevious()12
) {
837
238k
      assert(MD && "broken macro directive chain");
838
839
238k
      if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
840
        // The latest visibility directive for a name in a submodule affects
841
        // all the directives that come before it.
842
185
        if (VisMD->isPublic())
843
12
          ExplicitlyPublic = true;
844
173
        else if (!ExplicitlyPublic)
845
          // Private with no following public directive: not exported.
846
173
          break;
847
238k
      } else {
848
238k
        MacroInfo *Def = nullptr;
849
238k
        if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
850
236k
          Def = DefMD->getInfo();
851
852
        // FIXME: Issue a warning if multiple headers for the same submodule
853
        // define a macro, rather than silently ignoring all but the first.
854
238k
        bool IsNew;
855
        // Don't bother creating a module macro if it would represent a #undef
856
        // that doesn't override anything.
857
238k
        if (Def || 
!Macro.getOverriddenMacros().empty()1.66k
)
858
236k
          addModuleMacro(LeavingMod, II, Def,
859
236k
                         Macro.getOverriddenMacros(), IsNew);
860
861
238k
        if (!getLangOpts().ModulesLocalVisibility) {
862
          // This macro is exposed to the rest of this compilation as a
863
          // ModuleMacro; we don't need to track its MacroDirective any more.
864
237k
          Macro.setLatest(nullptr);
865
237k
          Macro.setOverriddenMacros(*this, {});
866
237k
        }
867
238k
        break;
868
238k
      }
869
238k
    }
870
238k
  }
871
23.6k
  PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
872
873
  // FIXME: Before we leave this submodule, we should parse all the other
874
  // headers within it. Otherwise, we're left with an inconsistent state
875
  // where we've made the module visible but don't yet have its complete
876
  // contents.
877
878
  // Put back the outer module's state, if we're tracking it.
879
23.6k
  if (getLangOpts().ModulesLocalVisibility)
880
333
    CurSubmoduleState = Info.OuterSubmoduleState;
881
882
23.6k
  BuildingSubmoduleStack.pop_back();
883
884
23.6k
  if (Callbacks)
885
23.6k
    Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
886
887
  // A nested #include makes the included submodule visible.
888
23.6k
  makeModuleVisible(LeavingMod, ImportLoc);
889
23.6k
  return LeavingMod;
890
23.6k
}