Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/include/clang/Lex/Preprocessor.h
Line
Count
Source (jump to first uncovered line)
1
//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// Defines the clang::Preprocessor interface.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15
#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
17
#include "clang/Basic/Builtins.h"
18
#include "clang/Basic/Diagnostic.h"
19
#include "clang/Basic/IdentifierTable.h"
20
#include "clang/Basic/LLVM.h"
21
#include "clang/Basic/LangOptions.h"
22
#include "clang/Basic/Module.h"
23
#include "clang/Basic/SourceLocation.h"
24
#include "clang/Basic/SourceManager.h"
25
#include "clang/Basic/TokenKinds.h"
26
#include "clang/Lex/Lexer.h"
27
#include "clang/Lex/MacroInfo.h"
28
#include "clang/Lex/ModuleLoader.h"
29
#include "clang/Lex/ModuleMap.h"
30
#include "clang/Lex/PPCallbacks.h"
31
#include "clang/Lex/Token.h"
32
#include "clang/Lex/TokenLexer.h"
33
#include "llvm/ADT/ArrayRef.h"
34
#include "llvm/ADT/DenseMap.h"
35
#include "llvm/ADT/FoldingSet.h"
36
#include "llvm/ADT/FunctionExtras.h"
37
#include "llvm/ADT/None.h"
38
#include "llvm/ADT/Optional.h"
39
#include "llvm/ADT/PointerUnion.h"
40
#include "llvm/ADT/STLExtras.h"
41
#include "llvm/ADT/SmallPtrSet.h"
42
#include "llvm/ADT/SmallVector.h"
43
#include "llvm/ADT/StringRef.h"
44
#include "llvm/ADT/TinyPtrVector.h"
45
#include "llvm/ADT/iterator_range.h"
46
#include "llvm/Support/Allocator.h"
47
#include "llvm/Support/Casting.h"
48
#include "llvm/Support/Registry.h"
49
#include <cassert>
50
#include <cstddef>
51
#include <cstdint>
52
#include <map>
53
#include <memory>
54
#include <string>
55
#include <utility>
56
#include <vector>
57
58
namespace llvm {
59
60
template<unsigned InternalLen> class SmallString;
61
62
} // namespace llvm
63
64
namespace clang {
65
66
class CodeCompletionHandler;
67
class CommentHandler;
68
class DirectoryEntry;
69
class DirectoryLookup;
70
class ExternalPreprocessorSource;
71
class FileEntry;
72
class FileManager;
73
class HeaderSearch;
74
class MacroArgs;
75
class PragmaHandler;
76
class PragmaNamespace;
77
class PreprocessingRecord;
78
class PreprocessorLexer;
79
class PreprocessorOptions;
80
class ScratchBuffer;
81
class TargetInfo;
82
83
/// Stores token information for comparing actual tokens with
84
/// predefined values.  Only handles simple tokens and identifiers.
85
class TokenValue {
86
  tok::TokenKind Kind;
87
  IdentifierInfo *II;
88
89
public:
90
664
  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
91
664
    assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
92
664
    assert(Kind != tok::identifier &&
93
664
           "Identifiers should be created by TokenValue(IdentifierInfo *)");
94
664
    assert(!tok::isLiteral(Kind) && "Literals are not supported.");
95
664
    assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
96
664
  }
97
98
197
  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
99
100
6.25k
  bool operator==(const Token &Tok) const {
101
6.25k
    return Tok.getKind() == Kind &&
102
6.25k
        
(855
!II855
||
II == Tok.getIdentifierInfo()208
);
103
6.25k
  }
104
};
105
106
/// Context in which macro name is used.
107
enum MacroUse {
108
  // other than #define or #undef
109
  MU_Other  = 0,
110
111
  // macro name specified in #define
112
  MU_Define = 1,
113
114
  // macro name specified in #undef
115
  MU_Undef  = 2
116
};
117
118
/// Engages in a tight little dance with the lexer to efficiently
119
/// preprocess tokens.
120
///
121
/// Lexers know only about tokens within a single source file, and don't
122
/// know anything about preprocessor-level issues like the \#include stack,
123
/// token expansion, etc.
124
class Preprocessor {
125
  friend class VAOptDefinitionContext;
126
  friend class VariadicMacroScopeGuard;
127
128
  llvm::unique_function<void(const clang::Token &)> OnToken;
129
  std::shared_ptr<PreprocessorOptions> PPOpts;
130
  DiagnosticsEngine        *Diags;
131
  LangOptions       &LangOpts;
132
  const TargetInfo *Target = nullptr;
133
  const TargetInfo *AuxTarget = nullptr;
134
  FileManager       &FileMgr;
135
  SourceManager     &SourceMgr;
136
  std::unique_ptr<ScratchBuffer> ScratchBuf;
137
  HeaderSearch      &HeaderInfo;
138
  ModuleLoader      &TheModuleLoader;
139
140
  /// External source of macros.
141
  ExternalPreprocessorSource *ExternalSource;
142
143
  /// A BumpPtrAllocator object used to quickly allocate and release
144
  /// objects internal to the Preprocessor.
145
  llvm::BumpPtrAllocator BP;
146
147
  /// Identifiers for builtin macros and other builtins.
148
  IdentifierInfo *Ident__LINE__, *Ident__FILE__;   // __LINE__, __FILE__
149
  IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
150
  IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
151
  IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
152
  IdentifierInfo *Ident__FILE_NAME__;              // __FILE_NAME__
153
  IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
154
  IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
155
  IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
156
  IdentifierInfo *Ident__identifier;               // __identifier
157
  IdentifierInfo *Ident__VA_ARGS__;                // __VA_ARGS__
158
  IdentifierInfo *Ident__VA_OPT__;                 // __VA_OPT__
159
  IdentifierInfo *Ident__has_feature;              // __has_feature
160
  IdentifierInfo *Ident__has_extension;            // __has_extension
161
  IdentifierInfo *Ident__has_builtin;              // __has_builtin
162
  IdentifierInfo *Ident__has_attribute;            // __has_attribute
163
  IdentifierInfo *Ident__has_include;              // __has_include
164
  IdentifierInfo *Ident__has_include_next;         // __has_include_next
165
  IdentifierInfo *Ident__has_warning;              // __has_warning
166
  IdentifierInfo *Ident__is_identifier;            // __is_identifier
167
  IdentifierInfo *Ident__building_module;          // __building_module
168
  IdentifierInfo *Ident__MODULE__;                 // __MODULE__
169
  IdentifierInfo *Ident__has_cpp_attribute;        // __has_cpp_attribute
170
  IdentifierInfo *Ident__has_c_attribute;          // __has_c_attribute
171
  IdentifierInfo *Ident__has_declspec;             // __has_declspec_attribute
172
  IdentifierInfo *Ident__is_target_arch;           // __is_target_arch
173
  IdentifierInfo *Ident__is_target_vendor;         // __is_target_vendor
174
  IdentifierInfo *Ident__is_target_os;             // __is_target_os
175
  IdentifierInfo *Ident__is_target_environment;    // __is_target_environment
176
177
  // Weak, only valid (and set) while InMacroArgs is true.
178
  Token* ArgMacro;
179
180
  SourceLocation DATELoc, TIMELoc;
181
182
  // Next __COUNTER__ value, starts at 0.
183
  unsigned CounterValue = 0;
184
185
  enum {
186
    /// Maximum depth of \#includes.
187
    MaxAllowedIncludeStackDepth = 200
188
  };
189
190
  // State that is set before the preprocessor begins.
191
  bool KeepComments : 1;
192
  bool KeepMacroComments : 1;
193
  bool SuppressIncludeNotFoundError : 1;
194
195
  // State that changes while the preprocessor runs:
196
  bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
197
198
  /// Whether the preprocessor owns the header search object.
199
  bool OwnsHeaderSearch : 1;
200
201
  /// True if macro expansion is disabled.
202
  bool DisableMacroExpansion : 1;
203
204
  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
205
  /// when parsing preprocessor directives.
206
  bool MacroExpansionInDirectivesOverride : 1;
207
208
  class ResetMacroExpansionHelper;
209
210
  /// Whether we have already loaded macros from the external source.
211
  mutable bool ReadMacrosFromExternalSource : 1;
212
213
  /// True if pragmas are enabled.
214
  bool PragmasEnabled : 1;
215
216
  /// True if the current build action is a preprocessing action.
217
  bool PreprocessedOutput : 1;
218
219
  /// True if we are currently preprocessing a #if or #elif directive
220
  bool ParsingIfOrElifDirective;
221
222
  /// True if we are pre-expanding macro arguments.
223
  bool InMacroArgPreExpansion;
224
225
  /// Mapping/lookup information for all identifiers in
226
  /// the program, including program keywords.
227
  mutable IdentifierTable Identifiers;
228
229
  /// This table contains all the selectors in the program.
230
  ///
231
  /// Unlike IdentifierTable above, this table *isn't* populated by the
232
  /// preprocessor. It is declared/expanded here because its role/lifetime is
233
  /// conceptually similar to the IdentifierTable. In addition, the current
234
  /// control flow (in clang::ParseAST()), make it convenient to put here.
235
  ///
236
  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
237
  /// the lifetime of the preprocessor.
238
  SelectorTable Selectors;
239
240
  /// Information about builtins.
241
  Builtin::Context BuiltinInfo;
242
243
  /// Tracks all of the pragmas that the client registered
244
  /// with this preprocessor.
245
  std::unique_ptr<PragmaNamespace> PragmaHandlers;
246
247
  /// Pragma handlers of the original source is stored here during the
248
  /// parsing of a model file.
249
  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
250
251
  /// Tracks all of the comment handlers that the client registered
252
  /// with this preprocessor.
253
  std::vector<CommentHandler *> CommentHandlers;
254
255
  /// True if we want to ignore EOF token and continue later on (thus
256
  /// avoid tearing the Lexer and etc. down).
257
  bool IncrementalProcessing = false;
258
259
  /// The kind of translation unit we are processing.
260
  TranslationUnitKind TUKind;
261
262
  /// The code-completion handler.
263
  CodeCompletionHandler *CodeComplete = nullptr;
264
265
  /// The file that we're performing code-completion for, if any.
266
  const FileEntry *CodeCompletionFile = nullptr;
267
268
  /// The offset in file for the code-completion point.
269
  unsigned CodeCompletionOffset = 0;
270
271
  /// The location for the code-completion point. This gets instantiated
272
  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
273
  SourceLocation CodeCompletionLoc;
274
275
  /// The start location for the file of the code-completion point.
276
  ///
277
  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
278
  /// for preprocessing.
279
  SourceLocation CodeCompletionFileLoc;
280
281
  /// The source location of the \c import contextual keyword we just
282
  /// lexed, if any.
283
  SourceLocation ModuleImportLoc;
284
285
  /// The module import path that we're currently processing.
286
  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath;
287
288
  /// Whether the last token we lexed was an '@'.
289
  bool LastTokenWasAt = false;
290
291
  /// A position within a C++20 import-seq.
292
  class ImportSeq {
293
  public:
294
    enum State : int {
295
      // Positive values represent a number of unclosed brackets.
296
      AtTopLevel = 0,
297
      AfterTopLevelTokenSeq = -1,
298
      AfterExport = -2,
299
      AfterImportSeq = -3,
300
    };
301
302
44.2k
    ImportSeq(State S) : S(S) {}
303
304
    /// Saw any kind of open bracket.
305
18.5k
    void handleOpenBracket() {
306
18.5k
      S = static_cast<State>(std::max<int>(S, 0) + 1);
307
18.5k
    }
308
    /// Saw any kind of close bracket other than '}'.
309
18.5k
    void handleCloseBracket() {
310
18.5k
      S = static_cast<State>(std::max<int>(S, 1) - 1);
311
18.5k
    }
312
    /// Saw a close brace.
313
4.46k
    void handleCloseBrace() {
314
4.46k
      handleCloseBracket();
315
4.46k
      if (S == AtTopLevel && 
!AfterHeaderName1.13k
)
316
1.13k
        S = AfterTopLevelTokenSeq;
317
4.46k
    }
318
    /// Saw a semicolon.
319
10.8k
    void handleSemi() {
320
10.8k
      if (atTopLevel()) {
321
990
        S = AfterTopLevelTokenSeq;
322
990
        AfterHeaderName = false;
323
990
      }
324
10.8k
    }
325
326
    /// Saw an 'export' identifier.
327
86
    void handleExport() {
328
86
      if (S == AfterTopLevelTokenSeq)
329
75
        S = AfterExport;
330
11
      else if (S <= 0)
331
2
        S = AtTopLevel;
332
86
    }
333
    /// Saw an 'import' identifier.
334
50
    void handleImport() {
335
50
      if (S == AfterTopLevelTokenSeq || 
S == AfterExport20
)
336
32
        S = AfterImportSeq;
337
18
      else if (S <= 0)
338
13
        S = AtTopLevel;
339
50
    }
340
341
    /// Saw a 'header-name' token; do not recognize any more 'import' tokens
342
    /// until we reach a top-level semicolon.
343
24
    void handleHeaderName() {
344
24
      if (S == AfterImportSeq)
345
24
        AfterHeaderName = true;
346
24
      handleMisc();
347
24
    }
348
349
    /// Saw any other token.
350
84.0k
    void handleMisc() {
351
84.0k
      if (S <= 0)
352
6.33k
        S = AtTopLevel;
353
84.0k
    }
354
355
10.8k
    bool atTopLevel() { return S <= 0; }
356
50
    bool afterImportSeq() { return S == AfterImportSeq; }
357
358
  private:
359
    State S;
360
    /// Whether we're in the pp-import-suffix following the header-name in a
361
    /// pp-import. If so, a close-brace is not sufficient to end the
362
    /// top-level-token-seq of an import-seq.
363
    bool AfterHeaderName = false;
364
  };
365
366
  /// Our current position within a C++20 import-seq.
367
  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
368
369
  /// Whether the module import expects an identifier next. Otherwise,
370
  /// it expects a '.' or ';'.
371
  bool ModuleImportExpectsIdentifier = false;
372
373
  /// The source location of the currently-active
374
  /// \#pragma clang arc_cf_code_audited begin.
375
  SourceLocation PragmaARCCFCodeAuditedLoc;
376
377
  /// The source location of the currently-active
378
  /// \#pragma clang assume_nonnull begin.
379
  SourceLocation PragmaAssumeNonNullLoc;
380
381
  /// True if we hit the code-completion point.
382
  bool CodeCompletionReached = false;
383
384
  /// The code completion token containing the information
385
  /// on the stem that is to be code completed.
386
  IdentifierInfo *CodeCompletionII = nullptr;
387
388
  /// Range for the code completion token.
389
  SourceRange CodeCompletionTokenRange;
390
391
  /// The directory that the main file should be considered to occupy,
392
  /// if it does not correspond to a real file (as happens when building a
393
  /// module).
394
  const DirectoryEntry *MainFileDir = nullptr;
395
396
  /// The number of bytes that we will initially skip when entering the
397
  /// main file, along with a flag that indicates whether skipping this number
398
  /// of bytes will place the lexer at the start of a line.
399
  ///
400
  /// This is used when loading a precompiled preamble.
401
  std::pair<int, bool> SkipMainFilePreamble;
402
403
  /// Whether we hit an error due to reaching max allowed include depth. Allows
404
  /// to avoid hitting the same error over and over again.
405
  bool HasReachedMaxIncludeDepth = false;
406
407
  /// The number of currently-active calls to Lex.
408
  ///
409
  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
410
  /// require asking for multiple additional tokens. This counter makes it
411
  /// possible for Lex to detect whether it's producing a token for the end
412
  /// of phase 4 of translation or for some other situation.
413
  unsigned LexLevel = 0;
414
415
public:
416
  struct PreambleSkipInfo {
417
    SourceLocation HashTokenLoc;
418
    SourceLocation IfTokenLoc;
419
    bool FoundNonSkipPortion;
420
    bool FoundElse;
421
    SourceLocation ElseLoc;
422
423
    PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
424
                     bool FoundNonSkipPortion, bool FoundElse,
425
                     SourceLocation ElseLoc)
426
        : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
427
          FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
428
23
          ElseLoc(ElseLoc) {}
429
  };
430
431
private:
432
  friend class ASTReader;
433
  friend class MacroArgs;
434
435
  class PreambleConditionalStackStore {
436
    enum State {
437
      Off = 0,
438
      Recording = 1,
439
      Replaying = 2,
440
    };
441
442
  public:
443
44.2k
    PreambleConditionalStackStore() = default;
444
445
89
    void startRecording() { ConditionalStackState = Recording; }
446
28
    void startReplaying() { ConditionalStackState = Replaying; }
447
1.60M
    bool isRecording() const { return ConditionalStackState == Recording; }
448
44.0k
    bool isReplaying() const { return ConditionalStackState == Replaying; }
449
450
35
    ArrayRef<PPConditionalInfo> getStack() const {
451
35
      return ConditionalStack;
452
35
    }
453
454
28
    void doneReplaying() {
455
28
      ConditionalStack.clear();
456
28
      ConditionalStackState = Off;
457
28
    }
458
459
117
    void setStack(ArrayRef<PPConditionalInfo> s) {
460
117
      if (!isRecording() && 
!isReplaying()28
)
461
0
        return;
462
117
      ConditionalStack.clear();
463
117
      ConditionalStack.append(s.begin(), s.end());
464
117
    }
465
466
89
    bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
467
468
4.21M
    bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
469
470
18
    void clearSkipInfo() { SkipInfo.reset(); }
471
472
    llvm::Optional<PreambleSkipInfo> SkipInfo;
473
474
  private:
475
    SmallVector<PPConditionalInfo, 4> ConditionalStack;
476
    State ConditionalStackState = Off;
477
  } PreambleConditionalStack;
478
479
  /// The current top of the stack that we're lexing from if
480
  /// not expanding a macro and we are lexing directly from source code.
481
  ///
482
  /// Only one of CurLexer, or CurTokenLexer will be non-null.
483
  std::unique_ptr<Lexer> CurLexer;
484
485
  /// The current top of the stack what we're lexing from
486
  /// if not expanding a macro.
487
  ///
488
  /// This is an alias for CurLexer.
489
  PreprocessorLexer *CurPPLexer = nullptr;
490
491
  /// Used to find the current FileEntry, if CurLexer is non-null
492
  /// and if applicable.
493
  ///
494
  /// This allows us to implement \#include_next and find directory-specific
495
  /// properties.
496
  const DirectoryLookup *CurDirLookup = nullptr;
497
498
  /// The current macro we are expanding, if we are expanding a macro.
499
  ///
500
  /// One of CurLexer and CurTokenLexer must be null.
501
  std::unique_ptr<TokenLexer> CurTokenLexer;
502
503
  /// The kind of lexer we're currently working with.
504
  enum CurLexerKind {
505
    CLK_Lexer,
506
    CLK_TokenLexer,
507
    CLK_CachingLexer,
508
    CLK_LexAfterModuleImport
509
  } CurLexerKind = CLK_Lexer;
510
511
  /// If the current lexer is for a submodule that is being built, this
512
  /// is that submodule.
513
  Module *CurLexerSubmodule = nullptr;
514
515
  /// Keeps track of the stack of files currently
516
  /// \#included, and macros currently being expanded from, not counting
517
  /// CurLexer/CurTokenLexer.
518
  struct IncludeStackInfo {
519
    enum CurLexerKind           CurLexerKind;
520
    Module                     *TheSubmodule;
521
    std::unique_ptr<Lexer>      TheLexer;
522
    PreprocessorLexer          *ThePPLexer;
523
    std::unique_ptr<TokenLexer> TheTokenLexer;
524
    const DirectoryLookup      *TheDirLookup;
525
526
    // The following constructors are completely useless copies of the default
527
    // versions, only needed to pacify MSVC.
528
    IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
529
                     std::unique_ptr<Lexer> &&TheLexer,
530
                     PreprocessorLexer *ThePPLexer,
531
                     std::unique_ptr<TokenLexer> &&TheTokenLexer,
532
                     const DirectoryLookup *TheDirLookup)
533
        : CurLexerKind(std::move(CurLexerKind)),
534
          TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
535
          ThePPLexer(std::move(ThePPLexer)),
536
          TheTokenLexer(std::move(TheTokenLexer)),
537
86.7M
          TheDirLookup(std::move(TheDirLookup)) {}
538
  };
539
  std::vector<IncludeStackInfo> IncludeMacroStack;
540
541
  /// Actions invoked when some preprocessor activity is
542
  /// encountered (e.g. a file is \#included, etc).
543
  std::unique_ptr<PPCallbacks> Callbacks;
544
545
  struct MacroExpandsInfo {
546
    Token Tok;
547
    MacroDefinition MD;
548
    SourceRange Range;
549
550
    MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
551
6
        : Tok(Tok), MD(MD), Range(Range) {}
552
  };
553
  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
554
555
  /// Information about a name that has been used to define a module macro.
556
  struct ModuleMacroInfo {
557
    /// The most recent macro directive for this identifier.
558
    MacroDirective *MD;
559
560
    /// The active module macros for this identifier.
561
    llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
562
563
    /// The generation number at which we last updated ActiveModuleMacros.
564
    /// \see Preprocessor::VisibleModules.
565
    unsigned ActiveModuleMacrosGeneration = 0;
566
567
    /// Whether this macro name is ambiguous.
568
    bool IsAmbiguous = false;
569
570
    /// The module macros that are overridden by this macro.
571
    llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
572
573
7.56k
    ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
574
  };
575
576
  /// The state of a macro for an identifier.
577
  class MacroState {
578
    mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
579
580
    ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
581
91.5M
                                   const IdentifierInfo *II) const {
582
91.5M
      if (II->isOutOfDate())
583
530
        PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
584
91.5M
      // FIXME: Find a spare bit on IdentifierInfo and store a
585
91.5M
      //        HasModuleMacros flag.
586
91.5M
      if (!II->hasMacroDefinition() ||
587
91.5M
          
(65.7M
!PP.getLangOpts().Modules65.7M
&&
588
65.7M
           
!PP.getLangOpts().ModulesLocalVisibility65.3M
) ||
589
91.5M
          
!PP.CurSubmoduleState->VisibleModules.getGeneration()359k
)
590
91.2M
        return nullptr;
591
313k
592
313k
      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
593
313k
      if (!Info) {
594
7.56k
        Info = new (PP.getPreprocessorAllocator())
595
7.56k
            ModuleMacroInfo(State.get<MacroDirective *>());
596
7.56k
        State = Info;
597
7.56k
      }
598
313k
599
313k
      if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
600
313k
          Info->ActiveModuleMacrosGeneration)
601
8.17k
        PP.updateModuleMacroInfo(II, *Info);
602
313k
      return Info;
603
313k
    }
604
605
  public:
606
25.7M
    MacroState() : MacroState(nullptr) {}
Unexecuted instantiation: clang::Preprocessor::MacroState::MacroState()
clang::Preprocessor::MacroState::MacroState()
Line
Count
Source
606
25.7M
    MacroState() : MacroState(nullptr) {}
607
25.8M
    MacroState(MacroDirective *MD) : State(MD) {}
608
609
36.1M
    MacroState(MacroState &&O) noexcept : State(O.State) {
610
36.1M
      O.State = (MacroDirective *)nullptr;
611
36.1M
    }
612
613
37.3k
    MacroState &operator=(MacroState &&O) noexcept {
614
37.3k
      auto S = O.State;
615
37.3k
      O.State = (MacroDirective *)nullptr;
616
37.3k
      State = S;
617
37.3k
      return *this;
618
37.3k
    }
619
620
49.0M
    ~MacroState() {
621
49.0M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
622
7.56k
        Info->~ModuleMacroInfo();
623
49.0M
    }
624
625
61.1M
    MacroDirective *getLatest() const {
626
61.1M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
627
157k
        return Info->MD;
628
61.0M
      return State.get<MacroDirective*>();
629
61.0M
    }
630
631
26.3M
    void setLatest(MacroDirective *MD) {
632
26.3M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
633
5.12k
        Info->MD = MD;
634
26.3M
      else
635
26.3M
        State = MD;
636
26.3M
    }
637
638
32.5M
    bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
639
32.5M
      auto *Info = getModuleInfo(PP, II);
640
32.5M
      return Info ? 
Info->IsAmbiguous155k
:
false32.4M
;
641
32.5M
    }
642
643
    ArrayRef<ModuleMacro *>
644
32.5M
    getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
645
32.5M
      if (auto *Info = getModuleInfo(PP, II))
646
155k
        return Info->ActiveModuleMacros;
647
32.4M
      return None;
648
32.4M
    }
649
650
    MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
651
56.4k
                                               SourceManager &SourceMgr) const {
652
56.4k
      // FIXME: Incorporate module macros into the result of this.
653
56.4k
      if (auto *Latest = getLatest())
654
56.4k
        return Latest->findDirectiveAtLoc(Loc, SourceMgr);
655
1
      return {};
656
1
    }
657
658
26.3M
    void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
659
26.3M
      if (auto *Info = getModuleInfo(PP, II)) {
660
1.53k
        Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
661
1.53k
                                      Info->ActiveModuleMacros.begin(),
662
1.53k
                                      Info->ActiveModuleMacros.end());
663
1.53k
        Info->ActiveModuleMacros.clear();
664
1.53k
        Info->IsAmbiguous = false;
665
1.53k
      }
666
26.3M
    }
667
668
114k
    ArrayRef<ModuleMacro*> getOverriddenMacros() const {
669
114k
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
670
3.23k
        return Info->OverriddenMacros;
671
111k
      return None;
672
111k
    }
673
674
    void setOverriddenMacros(Preprocessor &PP,
675
114k
                             ArrayRef<ModuleMacro *> Overrides) {
676
114k
      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
677
114k
      if (!Info) {
678
111k
        if (Overrides.empty())
679
111k
          return;
680
0
        Info = new (PP.getPreprocessorAllocator())
681
0
            ModuleMacroInfo(State.get<MacroDirective *>());
682
0
        State = Info;
683
0
      }
684
114k
      Info->OverriddenMacros.clear();
685
2.97k
      Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
686
2.97k
                                    Overrides.begin(), Overrides.end());
687
2.97k
      Info->ActiveModuleMacrosGeneration = 0;
688
2.97k
    }
689
  };
690
691
  /// For each IdentifierInfo that was associated with a macro, we
692
  /// keep a mapping to the history of all macro definitions and #undefs in
693
  /// the reverse order (the latest one is in the head of the list).
694
  ///
695
  /// This mapping lives within the \p CurSubmoduleState.
696
  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
697
698
  struct SubmoduleState;
699
700
  /// Information about a submodule that we're currently building.
701
  struct BuildingSubmoduleInfo {
702
    /// The module that we are building.
703
    Module *M;
704
705
    /// The location at which the module was included.
706
    SourceLocation ImportLoc;
707
708
    /// Whether we entered this submodule via a pragma.
709
    bool IsPragma;
710
711
    /// The previous SubmoduleState.
712
    SubmoduleState *OuterSubmoduleState;
713
714
    /// The number of pending module macro names when we started building this.
715
    unsigned OuterPendingModuleMacroNames;
716
717
    BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
718
                          SubmoduleState *OuterSubmoduleState,
719
                          unsigned OuterPendingModuleMacroNames)
720
        : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
721
          OuterSubmoduleState(OuterSubmoduleState),
722
2.87k
          OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
723
  };
724
  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
725
726
  /// Information about a submodule's preprocessor state.
727
  struct SubmoduleState {
728
    /// The macros for the submodule.
729
    MacroMap Macros;
730
731
    /// The set of modules that are visible within the submodule.
732
    VisibleModuleSet VisibleModules;
733
734
    // FIXME: CounterValue?
735
    // FIXME: PragmaPushMacroInfo?
736
  };
737
  std::map<Module *, SubmoduleState> Submodules;
738
739
  /// The preprocessor state for preprocessing outside of any submodule.
740
  SubmoduleState NullSubmoduleState;
741
742
  /// The current submodule state. Will be \p NullSubmoduleState if we're not
743
  /// in a submodule.
744
  SubmoduleState *CurSubmoduleState;
745
746
  /// The set of known macros exported from modules.
747
  llvm::FoldingSet<ModuleMacro> ModuleMacros;
748
749
  /// The names of potential module macros that we've not yet processed.
750
  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
751
752
  /// The list of module macros, for each identifier, that are not overridden by
753
  /// any other module macro.
754
  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
755
      LeafModuleMacros;
756
757
  /// Macros that we want to warn because they are not used at the end
758
  /// of the translation unit.
759
  ///
760
  /// We store just their SourceLocations instead of
761
  /// something like MacroInfo*. The benefit of this is that when we are
762
  /// deserializing from PCH, we don't need to deserialize identifier & macros
763
  /// just so that we can report that they are unused, we just warn using
764
  /// the SourceLocations of this set (that will be filled by the ASTReader).
765
  /// We are using SmallPtrSet instead of a vector for faster removal.
766
  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
767
  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
768
769
  /// A "freelist" of MacroArg objects that can be
770
  /// reused for quick allocation.
771
  MacroArgs *MacroArgCache = nullptr;
772
773
  /// For each IdentifierInfo used in a \#pragma push_macro directive,
774
  /// we keep a MacroInfo stack used to restore the previous macro value.
775
  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
776
      PragmaPushMacroInfo;
777
778
  // Various statistics we track for performance analysis.
779
  unsigned NumDirectives = 0;
780
  unsigned NumDefined = 0;
781
  unsigned NumUndefined = 0;
782
  unsigned NumPragma = 0;
783
  unsigned NumIf = 0;
784
  unsigned NumElse = 0;
785
  unsigned NumEndif = 0;
786
  unsigned NumEnteredSourceFiles = 0;
787
  unsigned MaxIncludeStackDepth = 0;
788
  unsigned NumMacroExpanded = 0;
789
  unsigned NumFnMacroExpanded = 0;
790
  unsigned NumBuiltinMacroExpanded = 0;
791
  unsigned NumFastMacroExpanded = 0;
792
  unsigned NumTokenPaste = 0;
793
  unsigned NumFastTokenPaste = 0;
794
  unsigned NumSkipped = 0;
795
796
  /// The predefined macros that preprocessor should use from the
797
  /// command line etc.
798
  std::string Predefines;
799
800
  /// The file ID for the preprocessor predefines.
801
  FileID PredefinesFileID;
802
803
  /// The file ID for the PCH through header.
804
  FileID PCHThroughHeaderFileID;
805
806
  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
807
  bool SkippingUntilPragmaHdrStop = false;
808
809
  /// Whether tokens are being skipped until the through header is seen.
810
  bool SkippingUntilPCHThroughHeader = false;
811
812
  /// \{
813
  /// Cache of macro expanders to reduce malloc traffic.
814
  enum { TokenLexerCacheSize = 8 };
815
  unsigned NumCachedTokenLexers;
816
  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
817
  /// \}
818
819
  /// Keeps macro expanded tokens for TokenLexers.
820
  //
821
  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
822
  /// going to lex in the cache and when it finishes the tokens are removed
823
  /// from the end of the cache.
824
  SmallVector<Token, 16> MacroExpandedTokens;
825
  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
826
827
  /// A record of the macro definitions and expansions that
828
  /// occurred during preprocessing.
829
  ///
830
  /// This is an optional side structure that can be enabled with
831
  /// \c createPreprocessingRecord() prior to preprocessing.
832
  PreprocessingRecord *Record = nullptr;
833
834
  /// Cached tokens state.
835
  using CachedTokensTy = SmallVector<Token, 1>;
836
837
  /// Cached tokens are stored here when we do backtracking or
838
  /// lookahead. They are "lexed" by the CachingLex() method.
839
  CachedTokensTy CachedTokens;
840
841
  /// The position of the cached token that CachingLex() should
842
  /// "lex" next.
843
  ///
844
  /// If it points beyond the CachedTokens vector, it means that a normal
845
  /// Lex() should be invoked.
846
  CachedTokensTy::size_type CachedLexPos = 0;
847
848
  /// Stack of backtrack positions, allowing nested backtracks.
849
  ///
850
  /// The EnableBacktrackAtThisPos() method pushes a position to
851
  /// indicate where CachedLexPos should be set when the BackTrack() method is
852
  /// invoked (at which point the last position is popped).
853
  std::vector<CachedTokensTy::size_type> BacktrackPositions;
854
855
  struct MacroInfoChain {
856
    MacroInfo MI;
857
    MacroInfoChain *Next;
858
  };
859
860
  /// MacroInfos are managed as a chain for easy disposal.  This is the head
861
  /// of that list.
862
  MacroInfoChain *MIChainHead = nullptr;
863
864
  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
865
866
public:
867
  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
868
               DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
869
               HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
870
               IdentifierInfoLookup *IILookup = nullptr,
871
               bool OwnsHeaderSearch = false,
872
               TranslationUnitKind TUKind = TU_Complete);
873
874
  ~Preprocessor();
875
876
  /// Initialize the preprocessor using information about the target.
877
  ///
878
  /// \param Target is owned by the caller and must remain valid for the
879
  /// lifetime of the preprocessor.
880
  /// \param AuxTarget is owned by the caller and must remain valid for
881
  /// the lifetime of the preprocessor.
882
  void Initialize(const TargetInfo &Target,
883
                  const TargetInfo *AuxTarget = nullptr);
884
885
  /// Initialize the preprocessor to parse a model file
886
  ///
887
  /// To parse model files the preprocessor of the original source is reused to
888
  /// preserver the identifier table. However to avoid some duplicate
889
  /// information in the preprocessor some cleanup is needed before it is used
890
  /// to parse model files. This method does that cleanup.
891
  void InitializeForModelFile();
892
893
  /// Cleanup after model file parsing
894
  void FinalizeForModelFile();
895
896
  /// Retrieve the preprocessor options used to initialize this
897
  /// preprocessor.
898
16.6k
  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
899
900
5.14M
  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
901
176
  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
902
903
2.21G
  const LangOptions &getLangOpts() const { return LangOpts; }
904
11.3M
  const TargetInfo &getTargetInfo() const { return *Target; }
905
909
  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
906
69.0k
  FileManager &getFileManager() const { return FileMgr; }
907
308M
  SourceManager &getSourceManager() const { return SourceMgr; }
908
288k
  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
909
910
1.24M
  IdentifierTable &getIdentifierTable() { return Identifiers; }
911
3.61k
  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
912
155k
  SelectorTable &getSelectorTable() { return Selectors; }
913
83.0k
  Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
914
184k
  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
915
916
4.03k
  void setExternalSource(ExternalPreprocessorSource *Source) {
917
4.03k
    ExternalSource = Source;
918
4.03k
  }
919
920
752k
  ExternalPreprocessorSource *getExternalSource() const {
921
752k
    return ExternalSource;
922
752k
  }
923
924
  /// Retrieve the module loader associated with this preprocessor.
925
8.98k
  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
926
927
37.4M
  bool hadModuleLoaderFatalFailure() const {
928
37.4M
    return TheModuleLoader.HadFatalFailure;
929
37.4M
  }
930
931
  /// True if we are currently preprocessing a #if or #elif directive
932
23.1k
  bool isParsingIfOrElifDirective() const {
933
23.1k
    return ParsingIfOrElifDirective;
934
23.1k
  }
935
936
  /// Control whether the preprocessor retains comments in output.
937
1.26k
  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
938
1.26k
    this->KeepComments = KeepComments | KeepMacroComments;
939
1.26k
    this->KeepMacroComments = KeepMacroComments;
940
1.26k
  }
941
942
168M
  bool getCommentRetentionState() const { return KeepComments; }
943
944
176
  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
945
88
  bool getPragmasEnabled() const { return PragmasEnabled; }
946
947
4
  void SetSuppressIncludeNotFoundError(bool Suppress) {
948
4
    SuppressIncludeNotFoundError = Suppress;
949
4
  }
950
951
0
  bool GetSuppressIncludeNotFoundError() {
952
0
    return SuppressIncludeNotFoundError;
953
0
  }
954
955
  /// Sets whether the preprocessor is responsible for producing output or if
956
  /// it is producing tokens to be consumed by Parse and Sema.
957
44.0k
  void setPreprocessedOutput(bool IsPreprocessedOutput) {
958
44.0k
    PreprocessedOutput = IsPreprocessedOutput;
959
44.0k
  }
960
961
  /// Returns true if the preprocessor is responsible for generating output,
962
  /// false if it is producing tokens to be consumed by Parse and Sema.
963
2.45k
  bool isPreprocessedOutput() const { return PreprocessedOutput; }
964
965
  /// Return true if we are lexing directly from the specified lexer.
966
0
  bool isCurrentLexer(const PreprocessorLexer *L) const {
967
0
    return CurPPLexer == L;
968
0
  }
969
970
  /// Return the current lexer being lexed from.
971
  ///
972
  /// Note that this ignores any potentially active macro expansions and _Pragma
973
  /// expansions going on at the time.
974
41.4k
  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
975
976
  /// Return the current file lexer being lexed from.
977
  ///
978
  /// Note that this ignores any potentially active macro expansions and _Pragma
979
  /// expansions going on at the time.
980
  PreprocessorLexer *getCurrentFileLexer() const;
981
982
  /// Return the submodule owning the file being lexed. This may not be
983
  /// the current module if we have changed modules since entering the file.
984
12.0k
  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
985
986
  /// Returns the FileID for the preprocessor predefines.
987
548k
  FileID getPredefinesFileID() const { return PredefinesFileID; }
988
989
  /// \{
990
  /// Accessors for preprocessor callbacks.
991
  ///
992
  /// Note that this class takes ownership of any PPCallbacks object given to
993
  /// it.
994
1.96M
  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
995
59.2k
  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
996
59.2k
    if (Callbacks)
997
16.4k
      C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
998
16.4k
                                                std::move(Callbacks));
999
59.2k
    Callbacks = std::move(C);
1000
59.2k
  }
1001
  /// \}
1002
1003
  /// Register a function that would be called on each token in the final
1004
  /// expanded token stream.
1005
  /// This also reports annotation tokens produced by the parser.
1006
  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1007
    OnToken = std::move(F);
1008
  }
1009
1010
243
  bool isMacroDefined(StringRef Id) {
1011
243
    return isMacroDefined(&Identifiers.get(Id));
1012
243
  }
1013
881k
  bool isMacroDefined(const IdentifierInfo *II) {
1014
881k
    return II->hasMacroDefinition() &&
1015
881k
           
(880k
!getLangOpts().Modules880k
||
(bool)getMacroDefinition(II)561
);
1016
881k
  }
1017
1018
  /// Determine whether II is defined as a macro within the module M,
1019
  /// if that is a module that we've already preprocessed. Does not check for
1020
  /// macros imported into M.
1021
158
  bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) {
1022
158
    if (!II->hasMacroDefinition())
1023
0
      return false;
1024
158
    auto I = Submodules.find(M);
1025
158
    if (I == Submodules.end())
1026
144
      return false;
1027
14
    auto J = I->second.Macros.find(II);
1028
14
    if (J == I->second.Macros.end())
1029
0
      return false;
1030
14
    auto *MD = J->second.getLatest();
1031
14
    return MD && MD->isDefined();
1032
14
  }
1033
1034
35.1M
  MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
1035
35.1M
    if (!II->hasMacroDefinition())
1036
2.56M
      return {};
1037
32.5M
1038
32.5M
    MacroState &S = CurSubmoduleState->Macros[II];
1039
32.5M
    auto *MD = S.getLatest();
1040
32.5M
    while (MD && 
isa<VisibilityMacroDirective>(MD)32.5M
)
1041
125
      MD = MD->getPrevious();
1042
32.5M
    return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1043
32.5M
                           S.getActiveModuleMacros(*this, II),
1044
32.5M
                           S.isAmbiguous(*this, II));
1045
32.5M
  }
1046
1047
  MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
1048
24
                                          SourceLocation Loc) {
1049
24
    if (!II->hadMacroDefinition())
1050
9
      return {};
1051
15
1052
15
    MacroState &S = CurSubmoduleState->Macros[II];
1053
15
    MacroDirective::DefInfo DI;
1054
15
    if (auto *MD = S.getLatest())
1055
15
      DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1056
15
    // FIXME: Compute the set of active module macros at the specified location.
1057
15
    return MacroDefinition(DI.getDirective(),
1058
15
                           S.getActiveModuleMacros(*this, II),
1059
15
                           S.isAmbiguous(*this, II));
1060
15
  }
1061
1062
  /// Given an identifier, return its latest non-imported MacroDirective
1063
  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1064
179
  MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const {
1065
179
    if (!II->hasMacroDefinition())
1066
9
      return nullptr;
1067
170
1068
170
    auto *MD = getLocalMacroDirectiveHistory(II);
1069
170
    if (!MD || MD->getDefinition().isUndefined())
1070
0
      return nullptr;
1071
170
1072
170
    return MD;
1073
170
  }
1074
1075
11.5M
  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1076
11.5M
    return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1077
11.5M
  }
1078
1079
50.3M
  MacroInfo *getMacroInfo(const IdentifierInfo *II) {
1080
50.3M
    if (!II->hasMacroDefinition())
1081
45.3M
      return nullptr;
1082
5.04M
    if (auto MD = getMacroDefinition(II))
1083
5.03M
      return MD.getMacroInfo();
1084
244
    return nullptr;
1085
244
  }
1086
1087
  /// Given an identifier, return the latest non-imported macro
1088
  /// directive for that identifier.
1089
  ///
1090
  /// One can iterate over all previous macro directives from the most recent
1091
  /// one.
1092
  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1093
1094
  /// Add a directive to the macro directive history for this identifier.
1095
  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1096
  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
1097
26.2M
                                             SourceLocation Loc) {
1098
26.2M
    DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1099
26.2M
    appendMacroDirective(II, MD);
1100
26.2M
    return MD;
1101
26.2M
  }
1102
  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II,
1103
26.2M
                                             MacroInfo *MI) {
1104
26.2M
    return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1105
26.2M
  }
1106
1107
  /// Set a MacroDirective that was loaded from a PCH file.
1108
  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1109
                               MacroDirective *MD);
1110
1111
  /// Register an exported macro for a module and identifier.
1112
  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1113
                              ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1114
  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1115
1116
  /// Get the list of leaf (non-overridden) module macros for a name.
1117
1.39M
  ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
1118
1.39M
    if (II->isOutOfDate())
1119
10
      updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1120
1.39M
    auto I = LeafModuleMacros.find(II);
1121
1.39M
    if (I != LeafModuleMacros.end())
1122
17.6k
      return I->second;
1123
1.37M
    return None;
1124
1.37M
  }
1125
1126
  /// \{
1127
  /// Iterators for the macro history table. Currently defined macros have
1128
  /// IdentifierInfo::hasMacroDefinition() set and an empty
1129
  /// MacroInfo::getUndefLoc() at the head of the list.
1130
  using macro_iterator = MacroMap::const_iterator;
1131
1132
  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1133
  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1134
1135
  llvm::iterator_range<macro_iterator>
1136
7
  macros(bool IncludeExternalMacros = true) const {
1137
7
    macro_iterator begin = macro_begin(IncludeExternalMacros);
1138
7
    macro_iterator end = macro_end(IncludeExternalMacros);
1139
7
    return llvm::make_range(begin, end);
1140
7
  }
1141
1142
  /// \}
1143
1144
  /// Return the name of the macro defined before \p Loc that has
1145
  /// spelling \p Tokens.  If there are multiple macros with same spelling,
1146
  /// return the last one defined.
1147
  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1148
                                     ArrayRef<TokenValue> Tokens) const;
1149
1150
0
  const std::string &getPredefines() const { return Predefines; }
1151
1152
  /// Set the predefines for this Preprocessor.
1153
  ///
1154
  /// These predefines are automatically injected when parsing the main file.
1155
0
  void setPredefines(const char *P) { Predefines = P; }
1156
46.4k
  void setPredefines(StringRef P) { Predefines = P; }
1157
1158
  /// Return information about the specified preprocessor
1159
  /// identifier token.
1160
232M
  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1161
232M
    return &Identifiers.get(Name);
1162
232M
  }
1163
1164
  /// Add the specified pragma handler to this preprocessor.
1165
  ///
1166
  /// If \p Namespace is non-null, then it is a token required to exist on the
1167
  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1168
  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1169
858k
  void AddPragmaHandler(PragmaHandler *Handler) {
1170
858k
    AddPragmaHandler(StringRef(), Handler);
1171
858k
  }
1172
1173
  /// Remove the specific pragma handler from this preprocessor.
1174
  ///
1175
  /// If \p Namespace is non-null, then it should be the namespace that
1176
  /// \p Handler was added to. It is an error to remove a handler that
1177
  /// has not been registered.
1178
  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1179
538k
  void RemovePragmaHandler(PragmaHandler *Handler) {
1180
538k
    RemovePragmaHandler(StringRef(), Handler);
1181
538k
  }
1182
1183
  /// Install empty handlers for all pragmas (making them ignored).
1184
  void IgnorePragmas();
1185
1186
  /// Add the specified comment handler to the preprocessor.
1187
  void addCommentHandler(CommentHandler *Handler);
1188
1189
  /// Remove the specified comment handler.
1190
  ///
1191
  /// It is an error to remove a handler that has not been registered.
1192
  void removeCommentHandler(CommentHandler *Handler);
1193
1194
  /// Set the code completion handler to the given object.
1195
41.4k
  void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1196
41.4k
    CodeComplete = &Handler;
1197
41.4k
  }
1198
1199
  /// Retrieve the current code-completion handler.
1200
16
  CodeCompletionHandler *getCodeCompletionHandler() const {
1201
16
    return CodeComplete;
1202
16
  }
1203
1204
  /// Clear out the code completion handler.
1205
41.4k
  void clearCodeCompletionHandler() {
1206
41.4k
    CodeComplete = nullptr;
1207
41.4k
  }
1208
1209
  /// Hook used by the lexer to invoke the "included file" code
1210
  /// completion point.
1211
  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1212
1213
  /// Hook used by the lexer to invoke the "natural language" code
1214
  /// completion point.
1215
  void CodeCompleteNaturalLanguage();
1216
1217
  /// Set the code completion token for filtering purposes.
1218
48
  void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1219
48
    CodeCompletionII = Filter;
1220
48
  }
1221
1222
  /// Set the code completion token range for detecting replacement range later
1223
  /// on.
1224
  void setCodeCompletionTokenRange(const SourceLocation Start,
1225
48
                                   const SourceLocation End) {
1226
48
    CodeCompletionTokenRange = {Start, End};
1227
48
  }
1228
0
  SourceRange getCodeCompletionTokenRange() const {
1229
0
    return CodeCompletionTokenRange;
1230
0
  }
1231
1232
  /// Get the code completion token for filtering purposes.
1233
215
  StringRef getCodeCompletionFilter() {
1234
215
    if (CodeCompletionII)
1235
26
      return CodeCompletionII->getName();
1236
189
    return {};
1237
189
  }
1238
1239
  /// Retrieve the preprocessing record, or NULL if there is no
1240
  /// preprocessing record.
1241
193k
  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1242
1243
  /// Create a new preprocessing record, which will keep track of
1244
  /// all macro expansions, macro definitions, etc.
1245
  void createPreprocessingRecord();
1246
1247
  /// Returns true if the FileEntry is the PCH through header.
1248
  bool isPCHThroughHeader(const FileEntry *FE);
1249
1250
  /// True if creating a PCH with a through header.
1251
  bool creatingPCHWithThroughHeader();
1252
1253
  /// True if using a PCH with a through header.
1254
  bool usingPCHWithThroughHeader();
1255
1256
  /// True if creating a PCH with a #pragma hdrstop.
1257
  bool creatingPCHWithPragmaHdrStop();
1258
1259
  /// True if using a PCH with a #pragma hdrstop.
1260
  bool usingPCHWithPragmaHdrStop();
1261
1262
  /// Skip tokens until after the #include of the through header or
1263
  /// until after a #pragma hdrstop.
1264
  void SkipTokensWhileUsingPCH();
1265
1266
  /// Process directives while skipping until the through header or
1267
  /// #pragma hdrstop is found.
1268
  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1269
                                           SourceLocation HashLoc);
1270
1271
  /// Enter the specified FileID as the main source file,
1272
  /// which implicitly adds the builtin defines etc.
1273
  void EnterMainSourceFile();
1274
1275
  /// Inform the preprocessor callbacks that processing is complete.
1276
  void EndSourceFile();
1277
1278
  /// Add a source file to the top of the include stack and
1279
  /// start lexing tokens from it instead of the current buffer.
1280
  ///
1281
  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1282
  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1283
                       SourceLocation Loc);
1284
1285
  /// Add a Macro to the top of the include stack and start lexing
1286
  /// tokens from it instead of the current buffer.
1287
  ///
1288
  /// \param Args specifies the tokens input to a function-like macro.
1289
  /// \param ILEnd specifies the location of the ')' for a function-like macro
1290
  /// or the identifier for an object-like macro.
1291
  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1292
                  MacroArgs *Args);
1293
1294
private:
1295
  /// Add a "macro" context to the top of the include stack,
1296
  /// which will cause the lexer to start returning the specified tokens.
1297
  ///
1298
  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1299
  /// will not be subject to further macro expansion. Otherwise, these tokens
1300
  /// will be re-macro-expanded when/if expansion is enabled.
1301
  ///
1302
  /// If \p OwnsTokens is false, this method assumes that the specified stream
1303
  /// of tokens has a permanent owner somewhere, so they do not need to be
1304
  /// copied. If it is true, it assumes the array of tokens is allocated with
1305
  /// \c new[] and the Preprocessor will delete[] it.
1306
  ///
1307
  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1308
  /// set, see the flag documentation for details.
1309
  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1310
                        bool DisableMacroExpansion, bool OwnsTokens,
1311
                        bool IsReinject);
1312
1313
public:
1314
  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1315
124k
                        bool DisableMacroExpansion, bool IsReinject) {
1316
124k
    EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1317
124k
                     IsReinject);
1318
124k
  }
1319
1320
  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1321
1.29M
                        bool IsReinject) {
1322
1.29M
    EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1323
1.29M
                     IsReinject);
1324
1.29M
  }
1325
1326
  /// Pop the current lexer/macro exp off the top of the lexer stack.
1327
  ///
1328
  /// This should only be used in situations where the current state of the
1329
  /// top-of-stack lexer is known.
1330
  void RemoveTopOfLexerStack();
1331
1332
  /// From the point that this method is called, and until
1333
  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1334
  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1335
  /// make the Preprocessor re-lex the same tokens.
1336
  ///
1337
  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1338
  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1339
  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1340
  ///
1341
  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1342
  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1343
  /// tokens will continue indefinitely.
1344
  ///
1345
  void EnableBacktrackAtThisPos();
1346
1347
  /// Disable the last EnableBacktrackAtThisPos call.
1348
  void CommitBacktrackedTokens();
1349
1350
  /// Make Preprocessor re-lex the tokens that were lexed since
1351
  /// EnableBacktrackAtThisPos() was previously called.
1352
  void Backtrack();
1353
1354
  /// True if EnableBacktrackAtThisPos() was called and
1355
  /// caching of tokens is on.
1356
62.2M
  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1357
1358
  /// Lex the next token for this preprocessor.
1359
  void Lex(Token &Result);
1360
1361
  /// Lex a token, forming a header-name token if possible.
1362
  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1363
1364
  bool LexAfterModuleImport(Token &Result);
1365
  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1366
1367
  void makeModuleVisible(Module *M, SourceLocation Loc);
1368
1369
57
  SourceLocation getModuleImportLoc(Module *M) const {
1370
57
    return CurSubmoduleState->VisibleModules.getImportLoc(M);
1371
57
  }
1372
1373
  /// Lex a string literal, which may be the concatenation of multiple
1374
  /// string literals and may even come from macro expansion.
1375
  /// \returns true on success, false if a error diagnostic has been generated.
1376
  bool LexStringLiteral(Token &Result, std::string &String,
1377
130
                        const char *DiagnosticTag, bool AllowMacroExpansion) {
1378
130
    if (AllowMacroExpansion)
1379
76
      Lex(Result);
1380
54
    else
1381
54
      LexUnexpandedToken(Result);
1382
130
    return FinishLexStringLiteral(Result, String, DiagnosticTag,
1383
130
                                  AllowMacroExpansion);
1384
130
  }
1385
1386
  /// Complete the lexing of a string literal where the first token has
1387
  /// already been lexed (see LexStringLiteral).
1388
  bool FinishLexStringLiteral(Token &Result, std::string &String,
1389
                              const char *DiagnosticTag,
1390
                              bool AllowMacroExpansion);
1391
1392
  /// Lex a token.  If it's a comment, keep lexing until we get
1393
  /// something not a comment.
1394
  ///
1395
  /// This is useful in -E -C mode where comments would foul up preprocessor
1396
  /// directive handling.
1397
13.7M
  void LexNonComment(Token &Result) {
1398
13.7M
    do
1399
13.7M
      Lex(Result);
1400
13.7M
    while (Result.getKind() == tok::comment);
1401
13.7M
  }
1402
1403
  /// Just like Lex, but disables macro expansion of identifier tokens.
1404
433M
  void LexUnexpandedToken(Token &Result) {
1405
433M
    // Disable macro expansion.
1406
433M
    bool OldVal = DisableMacroExpansion;
1407
433M
    DisableMacroExpansion = true;
1408
433M
    // Lex the token.
1409
433M
    Lex(Result);
1410
433M
1411
433M
    // Reenable it.
1412
433M
    DisableMacroExpansion = OldVal;
1413
433M
  }
1414
1415
  /// Like LexNonComment, but this disables macro expansion of
1416
  /// identifier tokens.
1417
5.73M
  void LexUnexpandedNonComment(Token &Result) {
1418
5.73M
    do
1419
5.73M
      LexUnexpandedToken(Result);
1420
5.73M
    while (Result.getKind() == tok::comment);
1421
5.73M
  }
1422
1423
  /// Parses a simple integer literal to get its numeric value.  Floating
1424
  /// point literals and user defined literals are rejected.  Used primarily to
1425
  /// handle pragmas that accept integer arguments.
1426
  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1427
1428
  /// Disables macro expansion everywhere except for preprocessor directives.
1429
46
  void SetMacroExpansionOnlyInDirectives() {
1430
46
    DisableMacroExpansion = true;
1431
46
    MacroExpansionInDirectivesOverride = true;
1432
46
  }
1433
1434
  /// Peeks ahead N tokens and returns that token without consuming any
1435
  /// tokens.
1436
  ///
1437
  /// LookAhead(0) returns the next token that would be returned by Lex(),
1438
  /// LookAhead(1) returns the token after it, etc.  This returns normal
1439
  /// tokens after phase 5.  As such, it is equivalent to using
1440
  /// 'Lex', not 'LexUnexpandedToken'.
1441
142M
  const Token &LookAhead(unsigned N) {
1442
142M
    assert(LexLevel == 0 && "cannot use lookahead while lexing");
1443
142M
    if (CachedLexPos + N < CachedTokens.size())
1444
74.6M
      return CachedTokens[CachedLexPos+N];
1445
67.5M
    else
1446
67.5M
      return PeekAhead(N+1);
1447
142M
  }
1448
1449
  /// When backtracking is enabled and tokens are cached,
1450
  /// this allows to revert a specific number of tokens.
1451
  ///
1452
  /// Note that the number of tokens being reverted should be up to the last
1453
  /// backtrack position, not more.
1454
41.5k
  void RevertCachedTokens(unsigned N) {
1455
41.5k
    assert(isBacktrackEnabled() &&
1456
41.5k
           "Should only be called when tokens are cached for backtracking");
1457
41.5k
    assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1458
41.5k
         && "Should revert tokens up to the last backtrack position, not more");
1459
41.5k
    assert(signed(CachedLexPos) - signed(N) >= 0 &&
1460
41.5k
           "Corrupted backtrack positions ?");
1461
41.5k
    CachedLexPos -= N;
1462
41.5k
  }
1463
1464
  /// Enters a token in the token stream to be lexed next.
1465
  ///
1466
  /// If BackTrack() is called afterwards, the token will remain at the
1467
  /// insertion point.
1468
  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1469
  /// flag set. See the flag documentation for details.
1470
3.70M
  void EnterToken(const Token &Tok, bool IsReinject) {
1471
3.70M
    if (LexLevel) {
1472
245
      // It's not correct in general to enter caching lex mode while in the
1473
245
      // middle of a nested lexing action.
1474
245
      auto TokCopy = llvm::make_unique<Token[]>(1);
1475
245
      TokCopy[0] = Tok;
1476
245
      EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1477
3.70M
    } else {
1478
3.70M
      EnterCachingLexMode();
1479
3.70M
      assert(IsReinject && "new tokens in the middle of cached stream");
1480
3.70M
      CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1481
3.70M
    }
1482
3.70M
  }
1483
1484
  /// We notify the Preprocessor that if it is caching tokens (because
1485
  /// backtrack is enabled) it should replace the most recent cached tokens
1486
  /// with the given annotation token. This function has no effect if
1487
  /// backtracking is not enabled.
1488
  ///
1489
  /// Note that the use of this function is just for optimization, so that the
1490
  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1491
  /// invoked.
1492
20.4M
  void AnnotateCachedTokens(const Token &Tok) {
1493
20.4M
    assert(Tok.isAnnotation() && "Expected annotation token");
1494
20.4M
    if (CachedLexPos != 0 && 
isBacktrackEnabled()6.19M
)
1495
947k
      AnnotatePreviousCachedTokens(Tok);
1496
20.4M
  }
1497
1498
  /// Get the location of the last cached token, suitable for setting the end
1499
  /// location of an annotation token.
1500
14.6k
  SourceLocation getLastCachedTokenLocation() const {
1501
14.6k
    assert(CachedLexPos != 0);
1502
14.6k
    return CachedTokens[CachedLexPos-1].getLastLoc();
1503
14.6k
  }
1504
1505
  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1506
  /// CachedTokens.
1507
  bool IsPreviousCachedToken(const Token &Tok) const;
1508
1509
  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1510
  /// in \p NewToks.
1511
  ///
1512
  /// Useful when a token needs to be split in smaller ones and CachedTokens
1513
  /// most recent token must to be updated to reflect that.
1514
  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1515
1516
  /// Replace the last token with an annotation token.
1517
  ///
1518
  /// Like AnnotateCachedTokens(), this routine replaces an
1519
  /// already-parsed (and resolved) token with an annotation
1520
  /// token. However, this routine only replaces the last token with
1521
  /// the annotation token; it does not affect any other cached
1522
  /// tokens. This function has no effect if backtracking is not
1523
  /// enabled.
1524
0
  void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1525
0
    assert(Tok.isAnnotation() && "Expected annotation token");
1526
0
    if (CachedLexPos != 0 && isBacktrackEnabled())
1527
0
      CachedTokens[CachedLexPos-1] = Tok;
1528
0
  }
1529
1530
  /// Enter an annotation token into the token stream.
1531
  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1532
                            void *AnnotationVal);
1533
1534
  /// Update the current token to represent the provided
1535
  /// identifier, in order to cache an action performed by typo correction.
1536
16
  void TypoCorrectToken(const Token &Tok) {
1537
16
    assert(Tok.getIdentifierInfo() && "Expected identifier token");
1538
16
    if (CachedLexPos != 0 && 
isBacktrackEnabled()9
)
1539
9
      CachedTokens[CachedLexPos-1] = Tok;
1540
16
  }
1541
1542
  /// Recompute the current lexer kind based on the CurLexer/
1543
  /// CurTokenLexer pointers.
1544
  void recomputeCurLexerKind();
1545
1546
  /// Returns true if incremental processing is enabled
1547
4.04M
  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1548
1549
  /// Enables the incremental processing
1550
  void enableIncrementalProcessing(bool value = true) {
1551
    IncrementalProcessing = value;
1552
  }
1553
1554
  /// Specify the point at which code-completion will be performed.
1555
  ///
1556
  /// \param File the file in which code completion should occur. If
1557
  /// this file is included multiple times, code-completion will
1558
  /// perform completion the first time it is included. If NULL, this
1559
  /// function clears out the code-completion point.
1560
  ///
1561
  /// \param Line the line at which code completion should occur
1562
  /// (1-based).
1563
  ///
1564
  /// \param Column the column at which code completion should occur
1565
  /// (1-based).
1566
  ///
1567
  /// \returns true if an error occurred, false otherwise.
1568
  bool SetCodeCompletionPoint(const FileEntry *File,
1569
                              unsigned Line, unsigned Column);
1570
1571
  /// Determine if we are performing code completion.
1572
242M
  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1573
1574
  /// Returns the location of the code-completion point.
1575
  ///
1576
  /// Returns an invalid location if code-completion is not enabled or the file
1577
  /// containing the code-completion point has not been lexed yet.
1578
917k
  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1579
1580
  /// Returns the start location of the file of code-completion point.
1581
  ///
1582
  /// Returns an invalid location if code-completion is not enabled or the file
1583
  /// containing the code-completion point has not been lexed yet.
1584
8.90M
  SourceLocation getCodeCompletionFileLoc() const {
1585
8.90M
    return CodeCompletionFileLoc;
1586
8.90M
  }
1587
1588
  /// Returns true if code-completion is enabled and we have hit the
1589
  /// code-completion point.
1590
7.12M
  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1591
1592
  /// Note that we hit the code-completion point.
1593
1.22k
  void setCodeCompletionReached() {
1594
1.22k
    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1595
1.22k
    CodeCompletionReached = true;
1596
1.22k
    // Silence any diagnostics that occur after we hit the code-completion.
1597
1.22k
    getDiagnostics().setSuppressAllDiagnostics(true);
1598
1.22k
  }
1599
1600
  /// The location of the currently-active \#pragma clang
1601
  /// arc_cf_code_audited begin.
1602
  ///
1603
  /// Returns an invalid location if there is no such pragma active.
1604
6.03M
  SourceLocation getPragmaARCCFCodeAuditedLoc() const {
1605
6.03M
    return PragmaARCCFCodeAuditedLoc;
1606
6.03M
  }
1607
1608
  /// Set the location of the currently-active \#pragma clang
1609
  /// arc_cf_code_audited begin.  An invalid location ends the pragma.
1610
1.87k
  void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
1611
1.87k
    PragmaARCCFCodeAuditedLoc = Loc;
1612
1.87k
  }
1613
1614
  /// The location of the currently-active \#pragma clang
1615
  /// assume_nonnull begin.
1616
  ///
1617
  /// Returns an invalid location if there is no such pragma active.
1618
31.8M
  SourceLocation getPragmaAssumeNonNullLoc() const {
1619
31.8M
    return PragmaAssumeNonNullLoc;
1620
31.8M
  }
1621
1622
  /// Set the location of the currently-active \#pragma clang
1623
  /// assume_nonnull begin.  An invalid location ends the pragma.
1624
8.62k
  void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
1625
8.62k
    PragmaAssumeNonNullLoc = Loc;
1626
8.62k
  }
1627
1628
  /// Set the directory in which the main file should be considered
1629
  /// to have been found, if it is not a real file.
1630
1.48k
  void setMainFileDir(const DirectoryEntry *Dir) {
1631
1.48k
    MainFileDir = Dir;
1632
1.48k
  }
1633
1634
  /// Instruct the preprocessor to skip part of the main source file.
1635
  ///
1636
  /// \param Bytes The number of bytes in the preamble to skip.
1637
  ///
1638
  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1639
  /// start of a line.
1640
44.1k
  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1641
44.1k
    SkipMainFilePreamble.first = Bytes;
1642
44.1k
    SkipMainFilePreamble.second = StartOfLine;
1643
44.1k
  }
1644
1645
  /// Forwarding function for diagnostics.  This emits a diagnostic at
1646
  /// the specified Token's location, translating the token's start
1647
  /// position in the current buffer into a SourcePosition object for rendering.
1648
19.7k
  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1649
19.7k
    return Diags->Report(Loc, DiagID);
1650
19.7k
  }
1651
1652
143k
  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1653
143k
    return Diags->Report(Tok.getLocation(), DiagID);
1654
143k
  }
1655
1656
  /// Return the 'spelling' of the token at the given
1657
  /// location; does not go up to the spelling location or down to the
1658
  /// expansion location.
1659
  ///
1660
  /// \param buffer A buffer which will be used only if the token requires
1661
  ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
1662
  /// \param invalid If non-null, will be set \c true if an error occurs.
1663
  StringRef getSpelling(SourceLocation loc,
1664
                        SmallVectorImpl<char> &buffer,
1665
316
                        bool *invalid = nullptr) const {
1666
316
    return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1667
316
  }
1668
1669
  /// Return the 'spelling' of the Tok token.
1670
  ///
1671
  /// The spelling of a token is the characters used to represent the token in
1672
  /// the source file after trigraph expansion and escaped-newline folding.  In
1673
  /// particular, this wants to get the true, uncanonicalized, spelling of
1674
  /// things like digraphs, UCNs, etc.
1675
  ///
1676
  /// \param Invalid If non-null, will be set \c true if an error occurs.
1677
1.27M
  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1678
1.27M
    return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1679
1.27M
  }
1680
1681
  /// Get the spelling of a token into a preallocated buffer, instead
1682
  /// of as an std::string.
1683
  ///
1684
  /// The caller is required to allocate enough space for the token, which is
1685
  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1686
  /// actual result is returned.
1687
  ///
1688
  /// Note that this method may do two possible things: it may either fill in
1689
  /// the buffer specified with characters, or it may *change the input pointer*
1690
  /// to point to a constant buffer with the data already in it (avoiding a
1691
  /// copy).  The caller is not allowed to modify the returned buffer pointer
1692
  /// if an internal buffer is returned.
1693
  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1694
12.6M
                       bool *Invalid = nullptr) const {
1695
12.6M
    return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1696
12.6M
  }
1697
1698
  /// Get the spelling of a token into a SmallVector.
1699
  ///
1700
  /// Note that the returned StringRef may not point to the
1701
  /// supplied buffer if a copy can be avoided.
1702
  StringRef getSpelling(const Token &Tok,
1703
                        SmallVectorImpl<char> &Buffer,
1704
                        bool *Invalid = nullptr) const;
1705
1706
  /// Relex the token at the specified location.
1707
  /// \returns true if there was a failure, false on success.
1708
  bool getRawToken(SourceLocation Loc, Token &Result,
1709
117
                   bool IgnoreWhiteSpace = false) {
1710
117
    return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1711
117
  }
1712
1713
  /// Given a Token \p Tok that is a numeric constant with length 1,
1714
  /// return the character.
1715
  char
1716
  getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
1717
3.84M
                                              bool *Invalid = nullptr) const {
1718
3.84M
    assert(Tok.is(tok::numeric_constant) &&
1719
3.84M
           Tok.getLength() == 1 && "Called on unsupported token");
1720
3.84M
    assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1721
3.84M
1722
3.84M
    // If the token is carrying a literal data pointer, just use it.
1723
3.84M
    if (const char *D = Tok.getLiteralData())
1724
3.84M
      return *D;
1725
125
1726
125
    // Otherwise, fall back on getCharacterData, which is slower, but always
1727
125
    // works.
1728
125
    return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1729
125
  }
1730
1731
  /// Retrieve the name of the immediate macro expansion.
1732
  ///
1733
  /// This routine starts from a source location, and finds the name of the
1734
  /// macro responsible for its immediate expansion. It looks through any
1735
  /// intervening macro argument expansions to compute this. It returns a
1736
  /// StringRef that refers to the SourceManager-owned buffer of the source
1737
  /// where that macro name is spelled. Thus, the result shouldn't out-live
1738
  /// the SourceManager.
1739
32
  StringRef getImmediateMacroName(SourceLocation Loc) {
1740
32
    return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1741
32
  }
1742
1743
  /// Plop the specified string into a scratch buffer and set the
1744
  /// specified token's location and length to it.
1745
  ///
1746
  /// If specified, the source location provides a location of the expansion
1747
  /// point of the token.
1748
  void CreateString(StringRef Str, Token &Tok,
1749
                    SourceLocation ExpansionLocStart = SourceLocation(),
1750
                    SourceLocation ExpansionLocEnd = SourceLocation());
1751
1752
  /// Split the first Length characters out of the token starting at TokLoc
1753
  /// and return a location pointing to the split token. Re-lexing from the
1754
  /// split token will return the split token rather than the original.
1755
  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1756
1757
  /// Computes the source location just past the end of the
1758
  /// token at this source location.
1759
  ///
1760
  /// This routine can be used to produce a source location that
1761
  /// points just past the end of the token referenced by \p Loc, and
1762
  /// is generally used when a diagnostic needs to point just after a
1763
  /// token where it expected something different that it received. If
1764
  /// the returned source location would not be meaningful (e.g., if
1765
  /// it points into a macro), this routine returns an invalid
1766
  /// source location.
1767
  ///
1768
  /// \param Offset an offset from the end of the token, where the source
1769
  /// location should refer to. The default offset (0) produces a source
1770
  /// location pointing just past the end of the token; an offset of 1 produces
1771
  /// a source location pointing to the last character in the token, etc.
1772
5.83k
  SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
1773
5.83k
    return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1774
5.83k
  }
1775
1776
  /// Returns true if the given MacroID location points at the first
1777
  /// token of the macro expansion.
1778
  ///
1779
  /// \param MacroBegin If non-null and function returns true, it is set to
1780
  /// begin location of the macro.
1781
  bool isAtStartOfMacroExpansion(SourceLocation loc,
1782
29
                                 SourceLocation *MacroBegin = nullptr) const {
1783
29
    return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1784
29
                                            MacroBegin);
1785
29
  }
1786
1787
  /// Returns true if the given MacroID location points at the last
1788
  /// token of the macro expansion.
1789
  ///
1790
  /// \param MacroEnd If non-null and function returns true, it is set to
1791
  /// end location of the macro.
1792
  bool isAtEndOfMacroExpansion(SourceLocation loc,
1793
30
                               SourceLocation *MacroEnd = nullptr) const {
1794
30
    return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1795
30
  }
1796
1797
  /// Print the token to stderr, used for debugging.
1798
  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1799
  void DumpLocation(SourceLocation Loc) const;
1800
  void DumpMacro(const MacroInfo &MI) const;
1801
  void dumpMacroInfo(const IdentifierInfo *II);
1802
1803
  /// Given a location that specifies the start of a
1804
  /// token, return a new location that specifies a character within the token.
1805
  SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
1806
127k
                                         unsigned Char) const {
1807
127k
    return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1808
127k
  }
1809
1810
  /// Increment the counters for the number of token paste operations
1811
  /// performed.
1812
  ///
1813
  /// If fast was specified, this is a 'fast paste' case we handled.
1814
1.04M
  void IncrementPasteCounter(bool isFast) {
1815
1.04M
    if (isFast)
1816
587k
      ++NumFastTokenPaste;
1817
454k
    else
1818
454k
      ++NumTokenPaste;
1819
1.04M
  }
1820
1821
  void PrintStats();
1822
1823
  size_t getTotalMemory() const;
1824
1825
  /// When the macro expander pastes together a comment (/##/) in Microsoft
1826
  /// mode, this method handles updating the current state, returning the
1827
  /// token on the next source line.
1828
  void HandleMicrosoftCommentPaste(Token &Tok);
1829
1830
  //===--------------------------------------------------------------------===//
1831
  // Preprocessor callback methods.  These are invoked by a lexer as various
1832
  // directives and events are found.
1833
1834
  /// Given a tok::raw_identifier token, look up the
1835
  /// identifier information for the token and install it into the token,
1836
  /// updating the token kind accordingly.
1837
  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1838
1839
private:
1840
  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1841
1842
public:
1843
  /// Specifies the reason for poisoning an identifier.
1844
  ///
1845
  /// If that identifier is accessed while poisoned, then this reason will be
1846
  /// used instead of the default "poisoned" diagnostic.
1847
  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1848
1849
  /// Display reason for poisoned identifier.
1850
  void HandlePoisonedIdentifier(Token & Identifier);
1851
1852
0
  void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1853
0
    if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1854
0
      if(II->isPoisoned()) {
1855
0
        HandlePoisonedIdentifier(Identifier);
1856
0
      }
1857
0
    }
1858
0
  }
1859
1860
private:
1861
  /// Identifiers used for SEH handling in Borland. These are only
1862
  /// allowed in particular circumstances
1863
  // __except block
1864
  IdentifierInfo *Ident__exception_code,
1865
                 *Ident___exception_code,
1866
                 *Ident_GetExceptionCode;
1867
  // __except filter expression
1868
  IdentifierInfo *Ident__exception_info,
1869
                 *Ident___exception_info,
1870
                 *Ident_GetExceptionInfo;
1871
  // __finally
1872
  IdentifierInfo *Ident__abnormal_termination,
1873
                 *Ident___abnormal_termination,
1874
                 *Ident_AbnormalTermination;
1875
1876
  const char *getCurLexerEndPos();
1877
  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1878
1879
public:
1880
  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1881
1882
  /// Callback invoked when the lexer reads an identifier and has
1883
  /// filled in the tokens IdentifierInfo member.
1884
  ///
1885
  /// This callback potentially macro expands it or turns it into a named
1886
  /// token (like 'for').
1887
  ///
1888
  /// \returns true if we actually computed a token, false if we need to
1889
  /// lex again.
1890
  bool HandleIdentifier(Token &Identifier);
1891
1892
  /// Callback invoked when the lexer hits the end of the current file.
1893
  ///
1894
  /// This either returns the EOF token and returns true, or
1895
  /// pops a level off the include stack and returns false, at which point the
1896
  /// client should call lex again.
1897
  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1898
1899
  /// Callback invoked when the current TokenLexer hits the end of its
1900
  /// token stream.
1901
  bool HandleEndOfTokenLexer(Token &Result);
1902
1903
  /// Callback invoked when the lexer sees a # token at the start of a
1904
  /// line.
1905
  ///
1906
  /// This consumes the directive, modifies the lexer/preprocessor state, and
1907
  /// advances the lexer(s) so that the next token read is the correct one.
1908
  void HandleDirective(Token &Result);
1909
1910
  /// Ensure that the next token is a tok::eod token.
1911
  ///
1912
  /// If not, emit a diagnostic and consume up until the eod.
1913
  /// If \p EnableMacros is true, then we consider macros that expand to zero
1914
  /// tokens as being ok.
1915
  ///
1916
  /// \return The location of the end of the directive (the terminating
1917
  /// newline).
1918
  SourceLocation CheckEndOfDirective(const char *DirType,
1919
                                     bool EnableMacros = false);
1920
1921
  /// Read and discard all tokens remaining on the current line until
1922
  /// the tok::eod token is found. Returns the range of the skipped tokens.
1923
  SourceRange DiscardUntilEndOfDirective();
1924
1925
  /// Returns true if the preprocessor has seen a use of
1926
  /// __DATE__ or __TIME__ in the file so far.
1927
3.61k
  bool SawDateOrTime() const {
1928
3.61k
    return DATELoc != SourceLocation() || 
TIMELoc != SourceLocation()3.61k
;
1929
3.61k
  }
1930
3.62k
  unsigned getCounterValue() const { return CounterValue; }
1931
136
  void setCounterValue(unsigned V) { CounterValue = V; }
1932
1933
  /// Retrieves the module that we're currently building, if any.
1934
  Module *getCurrentModule();
1935
1936
  /// Allocate a new MacroInfo object with the provided SourceLocation.
1937
  MacroInfo *AllocateMacroInfo(SourceLocation L);
1938
1939
  /// Turn the specified lexer token into a fully checked and spelled
1940
  /// filename, e.g. as an operand of \#include.
1941
  ///
1942
  /// The caller is expected to provide a buffer that is large enough to hold
1943
  /// the spelling of the filename, but is also expected to handle the case
1944
  /// when this method decides to use a different buffer.
1945
  ///
1946
  /// \returns true if the input filename was in <>'s or false if it was
1947
  /// in ""'s.
1948
  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1949
1950
  /// Given a "foo" or \<foo> reference, look up the indicated file.
1951
  ///
1952
  /// Returns null on failure.  \p isAngled indicates whether the file
1953
  /// reference is for system \#include's or not (i.e. using <> instead of "").
1954
  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1955
                              bool isAngled, const DirectoryLookup *FromDir,
1956
                              const FileEntry *FromFile,
1957
                              const DirectoryLookup *&CurDir,
1958
                              SmallVectorImpl<char> *SearchPath,
1959
                              SmallVectorImpl<char> *RelativePath,
1960
                              ModuleMap::KnownHeader *SuggestedModule,
1961
                              bool *IsMapped, bool *IsFrameworkFound,
1962
                              bool SkipCache = false);
1963
1964
  /// Get the DirectoryLookup structure used to find the current
1965
  /// FileEntry, if CurLexer is non-null and if applicable.
1966
  ///
1967
  /// This allows us to implement \#include_next and find directory-specific
1968
  /// properties.
1969
12.1k
  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1970
1971
  /// Return true if we're in the top-level file, not in a \#include.
1972
  bool isInPrimaryFile() const;
1973
1974
  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1975
  /// followed by EOD.  Return true if the token is not a valid on-off-switch.
1976
  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1977
1978
  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1979
                      bool *ShadowFlag = nullptr);
1980
1981
  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1982
  Module *LeaveSubmodule(bool ForPragma);
1983
1984
private:
1985
  friend void TokenLexer::ExpandFunctionArguments();
1986
1987
86.7M
  void PushIncludeMacroStack() {
1988
86.7M
    assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1989
86.7M
    IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1990
86.7M
                                   std::move(CurLexer), CurPPLexer,
1991
86.7M
                                   std::move(CurTokenLexer), CurDirLookup);
1992
86.7M
    CurPPLexer = nullptr;
1993
86.7M
  }
1994
1995
86.7M
  void PopIncludeMacroStack() {
1996
86.7M
    CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1997
86.7M
    CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1998
86.7M
    CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1999
86.7M
    CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
2000
86.7M
    CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2001
86.7M
    CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2002
86.7M
    IncludeMacroStack.pop_back();
2003
86.7M
  }
2004
2005
  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2006
2007
  /// Determine whether we need to create module macros for #defines in the
2008
  /// current context.
2009
  bool needModuleMacros() const;
2010
2011
  /// Update the set of active module macros and ambiguity flag for a module
2012
  /// macro name.
2013
  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2014
2015
  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2016
                                               SourceLocation Loc);
2017
  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2018
  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2019
                                                             bool isPublic);
2020
2021
  /// Lex and validate a macro name, which occurs after a
2022
  /// \#define or \#undef.
2023
  ///
2024
  /// \param MacroNameTok Token that represents the name defined or undefined.
2025
  /// \param IsDefineUndef Kind if preprocessor directive.
2026
  /// \param ShadowFlag Points to flag that is set if macro name shadows
2027
  ///                   a keyword.
2028
  ///
2029
  /// This emits a diagnostic, sets the token kind to eod,
2030
  /// and discards the rest of the macro line if the macro name is invalid.
2031
  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2032
                     bool *ShadowFlag = nullptr);
2033
2034
  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2035
  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2036
  /// doing so performs certain validity checks including (but not limited to):
2037
  ///   - # (stringization) is followed by a macro parameter
2038
  /// \param MacroNameTok - Token that represents the macro name
2039
  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2040
  ///
2041
  ///  Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2042
  ///  returns a nullptr if an invalid sequence of tokens is encountered.
2043
  MacroInfo *ReadOptionalMacroParameterListAndBody(
2044
      const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2045
2046
  /// The ( starting an argument list of a macro definition has just been read.
2047
  /// Lex the rest of the parameters and the closing ), updating \p MI with
2048
  /// what we learn and saving in \p LastTok the last token read.
2049
  /// Return true if an error occurs parsing the arg list.
2050
  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2051
2052
  /// We just read a \#if or related directive and decided that the
2053
  /// subsequent tokens are in the \#if'd out portion of the
2054
  /// file.  Lex the rest of the file, until we see an \#endif.  If \p
2055
  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2056
  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2057
  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2058
  /// already seen one so a \#else directive is a duplicate.  When this returns,
2059
  /// the caller can lex the first valid token.
2060
  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2061
                                    SourceLocation IfTokenLoc,
2062
                                    bool FoundNonSkipPortion, bool FoundElse,
2063
                                    SourceLocation ElseLoc = SourceLocation());
2064
2065
  /// Information about the result for evaluating an expression for a
2066
  /// preprocessor directive.
2067
  struct DirectiveEvalResult {
2068
    /// Whether the expression was evaluated as true or not.
2069
    bool Conditional;
2070
2071
    /// True if the expression contained identifiers that were undefined.
2072
    bool IncludedUndefinedIds;
2073
2074
    /// The source range for the expression.
2075
    SourceRange ExprRange;
2076
  };
2077
2078
  /// Evaluate an integer constant expression that may occur after a
2079
  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2080
  ///
2081
  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2082
  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2083
2084
  /// Install the standard preprocessor pragmas:
2085
  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2086
  void RegisterBuiltinPragmas();
2087
2088
  /// Register builtin macros such as __LINE__ with the identifier table.
2089
  void RegisterBuiltinMacros();
2090
2091
  /// If an identifier token is read that is to be expanded as a macro, handle
2092
  /// it and return the next token as 'Tok'.  If we lexed a token, return true;
2093
  /// otherwise the caller should lex again.
2094
  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2095
2096
  /// Cache macro expanded tokens for TokenLexers.
2097
  //
2098
  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2099
  /// going to lex in the cache and when it finishes the tokens are removed
2100
  /// from the end of the cache.
2101
  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2102
                                  ArrayRef<Token> tokens);
2103
2104
  void removeCachedMacroExpandedTokensOfLastLexer();
2105
2106
  /// Determine whether the next preprocessor token to be
2107
  /// lexed is a '('.  If so, consume the token and return true, if not, this
2108
  /// method should have no observable side-effect on the lexed tokens.
2109
  bool isNextPPTokenLParen();
2110
2111
  /// After reading "MACRO(", this method is invoked to read all of the formal
2112
  /// arguments specified for the macro invocation.  Returns null on error.
2113
  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2114
                                       SourceLocation &MacroEnd);
2115
2116
  /// If an identifier token is read that is to be expanded
2117
  /// as a builtin macro, handle it and return the next token as 'Tok'.
2118
  void ExpandBuiltinMacro(Token &Tok);
2119
2120
  /// Read a \c _Pragma directive, slice it up, process it, then
2121
  /// return the first token after the directive.
2122
  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2123
  void Handle_Pragma(Token &Tok);
2124
2125
  /// Like Handle_Pragma except the pragma text is not enclosed within
2126
  /// a string literal.
2127
  void HandleMicrosoft__pragma(Token &Tok);
2128
2129
  /// Add a lexer to the top of the include stack and
2130
  /// start lexing tokens from it instead of the current buffer.
2131
  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2132
2133
  /// Set the FileID for the preprocessor predefines.
2134
44.0k
  void setPredefinesFileID(FileID FID) {
2135
44.0k
    assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2136
44.0k
    PredefinesFileID = FID;
2137
44.0k
  }
2138
2139
  /// Set the FileID for the PCH through header.
2140
  void setPCHThroughHeaderFileID(FileID FID);
2141
2142
  /// Returns true if we are lexing from a file and not a
2143
  /// pragma or a macro.
2144
1.15M
  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2145
1.15M
    return L ? 
!L->isPragmaLexer()1.15M
:
P != nullptr9
;
2146
1.15M
  }
2147
2148
35.9k
  static bool IsFileLexer(const IncludeStackInfo& I) {
2149
35.9k
    return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2150
35.9k
  }
2151
2152
1.11M
  bool IsFileLexer() const {
2153
1.11M
    return IsFileLexer(CurLexer.get(), CurPPLexer);
2154
1.11M
  }
2155
2156
  //===--------------------------------------------------------------------===//
2157
  // Caching stuff.
2158
  void CachingLex(Token &Result);
2159
2160
324M
  bool InCachingLexMode() const {
2161
324M
    // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2162
324M
    // that we are past EOF, not that we are in CachingLex mode.
2163
324M
    return !CurPPLexer && 
!CurTokenLexer232M
&&
!IncludeMacroStack.empty()207M
;
2164
324M
  }
2165
2166
  void EnterCachingLexMode();
2167
  void EnterCachingLexModeUnchecked();
2168
2169
119M
  void ExitCachingLexMode() {
2170
119M
    if (InCachingLexMode())
2171
71.2M
      RemoveTopOfLexerStack();
2172
119M
  }
2173
2174
  const Token &PeekAhead(unsigned N);
2175
  void AnnotatePreviousCachedTokens(const Token &Tok);
2176
2177
  //===--------------------------------------------------------------------===//
2178
  /// Handle*Directive - implement the various preprocessor directives.  These
2179
  /// should side-effect the current preprocessor object so that the next call
2180
  /// to Lex() will return the appropriate token next.
2181
  void HandleLineDirective();
2182
  void HandleDigitDirective(Token &Tok);
2183
  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2184
  void HandleIdentSCCSDirective(Token &Tok);
2185
  void HandleMacroPublicDirective(Token &Tok);
2186
  void HandleMacroPrivateDirective();
2187
2188
  /// An additional notification that can be produced by a header inclusion or
2189
  /// import to tell the parser what happened.
2190
  struct ImportAction {
2191
    enum ActionKind {
2192
      None,
2193
      ModuleBegin,
2194
      ModuleImport,
2195
      SkippedModuleImport,
2196
    } Kind;
2197
    Module *ModuleForHeader = nullptr;
2198
2199
    ImportAction(ActionKind AK, Module *Mod = nullptr)
2200
999k
        : Kind(AK), ModuleForHeader(Mod) {
2201
999k
      assert((AK == None || Mod) && "no module for module action");
2202
999k
    }
2203
  };
2204
2205
  // File inclusion.
2206
  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2207
                              const DirectoryLookup *LookupFrom = nullptr,
2208
                              const FileEntry *LookupFromFile = nullptr);
2209
  ImportAction
2210
  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2211
                              Token &FilenameTok, SourceLocation EndLoc,
2212
                              const DirectoryLookup *LookupFrom = nullptr,
2213
                              const FileEntry *LookupFromFile = nullptr);
2214
  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2215
  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2216
  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2217
  void HandleMicrosoftImportDirective(Token &Tok);
2218
2219
public:
2220
  /// Check that the given module is available, producing a diagnostic if not.
2221
  /// \return \c true if the check failed (because the module is not available).
2222
  ///         \c false if the module appears to be usable.
2223
  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2224
                                     const TargetInfo &TargetInfo,
2225
                                     DiagnosticsEngine &Diags, Module *M);
2226
2227
  // Module inclusion testing.
2228
  /// Find the module that owns the source or header file that
2229
  /// \p Loc points to. If the location is in a file that was included
2230
  /// into a module, or is outside any module, returns nullptr.
2231
  Module *getModuleForLocation(SourceLocation Loc);
2232
2233
  /// We want to produce a diagnostic at location IncLoc concerning a
2234
  /// missing module import.
2235
  ///
2236
  /// \param IncLoc The location at which the missing import was detected.
2237
  /// \param M The desired module.
2238
  /// \param MLoc A location within the desired module at which some desired
2239
  ///        effect occurred (eg, where a desired entity was declared).
2240
  ///
2241
  /// \return A file that can be #included to import a module containing MLoc.
2242
  ///         Null if no such file could be determined or if a #include is not
2243
  ///         appropriate.
2244
  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2245
                                                          Module *M,
2246
                                                          SourceLocation MLoc);
2247
2248
605k
  bool isRecordingPreamble() const {
2249
605k
    return PreambleConditionalStack.isRecording();
2250
605k
  }
2251
2252
89
  bool hasRecordedPreamble() const {
2253
89
    return PreambleConditionalStack.hasRecordedPreamble();
2254
89
  }
2255
2256
7
  ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2257
7
      return PreambleConditionalStack.getStack();
2258
7
  }
2259
2260
89
  void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2261
89
    PreambleConditionalStack.setStack(s);
2262
89
  }
2263
2264
  void setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s,
2265
28
                                             llvm::Optional<PreambleSkipInfo> SkipInfo) {
2266
28
    PreambleConditionalStack.startReplaying();
2267
28
    PreambleConditionalStack.setStack(s);
2268
28
    PreambleConditionalStack.SkipInfo = SkipInfo;
2269
28
  }
2270
2271
7
  llvm::Optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2272
7
    return PreambleConditionalStack.SkipInfo;
2273
7
  }
2274
2275
private:
2276
  /// After processing predefined file, initialize the conditional stack from
2277
  /// the preamble.
2278
  void replayPreambleConditionalStack();
2279
2280
  // Macro handling.
2281
  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2282
  void HandleUndefDirective();
2283
2284
  // Conditional Inclusion.
2285
  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2286
                            bool isIfndef, bool ReadAnyTokensBeforeDirective);
2287
  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2288
                         bool ReadAnyTokensBeforeDirective);
2289
  void HandleEndifDirective(Token &EndifToken);
2290
  void HandleElseDirective(Token &Result, const Token &HashToken);
2291
  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2292
2293
  // Pragmas.
2294
  void HandlePragmaDirective(PragmaIntroducer Introducer);
2295
2296
public:
2297
  void HandlePragmaOnce(Token &OnceTok);
2298
  void HandlePragmaMark();
2299
  void HandlePragmaPoison();
2300
  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2301
  void HandlePragmaDependency(Token &DependencyTok);
2302
  void HandlePragmaPushMacro(Token &Tok);
2303
  void HandlePragmaPopMacro(Token &Tok);
2304
  void HandlePragmaIncludeAlias(Token &Tok);
2305
  void HandlePragmaModuleBuild(Token &Tok);
2306
  void HandlePragmaHdrstop(Token &Tok);
2307
  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2308
2309
  // Return true and store the first token only if any CommentHandler
2310
  // has inserted some tokens and getCommentRetentionState() is false.
2311
  bool HandleComment(Token &result, SourceRange Comment);
2312
2313
  /// A macro is used, update information about macros that need unused
2314
  /// warnings.
2315
  void markMacroAsUsed(MacroInfo *MI);
2316
};
2317
2318
/// Abstract base class that describes a handler that will receive
2319
/// source ranges for each of the comments encountered in the source file.
2320
class CommentHandler {
2321
public:
2322
  virtual ~CommentHandler();
2323
2324
  // The handler shall return true if it has pushed any tokens
2325
  // to be read using e.g. EnterToken or EnterTokenStream.
2326
  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2327
};
2328
2329
/// Registry of pragma handlers added by plugins
2330
using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2331
2332
} // namespace clang
2333
2334
#endif // LLVM_CLANG_LEX_PREPROCESSOR_H