Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/include/clang/Lex/Preprocessor.h
Line
Count
Source (jump to first uncovered line)
1
//===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// Defines the clang::Preprocessor interface.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15
#define LLVM_CLANG_LEX_PREPROCESSOR_H
16
17
#include "clang/Basic/Diagnostic.h"
18
#include "clang/Basic/IdentifierTable.h"
19
#include "clang/Basic/LLVM.h"
20
#include "clang/Basic/LangOptions.h"
21
#include "clang/Basic/Module.h"
22
#include "clang/Basic/SourceLocation.h"
23
#include "clang/Basic/SourceManager.h"
24
#include "clang/Basic/TokenKinds.h"
25
#include "clang/Lex/Lexer.h"
26
#include "clang/Lex/MacroInfo.h"
27
#include "clang/Lex/ModuleLoader.h"
28
#include "clang/Lex/ModuleMap.h"
29
#include "clang/Lex/PPCallbacks.h"
30
#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
31
#include "clang/Lex/Token.h"
32
#include "clang/Lex/TokenLexer.h"
33
#include "llvm/ADT/ArrayRef.h"
34
#include "llvm/ADT/DenseMap.h"
35
#include "llvm/ADT/FoldingSet.h"
36
#include "llvm/ADT/FunctionExtras.h"
37
#include "llvm/ADT/None.h"
38
#include "llvm/ADT/Optional.h"
39
#include "llvm/ADT/PointerUnion.h"
40
#include "llvm/ADT/STLExtras.h"
41
#include "llvm/ADT/SmallPtrSet.h"
42
#include "llvm/ADT/SmallVector.h"
43
#include "llvm/ADT/StringRef.h"
44
#include "llvm/ADT/TinyPtrVector.h"
45
#include "llvm/ADT/iterator_range.h"
46
#include "llvm/Support/Allocator.h"
47
#include "llvm/Support/Casting.h"
48
#include "llvm/Support/Registry.h"
49
#include <cassert>
50
#include <cstddef>
51
#include <cstdint>
52
#include <map>
53
#include <memory>
54
#include <string>
55
#include <utility>
56
#include <vector>
57
58
namespace llvm {
59
60
template<unsigned InternalLen> class SmallString;
61
62
} // namespace llvm
63
64
namespace clang {
65
66
class CodeCompletionHandler;
67
class CommentHandler;
68
class DirectoryEntry;
69
class DirectoryLookup;
70
class ExternalPreprocessorSource;
71
class FileEntry;
72
class FileManager;
73
class HeaderSearch;
74
class MacroArgs;
75
class PragmaHandler;
76
class PragmaNamespace;
77
class PreprocessingRecord;
78
class PreprocessorLexer;
79
class PreprocessorOptions;
80
class ScratchBuffer;
81
class TargetInfo;
82
83
namespace Builtin {
84
class Context;
85
}
86
87
/// Stores token information for comparing actual tokens with
88
/// predefined values.  Only handles simple tokens and identifiers.
89
class TokenValue {
90
  tok::TokenKind Kind;
91
  IdentifierInfo *II;
92
93
public:
94
1.33k
  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
95
1.33k
    assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
96
1.33k
    assert(Kind != tok::identifier &&
97
1.33k
           "Identifiers should be created by TokenValue(IdentifierInfo *)");
98
1.33k
    assert(!tok::isLiteral(Kind) && "Literals are not supported.");
99
1.33k
    assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
100
1.33k
  }
101
102
376
  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
103
104
12.7k
  bool operator==(const Token &Tok) const {
105
12.7k
    return Tok.getKind() == Kind &&
106
12.7k
        
(924
!II924
||
II == Tok.getIdentifierInfo()221
);
107
12.7k
  }
108
};
109
110
/// Context in which macro name is used.
111
enum MacroUse {
112
  // other than #define or #undef
113
  MU_Other  = 0,
114
115
  // macro name specified in #define
116
  MU_Define = 1,
117
118
  // macro name specified in #undef
119
  MU_Undef  = 2
120
};
121
122
/// Engages in a tight little dance with the lexer to efficiently
123
/// preprocess tokens.
124
///
125
/// Lexers know only about tokens within a single source file, and don't
126
/// know anything about preprocessor-level issues like the \#include stack,
127
/// token expansion, etc.
128
class Preprocessor {
129
  friend class VAOptDefinitionContext;
130
  friend class VariadicMacroScopeGuard;
131
132
  llvm::unique_function<void(const clang::Token &)> OnToken;
133
  std::shared_ptr<PreprocessorOptions> PPOpts;
134
  DiagnosticsEngine        *Diags;
135
  LangOptions       &LangOpts;
136
  const TargetInfo *Target = nullptr;
137
  const TargetInfo *AuxTarget = nullptr;
138
  FileManager       &FileMgr;
139
  SourceManager     &SourceMgr;
140
  std::unique_ptr<ScratchBuffer> ScratchBuf;
141
  HeaderSearch      &HeaderInfo;
142
  ModuleLoader      &TheModuleLoader;
143
144
  /// External source of macros.
145
  ExternalPreprocessorSource *ExternalSource;
146
147
  /// A BumpPtrAllocator object used to quickly allocate and release
148
  /// objects internal to the Preprocessor.
149
  llvm::BumpPtrAllocator BP;
150
151
  /// Identifiers for builtin macros and other builtins.
152
  IdentifierInfo *Ident__LINE__, *Ident__FILE__;   // __LINE__, __FILE__
153
  IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
154
  IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
155
  IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
156
  IdentifierInfo *Ident__FILE_NAME__;              // __FILE_NAME__
157
  IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
158
  IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
159
  IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
160
  IdentifierInfo *Ident__identifier;               // __identifier
161
  IdentifierInfo *Ident__VA_ARGS__;                // __VA_ARGS__
162
  IdentifierInfo *Ident__VA_OPT__;                 // __VA_OPT__
163
  IdentifierInfo *Ident__has_feature;              // __has_feature
164
  IdentifierInfo *Ident__has_extension;            // __has_extension
165
  IdentifierInfo *Ident__has_builtin;              // __has_builtin
166
  IdentifierInfo *Ident__has_attribute;            // __has_attribute
167
  IdentifierInfo *Ident__has_include;              // __has_include
168
  IdentifierInfo *Ident__has_include_next;         // __has_include_next
169
  IdentifierInfo *Ident__has_warning;              // __has_warning
170
  IdentifierInfo *Ident__is_identifier;            // __is_identifier
171
  IdentifierInfo *Ident__building_module;          // __building_module
172
  IdentifierInfo *Ident__MODULE__;                 // __MODULE__
173
  IdentifierInfo *Ident__has_cpp_attribute;        // __has_cpp_attribute
174
  IdentifierInfo *Ident__has_c_attribute;          // __has_c_attribute
175
  IdentifierInfo *Ident__has_declspec;             // __has_declspec_attribute
176
  IdentifierInfo *Ident__is_target_arch;           // __is_target_arch
177
  IdentifierInfo *Ident__is_target_vendor;         // __is_target_vendor
178
  IdentifierInfo *Ident__is_target_os;             // __is_target_os
179
  IdentifierInfo *Ident__is_target_environment;    // __is_target_environment
180
181
  // Weak, only valid (and set) while InMacroArgs is true.
182
  Token* ArgMacro;
183
184
  SourceLocation DATELoc, TIMELoc;
185
186
  // Next __COUNTER__ value, starts at 0.
187
  unsigned CounterValue = 0;
188
189
  enum {
190
    /// Maximum depth of \#includes.
191
    MaxAllowedIncludeStackDepth = 200
192
  };
193
194
  // State that is set before the preprocessor begins.
195
  bool KeepComments : 1;
196
  bool KeepMacroComments : 1;
197
  bool SuppressIncludeNotFoundError : 1;
198
199
  // State that changes while the preprocessor runs:
200
  bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
201
202
  /// Whether the preprocessor owns the header search object.
203
  bool OwnsHeaderSearch : 1;
204
205
  /// True if macro expansion is disabled.
206
  bool DisableMacroExpansion : 1;
207
208
  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
209
  /// when parsing preprocessor directives.
210
  bool MacroExpansionInDirectivesOverride : 1;
211
212
  class ResetMacroExpansionHelper;
213
214
  /// Whether we have already loaded macros from the external source.
215
  mutable bool ReadMacrosFromExternalSource : 1;
216
217
  /// True if pragmas are enabled.
218
  bool PragmasEnabled : 1;
219
220
  /// True if the current build action is a preprocessing action.
221
  bool PreprocessedOutput : 1;
222
223
  /// True if we are currently preprocessing a #if or #elif directive
224
  bool ParsingIfOrElifDirective;
225
226
  /// True if we are pre-expanding macro arguments.
227
  bool InMacroArgPreExpansion;
228
229
  /// Mapping/lookup information for all identifiers in
230
  /// the program, including program keywords.
231
  mutable IdentifierTable Identifiers;
232
233
  /// This table contains all the selectors in the program.
234
  ///
235
  /// Unlike IdentifierTable above, this table *isn't* populated by the
236
  /// preprocessor. It is declared/expanded here because its role/lifetime is
237
  /// conceptually similar to the IdentifierTable. In addition, the current
238
  /// control flow (in clang::ParseAST()), make it convenient to put here.
239
  ///
240
  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
241
  /// the lifetime of the preprocessor.
242
  SelectorTable Selectors;
243
244
  /// Information about builtins.
245
  std::unique_ptr<Builtin::Context> BuiltinInfo;
246
247
  /// Tracks all of the pragmas that the client registered
248
  /// with this preprocessor.
249
  std::unique_ptr<PragmaNamespace> PragmaHandlers;
250
251
  /// Pragma handlers of the original source is stored here during the
252
  /// parsing of a model file.
253
  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
254
255
  /// Tracks all of the comment handlers that the client registered
256
  /// with this preprocessor.
257
  std::vector<CommentHandler *> CommentHandlers;
258
259
  /// True if we want to ignore EOF token and continue later on (thus
260
  /// avoid tearing the Lexer and etc. down).
261
  bool IncrementalProcessing = false;
262
263
  /// The kind of translation unit we are processing.
264
  TranslationUnitKind TUKind;
265
266
  /// The code-completion handler.
267
  CodeCompletionHandler *CodeComplete = nullptr;
268
269
  /// The file that we're performing code-completion for, if any.
270
  const FileEntry *CodeCompletionFile = nullptr;
271
272
  /// The offset in file for the code-completion point.
273
  unsigned CodeCompletionOffset = 0;
274
275
  /// The location for the code-completion point. This gets instantiated
276
  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
277
  SourceLocation CodeCompletionLoc;
278
279
  /// The start location for the file of the code-completion point.
280
  ///
281
  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
282
  /// for preprocessing.
283
  SourceLocation CodeCompletionFileLoc;
284
285
  /// The source location of the \c import contextual keyword we just
286
  /// lexed, if any.
287
  SourceLocation ModuleImportLoc;
288
289
  /// The module import path that we're currently processing.
290
  SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2> ModuleImportPath;
291
292
  /// Whether the last token we lexed was an '@'.
293
  bool LastTokenWasAt = false;
294
295
  /// A position within a C++20 import-seq.
296
  class ImportSeq {
297
  public:
298
    enum State : int {
299
      // Positive values represent a number of unclosed brackets.
300
      AtTopLevel = 0,
301
      AfterTopLevelTokenSeq = -1,
302
      AfterExport = -2,
303
      AfterImportSeq = -3,
304
    };
305
306
59.4k
    ImportSeq(State S) : S(S) {}
307
308
    /// Saw any kind of open bracket.
309
29.5k
    void handleOpenBracket() {
310
29.5k
      S = static_cast<State>(std::max<int>(S, 0) + 1);
311
29.5k
    }
312
    /// Saw any kind of close bracket other than '}'.
313
29.5k
    void handleCloseBracket() {
314
29.5k
      S = static_cast<State>(std::max<int>(S, 1) - 1);
315
29.5k
    }
316
    /// Saw a close brace.
317
8.05k
    void handleCloseBrace() {
318
8.05k
      handleCloseBracket();
319
8.05k
      if (S == AtTopLevel && 
!AfterHeaderName2.30k
)
320
2.30k
        S = AfterTopLevelTokenSeq;
321
8.05k
    }
322
    /// Saw a semicolon.
323
18.2k
    void handleSemi() {
324
18.2k
      if (atTopLevel()) {
325
2.45k
        S = AfterTopLevelTokenSeq;
326
2.45k
        AfterHeaderName = false;
327
2.45k
      }
328
18.2k
    }
329
330
    /// Saw an 'export' identifier.
331
86
    void handleExport() {
332
86
      if (S == AfterTopLevelTokenSeq)
333
75
        S = AfterExport;
334
11
      else if (S <= 0)
335
2
        S = AtTopLevel;
336
86
    }
337
    /// Saw an 'import' identifier.
338
50
    void handleImport() {
339
50
      if (S == AfterTopLevelTokenSeq || 
S == AfterExport20
)
340
32
        S = AfterImportSeq;
341
18
      else if (S <= 0)
342
13
        S = AtTopLevel;
343
50
    }
344
345
    /// Saw a 'header-name' token; do not recognize any more 'import' tokens
346
    /// until we reach a top-level semicolon.
347
24
    void handleHeaderName() {
348
24
      if (S == AfterImportSeq)
349
24
        AfterHeaderName = true;
350
24
      handleMisc();
351
24
    }
352
353
    /// Saw any other token.
354
139k
    void handleMisc() {
355
139k
      if (S <= 0)
356
16.5k
        S = AtTopLevel;
357
139k
    }
358
359
18.2k
    bool atTopLevel() { return S <= 0; }
360
50
    bool afterImportSeq() { return S == AfterImportSeq; }
361
362
  private:
363
    State S;
364
    /// Whether we're in the pp-import-suffix following the header-name in a
365
    /// pp-import. If so, a close-brace is not sufficient to end the
366
    /// top-level-token-seq of an import-seq.
367
    bool AfterHeaderName = false;
368
  };
369
370
  /// Our current position within a C++20 import-seq.
371
  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
372
373
  /// Whether the module import expects an identifier next. Otherwise,
374
  /// it expects a '.' or ';'.
375
  bool ModuleImportExpectsIdentifier = false;
376
377
  /// The identifier and source location of the currently-active
378
  /// \#pragma clang arc_cf_code_audited begin.
379
  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
380
381
  /// The source location of the currently-active
382
  /// \#pragma clang assume_nonnull begin.
383
  SourceLocation PragmaAssumeNonNullLoc;
384
385
  /// True if we hit the code-completion point.
386
  bool CodeCompletionReached = false;
387
388
  /// The code completion token containing the information
389
  /// on the stem that is to be code completed.
390
  IdentifierInfo *CodeCompletionII = nullptr;
391
392
  /// Range for the code completion token.
393
  SourceRange CodeCompletionTokenRange;
394
395
  /// The directory that the main file should be considered to occupy,
396
  /// if it does not correspond to a real file (as happens when building a
397
  /// module).
398
  const DirectoryEntry *MainFileDir = nullptr;
399
400
  /// The number of bytes that we will initially skip when entering the
401
  /// main file, along with a flag that indicates whether skipping this number
402
  /// of bytes will place the lexer at the start of a line.
403
  ///
404
  /// This is used when loading a precompiled preamble.
405
  std::pair<int, bool> SkipMainFilePreamble;
406
407
  /// Whether we hit an error due to reaching max allowed include depth. Allows
408
  /// to avoid hitting the same error over and over again.
409
  bool HasReachedMaxIncludeDepth = false;
410
411
  /// The number of currently-active calls to Lex.
412
  ///
413
  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
414
  /// require asking for multiple additional tokens. This counter makes it
415
  /// possible for Lex to detect whether it's producing a token for the end
416
  /// of phase 4 of translation or for some other situation.
417
  unsigned LexLevel = 0;
418
419
  /// The number of (LexLevel 0) preprocessor tokens.
420
  unsigned TokenCount = 0;
421
422
  /// The maximum number of (LexLevel 0) tokens before issuing a -Wmax-tokens
423
  /// warning, or zero for unlimited.
424
  unsigned MaxTokens = 0;
425
  SourceLocation MaxTokensOverrideLoc;
426
427
public:
428
  struct PreambleSkipInfo {
429
    SourceLocation HashTokenLoc;
430
    SourceLocation IfTokenLoc;
431
    bool FoundNonSkipPortion;
432
    bool FoundElse;
433
    SourceLocation ElseLoc;
434
435
    PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc,
436
                     bool FoundNonSkipPortion, bool FoundElse,
437
                     SourceLocation ElseLoc)
438
        : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
439
          FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
440
23
          ElseLoc(ElseLoc) {}
441
  };
442
443
private:
444
  friend class ASTReader;
445
  friend class MacroArgs;
446
447
  class PreambleConditionalStackStore {
448
    enum State {
449
      Off = 0,
450
      Recording = 1,
451
      Replaying = 2,
452
    };
453
454
  public:
455
59.4k
    PreambleConditionalStackStore() = default;
456
457
89
    void startRecording() { ConditionalStackState = Recording; }
458
28
    void startReplaying() { ConditionalStackState = Replaying; }
459
3.72M
    bool isRecording() const { return ConditionalStackState == Recording; }
460
59.2k
    bool isReplaying() const { return ConditionalStackState == Replaying; }
461
462
35
    ArrayRef<PPConditionalInfo> getStack() const {
463
35
      return ConditionalStack;
464
35
    }
465
466
28
    void doneReplaying() {
467
28
      ConditionalStack.clear();
468
28
      ConditionalStackState = Off;
469
28
    }
470
471
117
    void setStack(ArrayRef<PPConditionalInfo> s) {
472
117
      if (!isRecording() && 
!isReplaying()28
)
473
0
        return;
474
117
      ConditionalStack.clear();
475
117
      ConditionalStack.append(s.begin(), s.end());
476
117
    }
477
478
89
    bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
479
480
4.32M
    bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
481
482
18
    void clearSkipInfo() { SkipInfo.reset(); }
483
484
    llvm::Optional<PreambleSkipInfo> SkipInfo;
485
486
  private:
487
    SmallVector<PPConditionalInfo, 4> ConditionalStack;
488
    State ConditionalStackState = Off;
489
  } PreambleConditionalStack;
490
491
  /// The current top of the stack that we're lexing from if
492
  /// not expanding a macro and we are lexing directly from source code.
493
  ///
494
  /// Only one of CurLexer, or CurTokenLexer will be non-null.
495
  std::unique_ptr<Lexer> CurLexer;
496
497
  /// The current top of the stack what we're lexing from
498
  /// if not expanding a macro.
499
  ///
500
  /// This is an alias for CurLexer.
501
  PreprocessorLexer *CurPPLexer = nullptr;
502
503
  /// Used to find the current FileEntry, if CurLexer is non-null
504
  /// and if applicable.
505
  ///
506
  /// This allows us to implement \#include_next and find directory-specific
507
  /// properties.
508
  const DirectoryLookup *CurDirLookup = nullptr;
509
510
  /// The current macro we are expanding, if we are expanding a macro.
511
  ///
512
  /// One of CurLexer and CurTokenLexer must be null.
513
  std::unique_ptr<TokenLexer> CurTokenLexer;
514
515
  /// The kind of lexer we're currently working with.
516
  enum CurLexerKind {
517
    CLK_Lexer,
518
    CLK_TokenLexer,
519
    CLK_CachingLexer,
520
    CLK_LexAfterModuleImport
521
  } CurLexerKind = CLK_Lexer;
522
523
  /// If the current lexer is for a submodule that is being built, this
524
  /// is that submodule.
525
  Module *CurLexerSubmodule = nullptr;
526
527
  /// Keeps track of the stack of files currently
528
  /// \#included, and macros currently being expanded from, not counting
529
  /// CurLexer/CurTokenLexer.
530
  struct IncludeStackInfo {
531
    enum CurLexerKind           CurLexerKind;
532
    Module                     *TheSubmodule;
533
    std::unique_ptr<Lexer>      TheLexer;
534
    PreprocessorLexer          *ThePPLexer;
535
    std::unique_ptr<TokenLexer> TheTokenLexer;
536
    const DirectoryLookup      *TheDirLookup;
537
538
    // The following constructors are completely useless copies of the default
539
    // versions, only needed to pacify MSVC.
540
    IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
541
                     std::unique_ptr<Lexer> &&TheLexer,
542
                     PreprocessorLexer *ThePPLexer,
543
                     std::unique_ptr<TokenLexer> &&TheTokenLexer,
544
                     const DirectoryLookup *TheDirLookup)
545
        : CurLexerKind(std::move(CurLexerKind)),
546
          TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
547
          ThePPLexer(std::move(ThePPLexer)),
548
          TheTokenLexer(std::move(TheTokenLexer)),
549
123M
          TheDirLookup(std::move(TheDirLookup)) {}
550
  };
551
  std::vector<IncludeStackInfo> IncludeMacroStack;
552
553
  /// Actions invoked when some preprocessor activity is
554
  /// encountered (e.g. a file is \#included, etc).
555
  std::unique_ptr<PPCallbacks> Callbacks;
556
557
  struct MacroExpandsInfo {
558
    Token Tok;
559
    MacroDefinition MD;
560
    SourceRange Range;
561
562
    MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
563
6
        : Tok(Tok), MD(MD), Range(Range) {}
564
  };
565
  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
566
567
  /// Information about a name that has been used to define a module macro.
568
  struct ModuleMacroInfo {
569
    /// The most recent macro directive for this identifier.
570
    MacroDirective *MD;
571
572
    /// The active module macros for this identifier.
573
    llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
574
575
    /// The generation number at which we last updated ActiveModuleMacros.
576
    /// \see Preprocessor::VisibleModules.
577
    unsigned ActiveModuleMacrosGeneration = 0;
578
579
    /// Whether this macro name is ambiguous.
580
    bool IsAmbiguous = false;
581
582
    /// The module macros that are overridden by this macro.
583
    llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
584
585
109k
    ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
586
  };
587
588
  /// The state of a macro for an identifier.
589
  class MacroState {
590
    mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
591
592
    ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
593
338M
                                   const IdentifierInfo *II) const {
594
338M
      if (II->isOutOfDate())
595
1.00k
        PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
596
338M
      // FIXME: Find a spare bit on IdentifierInfo and store a
597
338M
      //        HasModuleMacros flag.
598
338M
      if (!II->hasMacroDefinition() ||
599
338M
          
(303M
!PP.getLangOpts().Modules303M
&&
600
303M
           
!PP.getLangOpts().ModulesLocalVisibility294M
) ||
601
338M
          
!PP.CurSubmoduleState->VisibleModules.getGeneration()8.43M
)
602
330M
        return nullptr;
603
8.34M
604
8.34M
      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
605
8.34M
      if (!Info) {
606
109k
        Info = new (PP.getPreprocessorAllocator())
607
109k
            ModuleMacroInfo(State.get<MacroDirective *>());
608
109k
        State = Info;
609
109k
      }
610
8.34M
611
8.34M
      if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
612
8.34M
          Info->ActiveModuleMacrosGeneration)
613
329k
        PP.updateModuleMacroInfo(II, *Info);
614
8.34M
      return Info;
615
8.34M
    }
616
617
  public:
618
40.1M
    MacroState() : MacroState(nullptr) {}
Unexecuted instantiation: clang::Preprocessor::MacroState::MacroState()
clang::Preprocessor::MacroState::MacroState()
Line
Count
Source
618
40.1M
    MacroState() : MacroState(nullptr) {}
619
40.2M
    MacroState(MacroDirective *MD) : State(MD) {}
620
621
60.6M
    MacroState(MacroState &&O) noexcept : State(O.State) {
622
60.6M
      O.State = (MacroDirective *)nullptr;
623
60.6M
    }
624
625
37.5k
    MacroState &operator=(MacroState &&O) noexcept {
626
37.5k
      auto S = O.State;
627
37.5k
      O.State = (MacroDirective *)nullptr;
628
37.5k
      State = S;
629
37.5k
      return *this;
630
37.5k
    }
631
632
88.3M
    ~MacroState() {
633
88.3M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
634
107k
        Info->~ModuleMacroInfo();
635
88.3M
    }
636
637
190M
    MacroDirective *getLatest() const {
638
190M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
639
4.23M
        return Info->MD;
640
186M
      return State.get<MacroDirective*>();
641
186M
    }
642
643
36.3M
    void setLatest(MacroDirective *MD) {
644
36.3M
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
645
89.9k
        Info->MD = MD;
646
36.2M
      else
647
36.2M
        State = MD;
648
36.3M
    }
649
650
151M
    bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
651
151M
      auto *Info = getModuleInfo(PP, II);
652
151M
      return Info ? 
Info->IsAmbiguous4.16M
:
false147M
;
653
151M
    }
654
655
    ArrayRef<ModuleMacro *>
656
151M
    getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
657
151M
      if (auto *Info = getModuleInfo(PP, II))
658
4.16M
        return Info->ActiveModuleMacros;
659
147M
      return None;
660
147M
    }
661
662
    MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
663
115k
                                               SourceManager &SourceMgr) const {
664
115k
      // FIXME: Incorporate module macros into the result of this.
665
115k
      if (auto *Latest = getLatest())
666
115k
        return Latest->findDirectiveAtLoc(Loc, SourceMgr);
667
1
      return {};
668
1
    }
669
670
35.9M
    void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
671
35.9M
      if (auto *Info = getModuleInfo(PP, II)) {
672
12.3k
        Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
673
12.3k
                                      Info->ActiveModuleMacros.begin(),
674
12.3k
                                      Info->ActiveModuleMacros.end());
675
12.3k
        Info->ActiveModuleMacros.clear();
676
12.3k
        Info->IsAmbiguous = false;
677
12.3k
      }
678
35.9M
    }
679
680
498k
    ArrayRef<ModuleMacro*> getOverriddenMacros() const {
681
498k
      if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
682
74.8k
        return Info->OverriddenMacros;
683
423k
      return None;
684
423k
    }
685
686
    void setOverriddenMacros(Preprocessor &PP,
687
497k
                             ArrayRef<ModuleMacro *> Overrides) {
688
497k
      auto *Info = State.dyn_cast<ModuleMacroInfo*>();
689
497k
      if (!Info) {
690
423k
        if (Overrides.empty())
691
423k
          return;
692
0
        Info = new (PP.getPreprocessorAllocator())
693
0
            ModuleMacroInfo(State.get<MacroDirective *>());
694
0
        State = Info;
695
0
      }
696
497k
      Info->OverriddenMacros.clear();
697
74.2k
      Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
698
74.2k
                                    Overrides.begin(), Overrides.end());
699
74.2k
      Info->ActiveModuleMacrosGeneration = 0;
700
74.2k
    }
701
  };
702
703
  /// For each IdentifierInfo that was associated with a macro, we
704
  /// keep a mapping to the history of all macro definitions and #undefs in
705
  /// the reverse order (the latest one is in the head of the list).
706
  ///
707
  /// This mapping lives within the \p CurSubmoduleState.
708
  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
709
710
  struct SubmoduleState;
711
712
  /// Information about a submodule that we're currently building.
713
  struct BuildingSubmoduleInfo {
714
    /// The module that we are building.
715
    Module *M;
716
717
    /// The location at which the module was included.
718
    SourceLocation ImportLoc;
719
720
    /// Whether we entered this submodule via a pragma.
721
    bool IsPragma;
722
723
    /// The previous SubmoduleState.
724
    SubmoduleState *OuterSubmoduleState;
725
726
    /// The number of pending module macro names when we started building this.
727
    unsigned OuterPendingModuleMacroNames;
728
729
    BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
730
                          SubmoduleState *OuterSubmoduleState,
731
                          unsigned OuterPendingModuleMacroNames)
732
        : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
733
          OuterSubmoduleState(OuterSubmoduleState),
734
79.5k
          OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
735
  };
736
  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
737
738
  /// Information about a submodule's preprocessor state.
739
  struct SubmoduleState {
740
    /// The macros for the submodule.
741
    MacroMap Macros;
742
743
    /// The set of modules that are visible within the submodule.
744
    VisibleModuleSet VisibleModules;
745
746
    // FIXME: CounterValue?
747
    // FIXME: PragmaPushMacroInfo?
748
  };
749
  std::map<Module *, SubmoduleState> Submodules;
750
751
  /// The preprocessor state for preprocessing outside of any submodule.
752
  SubmoduleState NullSubmoduleState;
753
754
  /// The current submodule state. Will be \p NullSubmoduleState if we're not
755
  /// in a submodule.
756
  SubmoduleState *CurSubmoduleState;
757
758
  /// The set of known macros exported from modules.
759
  llvm::FoldingSet<ModuleMacro> ModuleMacros;
760
761
  /// The names of potential module macros that we've not yet processed.
762
  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
763
764
  /// The list of module macros, for each identifier, that are not overridden by
765
  /// any other module macro.
766
  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
767
      LeafModuleMacros;
768
769
  /// Macros that we want to warn because they are not used at the end
770
  /// of the translation unit.
771
  ///
772
  /// We store just their SourceLocations instead of
773
  /// something like MacroInfo*. The benefit of this is that when we are
774
  /// deserializing from PCH, we don't need to deserialize identifier & macros
775
  /// just so that we can report that they are unused, we just warn using
776
  /// the SourceLocations of this set (that will be filled by the ASTReader).
777
  /// We are using SmallPtrSet instead of a vector for faster removal.
778
  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
779
  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
780
781
  /// A "freelist" of MacroArg objects that can be
782
  /// reused for quick allocation.
783
  MacroArgs *MacroArgCache = nullptr;
784
785
  /// For each IdentifierInfo used in a \#pragma push_macro directive,
786
  /// we keep a MacroInfo stack used to restore the previous macro value.
787
  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
788
      PragmaPushMacroInfo;
789
790
  // Various statistics we track for performance analysis.
791
  unsigned NumDirectives = 0;
792
  unsigned NumDefined = 0;
793
  unsigned NumUndefined = 0;
794
  unsigned NumPragma = 0;
795
  unsigned NumIf = 0;
796
  unsigned NumElse = 0;
797
  unsigned NumEndif = 0;
798
  unsigned NumEnteredSourceFiles = 0;
799
  unsigned MaxIncludeStackDepth = 0;
800
  unsigned NumMacroExpanded = 0;
801
  unsigned NumFnMacroExpanded = 0;
802
  unsigned NumBuiltinMacroExpanded = 0;
803
  unsigned NumFastMacroExpanded = 0;
804
  unsigned NumTokenPaste = 0;
805
  unsigned NumFastTokenPaste = 0;
806
  unsigned NumSkipped = 0;
807
808
  /// The predefined macros that preprocessor should use from the
809
  /// command line etc.
810
  std::string Predefines;
811
812
  /// The file ID for the preprocessor predefines.
813
  FileID PredefinesFileID;
814
815
  /// The file ID for the PCH through header.
816
  FileID PCHThroughHeaderFileID;
817
818
  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
819
  bool SkippingUntilPragmaHdrStop = false;
820
821
  /// Whether tokens are being skipped until the through header is seen.
822
  bool SkippingUntilPCHThroughHeader = false;
823
824
  /// \{
825
  /// Cache of macro expanders to reduce malloc traffic.
826
  enum { TokenLexerCacheSize = 8 };
827
  unsigned NumCachedTokenLexers;
828
  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
829
  /// \}
830
831
  /// Keeps macro expanded tokens for TokenLexers.
832
  //
833
  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
834
  /// going to lex in the cache and when it finishes the tokens are removed
835
  /// from the end of the cache.
836
  SmallVector<Token, 16> MacroExpandedTokens;
837
  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
838
839
  /// A record of the macro definitions and expansions that
840
  /// occurred during preprocessing.
841
  ///
842
  /// This is an optional side structure that can be enabled with
843
  /// \c createPreprocessingRecord() prior to preprocessing.
844
  PreprocessingRecord *Record = nullptr;
845
846
  /// Cached tokens state.
847
  using CachedTokensTy = SmallVector<Token, 1>;
848
849
  /// Cached tokens are stored here when we do backtracking or
850
  /// lookahead. They are "lexed" by the CachingLex() method.
851
  CachedTokensTy CachedTokens;
852
853
  /// The position of the cached token that CachingLex() should
854
  /// "lex" next.
855
  ///
856
  /// If it points beyond the CachedTokens vector, it means that a normal
857
  /// Lex() should be invoked.
858
  CachedTokensTy::size_type CachedLexPos = 0;
859
860
  /// Stack of backtrack positions, allowing nested backtracks.
861
  ///
862
  /// The EnableBacktrackAtThisPos() method pushes a position to
863
  /// indicate where CachedLexPos should be set when the BackTrack() method is
864
  /// invoked (at which point the last position is popped).
865
  std::vector<CachedTokensTy::size_type> BacktrackPositions;
866
867
  struct MacroInfoChain {
868
    MacroInfo MI;
869
    MacroInfoChain *Next;
870
  };
871
872
  /// MacroInfos are managed as a chain for easy disposal.  This is the head
873
  /// of that list.
874
  MacroInfoChain *MIChainHead = nullptr;
875
876
  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
877
878
public:
879
  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
880
               DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
881
               HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
882
               IdentifierInfoLookup *IILookup = nullptr,
883
               bool OwnsHeaderSearch = false,
884
               TranslationUnitKind TUKind = TU_Complete);
885
886
  ~Preprocessor();
887
888
  /// Initialize the preprocessor using information about the target.
889
  ///
890
  /// \param Target is owned by the caller and must remain valid for the
891
  /// lifetime of the preprocessor.
892
  /// \param AuxTarget is owned by the caller and must remain valid for
893
  /// the lifetime of the preprocessor.
894
  void Initialize(const TargetInfo &Target,
895
                  const TargetInfo *AuxTarget = nullptr);
896
897
  /// Initialize the preprocessor to parse a model file
898
  ///
899
  /// To parse model files the preprocessor of the original source is reused to
900
  /// preserver the identifier table. However to avoid some duplicate
901
  /// information in the preprocessor some cleanup is needed before it is used
902
  /// to parse model files. This method does that cleanup.
903
  void InitializeForModelFile();
904
905
  /// Cleanup after model file parsing
906
  void FinalizeForModelFile();
907
908
  /// Retrieve the preprocessor options used to initialize this
909
  /// preprocessor.
910
108k
  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
911
912
5.73M
  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
913
200
  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
914
915
4.17G
  const LangOptions &getLangOpts() const { return LangOpts; }
916
13.9M
  const TargetInfo &getTargetInfo() const { return *Target; }
917
1.03k
  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
918
175k
  FileManager &getFileManager() const { return FileMgr; }
919
1.23G
  SourceManager &getSourceManager() const { return SourceMgr; }
920
2.57M
  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
921
922
24.7M
  IdentifierTable &getIdentifierTable() { return Identifiers; }
923
4.35k
  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
924
1.75M
  SelectorTable &getSelectorTable() { return Selectors; }
925
111k
  Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
926
2.54M
  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
927
928
6.44k
  void setExternalSource(ExternalPreprocessorSource *Source) {
929
6.44k
    ExternalSource = Source;
930
6.44k
  }
931
932
1.33M
  ExternalPreprocessorSource *getExternalSource() const {
933
1.33M
    return ExternalSource;
934
1.33M
  }
935
936
  /// Retrieve the module loader associated with this preprocessor.
937
106k
  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
938
939
48.9M
  bool hadModuleLoaderFatalFailure() const {
940
48.9M
    return TheModuleLoader.HadFatalFailure;
941
48.9M
  }
942
943
  /// Retrieve the number of Directives that have been processed by the
944
  /// Preprocessor.
945
684k
  unsigned getNumDirectives() const {
946
684k
    return NumDirectives;
947
684k
  }
948
949
  /// True if we are currently preprocessing a #if or #elif directive
950
17.0k
  bool isParsingIfOrElifDirective() const {
951
17.0k
    return ParsingIfOrElifDirective;
952
17.0k
  }
953
954
  /// Control whether the preprocessor retains comments in output.
955
588
  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
956
588
    this->KeepComments = KeepComments | KeepMacroComments;
957
588
    this->KeepMacroComments = KeepMacroComments;
958
588
  }
959
960
198M
  bool getCommentRetentionState() const { return KeepComments; }
961
962
200
  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
963
100
  bool getPragmasEnabled() const { return PragmasEnabled; }
964
965
4
  void SetSuppressIncludeNotFoundError(bool Suppress) {
966
4
    SuppressIncludeNotFoundError = Suppress;
967
4
  }
968
969
0
  bool GetSuppressIncludeNotFoundError() {
970
0
    return SuppressIncludeNotFoundError;
971
0
  }
972
973
  /// Sets whether the preprocessor is responsible for producing output or if
974
  /// it is producing tokens to be consumed by Parse and Sema.
975
59.3k
  void setPreprocessedOutput(bool IsPreprocessedOutput) {
976
59.3k
    PreprocessedOutput = IsPreprocessedOutput;
977
59.3k
  }
978
979
  /// Returns true if the preprocessor is responsible for generating output,
980
  /// false if it is producing tokens to be consumed by Parse and Sema.
981
5.67k
  bool isPreprocessedOutput() const { return PreprocessedOutput; }
982
983
  /// Return true if we are lexing directly from the specified lexer.
984
0
  bool isCurrentLexer(const PreprocessorLexer *L) const {
985
0
    return CurPPLexer == L;
986
0
  }
987
988
  /// Return the current lexer being lexed from.
989
  ///
990
  /// Note that this ignores any potentially active macro expansions and _Pragma
991
  /// expansions going on at the time.
992
53.7k
  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
993
994
  /// Return the current file lexer being lexed from.
995
  ///
996
  /// Note that this ignores any potentially active macro expansions and _Pragma
997
  /// expansions going on at the time.
998
  PreprocessorLexer *getCurrentFileLexer() const;
999
1000
  /// Return the submodule owning the file being lexed. This may not be
1001
  /// the current module if we have changed modules since entering the file.
1002
4.43k
  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
1003
1004
  /// Returns the FileID for the preprocessor predefines.
1005
660k
  FileID getPredefinesFileID() const { return PredefinesFileID; }
1006
1007
  /// \{
1008
  /// Accessors for preprocessor callbacks.
1009
  ///
1010
  /// Note that this class takes ownership of any PPCallbacks object given to
1011
  /// it.
1012
2.40M
  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1013
93.0k
  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1014
93.0k
    if (Callbacks)
1015
35.0k
      C = std::make_unique<PPChainedCallbacks>(std::move(C),
1016
35.0k
                                                std::move(Callbacks));
1017
93.0k
    Callbacks = std::move(C);
1018
93.0k
  }
1019
  /// \}
1020
1021
  /// Get the number of tokens processed so far.
1022
10
  unsigned getTokenCount() const { return TokenCount; }
1023
1024
  /// Get the max number of tokens before issuing a -Wmax-tokens warning.
1025
54.5k
  unsigned getMaxTokens() const { return MaxTokens; }
1026
1027
1
  void overrideMaxTokens(unsigned Value, SourceLocation Loc) {
1028
1
    MaxTokens = Value;
1029
1
    MaxTokensOverrideLoc = Loc;
1030
1
  };
1031
1032
2
  SourceLocation getMaxTokensOverrideLoc() const { return MaxTokensOverrideLoc; }
1033
1034
  /// Register a function that would be called on each token in the final
1035
  /// expanded token stream.
1036
  /// This also reports annotation tokens produced by the parser.
1037
118
  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1038
118
    OnToken = std::move(F);
1039
118
  }
1040
1041
3.46k
  bool isMacroDefined(StringRef Id) {
1042
3.46k
    return isMacroDefined(&Identifiers.get(Id));
1043
3.46k
  }
1044
1.54M
  bool isMacroDefined(const IdentifierInfo *II) {
1045
1.54M
    return II->hasMacroDefinition() &&
1046
1.54M
           
(1.53M
!getLangOpts().Modules1.53M
||
(bool)getMacroDefinition(II)21.1k
);
1047
1.54M
  }
1048
1049
  /// Determine whether II is defined as a macro within the module M,
1050
  /// if that is a module that we've already preprocessed. Does not check for
1051
  /// macros imported into M.
1052
53.0k
  bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M) {
1053
53.0k
    if (!II->hasMacroDefinition())
1054
77
      return false;
1055
52.9k
    auto I = Submodules.find(M);
1056
52.9k
    if (I == Submodules.end())
1057
52.9k
      return false;
1058
14
    auto J = I->second.Macros.find(II);
1059
14
    if (J == I->second.Macros.end())
1060
0
      return false;
1061
14
    auto *MD = J->second.getLatest();
1062
14
    return MD && MD->isDefined();
1063
14
  }
1064
1065
154M
  MacroDefinition getMacroDefinition(const IdentifierInfo *II) {
1066
154M
    if (!II->hasMacroDefinition())
1067
3.55M
      return {};
1068
151M
1069
151M
    MacroState &S = CurSubmoduleState->Macros[II];
1070
151M
    auto *MD = S.getLatest();
1071
151M
    while (MD && 
isa<VisibilityMacroDirective>(MD)147M
)
1072
125
      MD = MD->getPrevious();
1073
151M
    return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1074
151M
                           S.getActiveModuleMacros(*this, II),
1075
151M
                           S.isAmbiguous(*this, II));
1076
151M
  }
1077
1078
  MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II,
1079
24
                                          SourceLocation Loc) {
1080
24
    if (!II->hadMacroDefinition())
1081
9
      return {};
1082
15
1083
15
    MacroState &S = CurSubmoduleState->Macros[II];
1084
15
    MacroDirective::DefInfo DI;
1085
15
    if (auto *MD = S.getLatest())
1086
15
      DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1087
15
    // FIXME: Compute the set of active module macros at the specified location.
1088
15
    return MacroDefinition(DI.getDirective(),
1089
15
                           S.getActiveModuleMacros(*this, II),
1090
15
                           S.isAmbiguous(*this, II));
1091
15
  }
1092
1093
  /// Given an identifier, return its latest non-imported MacroDirective
1094
  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1095
179
  MacroDirective *getLocalMacroDirective(const IdentifierInfo *II) const {
1096
179
    if (!II->hasMacroDefinition())
1097
9
      return nullptr;
1098
170
1099
170
    auto *MD = getLocalMacroDirectiveHistory(II);
1100
170
    if (!MD || MD->getDefinition().isUndefined())
1101
0
      return nullptr;
1102
170
1103
170
    return MD;
1104
170
  }
1105
1106
25.4M
  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1107
25.4M
    return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1108
25.4M
  }
1109
1110
151M
  MacroInfo *getMacroInfo(const IdentifierInfo *II) {
1111
151M
    if (!II->hasMacroDefinition())
1112
108M
      return nullptr;
1113
43.7M
    if (auto MD = getMacroDefinition(II))
1114
43.6M
      return MD.getMacroInfo();
1115
17.5k
    return nullptr;
1116
17.5k
  }
1117
1118
  /// Given an identifier, return the latest non-imported macro
1119
  /// directive for that identifier.
1120
  ///
1121
  /// One can iterate over all previous macro directives from the most recent
1122
  /// one.
1123
  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1124
1125
  /// Add a directive to the macro directive history for this identifier.
1126
  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1127
  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI,
1128
35.7M
                                             SourceLocation Loc) {
1129
35.7M
    DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1130
35.7M
    appendMacroDirective(II, MD);
1131
35.7M
    return MD;
1132
35.7M
  }
1133
  DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II,
1134
35.7M
                                             MacroInfo *MI) {
1135
35.7M
    return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1136
35.7M
  }
1137
1138
  /// Set a MacroDirective that was loaded from a PCH file.
1139
  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1140
                               MacroDirective *MD);
1141
1142
  /// Register an exported macro for a module and identifier.
1143
  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1144
                              ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1145
  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1146
1147
  /// Get the list of leaf (non-overridden) module macros for a name.
1148
6.81M
  ArrayRef<ModuleMacro*> getLeafModuleMacros(const IdentifierInfo *II) const {
1149
6.81M
    if (II->isOutOfDate())
1150
83
      updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1151
6.81M
    auto I = LeafModuleMacros.find(II);
1152
6.81M
    if (I != LeafModuleMacros.end())
1153
5.08M
      return I->second;
1154
1.73M
    return None;
1155
1.73M
  }
1156
1157
  /// \{
1158
  /// Iterators for the macro history table. Currently defined macros have
1159
  /// IdentifierInfo::hasMacroDefinition() set and an empty
1160
  /// MacroInfo::getUndefLoc() at the head of the list.
1161
  using macro_iterator = MacroMap::const_iterator;
1162
1163
  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1164
  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1165
1166
  llvm::iterator_range<macro_iterator>
1167
7
  macros(bool IncludeExternalMacros = true) const {
1168
7
    macro_iterator begin = macro_begin(IncludeExternalMacros);
1169
7
    macro_iterator end = macro_end(IncludeExternalMacros);
1170
7
    return llvm::make_range(begin, end);
1171
7
  }
1172
1173
  /// \}
1174
1175
  /// Return the name of the macro defined before \p Loc that has
1176
  /// spelling \p Tokens.  If there are multiple macros with same spelling,
1177
  /// return the last one defined.
1178
  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1179
                                     ArrayRef<TokenValue> Tokens) const;
1180
1181
0
  const std::string &getPredefines() const { return Predefines; }
1182
1183
  /// Set the predefines for this Preprocessor.
1184
  ///
1185
  /// These predefines are automatically injected when parsing the main file.
1186
0
  void setPredefines(const char *P) { Predefines = P; }
1187
62.0k
  void setPredefines(StringRef P) { Predefines = std::string(P); }
1188
1189
  /// Return information about the specified preprocessor
1190
  /// identifier token.
1191
396M
  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1192
396M
    return &Identifiers.get(Name);
1193
396M
  }
1194
1195
  /// Add the specified pragma handler to this preprocessor.
1196
  ///
1197
  /// If \p Namespace is non-null, then it is a token required to exist on the
1198
  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1199
  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1200
1.13M
  void AddPragmaHandler(PragmaHandler *Handler) {
1201
1.13M
    AddPragmaHandler(StringRef(), Handler);
1202
1.13M
  }
1203
1204
  /// Remove the specific pragma handler from this preprocessor.
1205
  ///
1206
  /// If \p Namespace is non-null, then it should be the namespace that
1207
  /// \p Handler was added to. It is an error to remove a handler that
1208
  /// has not been registered.
1209
  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1210
705k
  void RemovePragmaHandler(PragmaHandler *Handler) {
1211
705k
    RemovePragmaHandler(StringRef(), Handler);
1212
705k
  }
1213
1214
  /// Install empty handlers for all pragmas (making them ignored).
1215
  void IgnorePragmas();
1216
1217
  /// Add the specified comment handler to the preprocessor.
1218
  void addCommentHandler(CommentHandler *Handler);
1219
1220
  /// Remove the specified comment handler.
1221
  ///
1222
  /// It is an error to remove a handler that has not been registered.
1223
  void removeCommentHandler(CommentHandler *Handler);
1224
1225
  /// Set the code completion handler to the given object.
1226
54.5k
  void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
1227
54.5k
    CodeComplete = &Handler;
1228
54.5k
  }
1229
1230
  /// Retrieve the current code-completion handler.
1231
16
  CodeCompletionHandler *getCodeCompletionHandler() const {
1232
16
    return CodeComplete;
1233
16
  }
1234
1235
  /// Clear out the code completion handler.
1236
54.4k
  void clearCodeCompletionHandler() {
1237
54.4k
    CodeComplete = nullptr;
1238
54.4k
  }
1239
1240
  /// Hook used by the lexer to invoke the "included file" code
1241
  /// completion point.
1242
  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1243
1244
  /// Hook used by the lexer to invoke the "natural language" code
1245
  /// completion point.
1246
  void CodeCompleteNaturalLanguage();
1247
1248
  /// Set the code completion token for filtering purposes.
1249
96
  void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter) {
1250
96
    CodeCompletionII = Filter;
1251
96
  }
1252
1253
  /// Set the code completion token range for detecting replacement range later
1254
  /// on.
1255
  void setCodeCompletionTokenRange(const SourceLocation Start,
1256
96
                                   const SourceLocation End) {
1257
96
    CodeCompletionTokenRange = {Start, End};
1258
96
  }
1259
0
  SourceRange getCodeCompletionTokenRange() const {
1260
0
    return CodeCompletionTokenRange;
1261
0
  }
1262
1263
  /// Get the code completion token for filtering purposes.
1264
312
  StringRef getCodeCompletionFilter() {
1265
312
    if (CodeCompletionII)
1266
74
      return CodeCompletionII->getName();
1267
238
    return {};
1268
238
  }
1269
1270
  /// Retrieve the preprocessing record, or NULL if there is no
1271
  /// preprocessing record.
1272
193k
  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1273
1274
  /// Create a new preprocessing record, which will keep track of
1275
  /// all macro expansions, macro definitions, etc.
1276
  void createPreprocessingRecord();
1277
1278
  /// Returns true if the FileEntry is the PCH through header.
1279
  bool isPCHThroughHeader(const FileEntry *FE);
1280
1281
  /// True if creating a PCH with a through header.
1282
  bool creatingPCHWithThroughHeader();
1283
1284
  /// True if using a PCH with a through header.
1285
  bool usingPCHWithThroughHeader();
1286
1287
  /// True if creating a PCH with a #pragma hdrstop.
1288
  bool creatingPCHWithPragmaHdrStop();
1289
1290
  /// True if using a PCH with a #pragma hdrstop.
1291
  bool usingPCHWithPragmaHdrStop();
1292
1293
  /// Skip tokens until after the #include of the through header or
1294
  /// until after a #pragma hdrstop.
1295
  void SkipTokensWhileUsingPCH();
1296
1297
  /// Process directives while skipping until the through header or
1298
  /// #pragma hdrstop is found.
1299
  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1300
                                           SourceLocation HashLoc);
1301
1302
  /// Enter the specified FileID as the main source file,
1303
  /// which implicitly adds the builtin defines etc.
1304
  void EnterMainSourceFile();
1305
1306
  /// Inform the preprocessor callbacks that processing is complete.
1307
  void EndSourceFile();
1308
1309
  /// Add a source file to the top of the include stack and
1310
  /// start lexing tokens from it instead of the current buffer.
1311
  ///
1312
  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1313
  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1314
                       SourceLocation Loc);
1315
1316
  /// Add a Macro to the top of the include stack and start lexing
1317
  /// tokens from it instead of the current buffer.
1318
  ///
1319
  /// \param Args specifies the tokens input to a function-like macro.
1320
  /// \param ILEnd specifies the location of the ')' for a function-like macro
1321
  /// or the identifier for an object-like macro.
1322
  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1323
                  MacroArgs *Args);
1324
1325
private:
1326
  /// Add a "macro" context to the top of the include stack,
1327
  /// which will cause the lexer to start returning the specified tokens.
1328
  ///
1329
  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1330
  /// will not be subject to further macro expansion. Otherwise, these tokens
1331
  /// will be re-macro-expanded when/if expansion is enabled.
1332
  ///
1333
  /// If \p OwnsTokens is false, this method assumes that the specified stream
1334
  /// of tokens has a permanent owner somewhere, so they do not need to be
1335
  /// copied. If it is true, it assumes the array of tokens is allocated with
1336
  /// \c new[] and the Preprocessor will delete[] it.
1337
  ///
1338
  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1339
  /// set, see the flag documentation for details.
1340
  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1341
                        bool DisableMacroExpansion, bool OwnsTokens,
1342
                        bool IsReinject);
1343
1344
public:
1345
  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1346
289k
                        bool DisableMacroExpansion, bool IsReinject) {
1347
289k
    EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1348
289k
                     IsReinject);
1349
289k
  }
1350
1351
  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1352
798k
                        bool IsReinject) {
1353
798k
    EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1354
798k
                     IsReinject);
1355
798k
  }
1356
1357
  /// Pop the current lexer/macro exp off the top of the lexer stack.
1358
  ///
1359
  /// This should only be used in situations where the current state of the
1360
  /// top-of-stack lexer is known.
1361
  void RemoveTopOfLexerStack();
1362
1363
  /// From the point that this method is called, and until
1364
  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1365
  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1366
  /// make the Preprocessor re-lex the same tokens.
1367
  ///
1368
  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1369
  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1370
  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1371
  ///
1372
  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1373
  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1374
  /// tokens will continue indefinitely.
1375
  ///
1376
  void EnableBacktrackAtThisPos();
1377
1378
  /// Disable the last EnableBacktrackAtThisPos call.
1379
  void CommitBacktrackedTokens();
1380
1381
  /// Make Preprocessor re-lex the tokens that were lexed since
1382
  /// EnableBacktrackAtThisPos() was previously called.
1383
  void Backtrack();
1384
1385
  /// True if EnableBacktrackAtThisPos() was called and
1386
  /// caching of tokens is on.
1387
52.7M
  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1388
1389
  /// Lex the next token for this preprocessor.
1390
  void Lex(Token &Result);
1391
1392
  /// Lex a token, forming a header-name token if possible.
1393
  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1394
1395
  bool LexAfterModuleImport(Token &Result);
1396
  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1397
1398
  void makeModuleVisible(Module *M, SourceLocation Loc);
1399
1400
59
  SourceLocation getModuleImportLoc(Module *M) const {
1401
59
    return CurSubmoduleState->VisibleModules.getImportLoc(M);
1402
59
  }
1403
1404
  /// Lex a string literal, which may be the concatenation of multiple
1405
  /// string literals and may even come from macro expansion.
1406
  /// \returns true on success, false if a error diagnostic has been generated.
1407
  bool LexStringLiteral(Token &Result, std::string &String,
1408
147
                        const char *DiagnosticTag, bool AllowMacroExpansion) {
1409
147
    if (AllowMacroExpansion)
1410
92
      Lex(Result);
1411
55
    else
1412
55
      LexUnexpandedToken(Result);
1413
147
    return FinishLexStringLiteral(Result, String, DiagnosticTag,
1414
147
                                  AllowMacroExpansion);
1415
147
  }
1416
1417
  /// Complete the lexing of a string literal where the first token has
1418
  /// already been lexed (see LexStringLiteral).
1419
  bool FinishLexStringLiteral(Token &Result, std::string &String,
1420
                              const char *DiagnosticTag,
1421
                              bool AllowMacroExpansion);
1422
1423
  /// Lex a token.  If it's a comment, keep lexing until we get
1424
  /// something not a comment.
1425
  ///
1426
  /// This is useful in -E -C mode where comments would foul up preprocessor
1427
  /// directive handling.
1428
14.3M
  void LexNonComment(Token &Result) {
1429
14.3M
    do
1430
14.3M
      Lex(Result);
1431
14.3M
    while (Result.getKind() == tok::comment);
1432
14.3M
  }
1433
1434
  /// Just like Lex, but disables macro expansion of identifier tokens.
1435
791M
  void LexUnexpandedToken(Token &Result) {
1436
791M
    // Disable macro expansion.
1437
791M
    bool OldVal = DisableMacroExpansion;
1438
791M
    DisableMacroExpansion = true;
1439
791M
    // Lex the token.
1440
791M
    Lex(Result);
1441
791M
1442
791M
    // Reenable it.
1443
791M
    DisableMacroExpansion = OldVal;
1444
791M
  }
1445
1446
  /// Like LexNonComment, but this disables macro expansion of
1447
  /// identifier tokens.
1448
5.91M
  void LexUnexpandedNonComment(Token &Result) {
1449
5.91M
    do
1450
5.91M
      LexUnexpandedToken(Result);
1451
5.91M
    while (Result.getKind() == tok::comment);
1452
5.91M
  }
1453
1454
  /// Parses a simple integer literal to get its numeric value.  Floating
1455
  /// point literals and user defined literals are rejected.  Used primarily to
1456
  /// handle pragmas that accept integer arguments.
1457
  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1458
1459
  /// Disables macro expansion everywhere except for preprocessor directives.
1460
54
  void SetMacroExpansionOnlyInDirectives() {
1461
54
    DisableMacroExpansion = true;
1462
54
    MacroExpansionInDirectivesOverride = true;
1463
54
  }
1464
1465
  /// Peeks ahead N tokens and returns that token without consuming any
1466
  /// tokens.
1467
  ///
1468
  /// LookAhead(0) returns the next token that would be returned by Lex(),
1469
  /// LookAhead(1) returns the token after it, etc.  This returns normal
1470
  /// tokens after phase 5.  As such, it is equivalent to using
1471
  /// 'Lex', not 'LexUnexpandedToken'.
1472
116M
  const Token &LookAhead(unsigned N) {
1473
116M
    assert(LexLevel == 0 && "cannot use lookahead while lexing");
1474
116M
    if (CachedLexPos + N < CachedTokens.size())
1475
60.1M
      return CachedTokens[CachedLexPos+N];
1476
56.0M
    else
1477
56.0M
      return PeekAhead(N+1);
1478
116M
  }
1479
1480
  /// When backtracking is enabled and tokens are cached,
1481
  /// this allows to revert a specific number of tokens.
1482
  ///
1483
  /// Note that the number of tokens being reverted should be up to the last
1484
  /// backtrack position, not more.
1485
33.7k
  void RevertCachedTokens(unsigned N) {
1486
33.7k
    assert(isBacktrackEnabled() &&
1487
33.7k
           "Should only be called when tokens are cached for backtracking");
1488
33.7k
    assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1489
33.7k
         && "Should revert tokens up to the last backtrack position, not more");
1490
33.7k
    assert(signed(CachedLexPos) - signed(N) >= 0 &&
1491
33.7k
           "Corrupted backtrack positions ?");
1492
33.7k
    CachedLexPos -= N;
1493
33.7k
  }
1494
1495
  /// Enters a token in the token stream to be lexed next.
1496
  ///
1497
  /// If BackTrack() is called afterwards, the token will remain at the
1498
  /// insertion point.
1499
  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1500
  /// flag set. See the flag documentation for details.
1501
2.95M
  void EnterToken(const Token &Tok, bool IsReinject) {
1502
2.95M
    if (LexLevel) {
1503
247
      // It's not correct in general to enter caching lex mode while in the
1504
247
      // middle of a nested lexing action.
1505
247
      auto TokCopy = std::make_unique<Token[]>(1);
1506
247
      TokCopy[0] = Tok;
1507
247
      EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1508
2.95M
    } else {
1509
2.95M
      EnterCachingLexMode();
1510
2.95M
      assert(IsReinject && "new tokens in the middle of cached stream");
1511
2.95M
      CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1512
2.95M
    }
1513
2.95M
  }
1514
1515
  /// We notify the Preprocessor that if it is caching tokens (because
1516
  /// backtrack is enabled) it should replace the most recent cached tokens
1517
  /// with the given annotation token. This function has no effect if
1518
  /// backtracking is not enabled.
1519
  ///
1520
  /// Note that the use of this function is just for optimization, so that the
1521
  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1522
  /// invoked.
1523
20.5M
  void AnnotateCachedTokens(const Token &Tok) {
1524
20.5M
    assert(Tok.isAnnotation() && "Expected annotation token");
1525
20.5M
    if (CachedLexPos != 0 && 
isBacktrackEnabled()5.17M
)
1526
930k
      AnnotatePreviousCachedTokens(Tok);
1527
20.5M
  }
1528
1529
  /// Get the location of the last cached token, suitable for setting the end
1530
  /// location of an annotation token.
1531
13.3k
  SourceLocation getLastCachedTokenLocation() const {
1532
13.3k
    assert(CachedLexPos != 0);
1533
13.3k
    return CachedTokens[CachedLexPos-1].getLastLoc();
1534
13.3k
  }
1535
1536
  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1537
  /// CachedTokens.
1538
  bool IsPreviousCachedToken(const Token &Tok) const;
1539
1540
  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1541
  /// in \p NewToks.
1542
  ///
1543
  /// Useful when a token needs to be split in smaller ones and CachedTokens
1544
  /// most recent token must to be updated to reflect that.
1545
  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1546
1547
  /// Replace the last token with an annotation token.
1548
  ///
1549
  /// Like AnnotateCachedTokens(), this routine replaces an
1550
  /// already-parsed (and resolved) token with an annotation
1551
  /// token. However, this routine only replaces the last token with
1552
  /// the annotation token; it does not affect any other cached
1553
  /// tokens. This function has no effect if backtracking is not
1554
  /// enabled.
1555
0
  void ReplaceLastTokenWithAnnotation(const Token &Tok) {
1556
0
    assert(Tok.isAnnotation() && "Expected annotation token");
1557
0
    if (CachedLexPos != 0 && isBacktrackEnabled())
1558
0
      CachedTokens[CachedLexPos-1] = Tok;
1559
0
  }
1560
1561
  /// Enter an annotation token into the token stream.
1562
  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1563
                            void *AnnotationVal);
1564
1565
  /// Update the current token to represent the provided
1566
  /// identifier, in order to cache an action performed by typo correction.
1567
16
  void TypoCorrectToken(const Token &Tok) {
1568
16
    assert(Tok.getIdentifierInfo() && "Expected identifier token");
1569
16
    if (CachedLexPos != 0 && 
isBacktrackEnabled()9
)
1570
9
      CachedTokens[CachedLexPos-1] = Tok;
1571
16
  }
1572
1573
  /// Recompute the current lexer kind based on the CurLexer/
1574
  /// CurTokenLexer pointers.
1575
  void recomputeCurLexerKind();
1576
1577
  /// Returns true if incremental processing is enabled
1578
8.40M
  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1579
1580
  /// Enables the incremental processing
1581
1.17k
  void enableIncrementalProcessing(bool value = true) {
1582
1.17k
    IncrementalProcessing = value;
1583
1.17k
  }
1584
1585
  /// Specify the point at which code-completion will be performed.
1586
  ///
1587
  /// \param File the file in which code completion should occur. If
1588
  /// this file is included multiple times, code-completion will
1589
  /// perform completion the first time it is included. If NULL, this
1590
  /// function clears out the code-completion point.
1591
  ///
1592
  /// \param Line the line at which code completion should occur
1593
  /// (1-based).
1594
  ///
1595
  /// \param Column the column at which code completion should occur
1596
  /// (1-based).
1597
  ///
1598
  /// \returns true if an error occurred, false otherwise.
1599
  bool SetCodeCompletionPoint(const FileEntry *File,
1600
                              unsigned Line, unsigned Column);
1601
1602
  /// Determine if we are performing code completion.
1603
433M
  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1604
1605
  /// Returns the location of the code-completion point.
1606
  ///
1607
  /// Returns an invalid location if code-completion is not enabled or the file
1608
  /// containing the code-completion point has not been lexed yet.
1609
1.00M
  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1610
1611
  /// Returns the start location of the file of code-completion point.
1612
  ///
1613
  /// Returns an invalid location if code-completion is not enabled or the file
1614
  /// containing the code-completion point has not been lexed yet.
1615
52.1M
  SourceLocation getCodeCompletionFileLoc() const {
1616
52.1M
    return CodeCompletionFileLoc;
1617
52.1M
  }
1618
1619
  /// Returns true if code-completion is enabled and we have hit the
1620
  /// code-completion point.
1621
10.5M
  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1622
1623
  /// Note that we hit the code-completion point.
1624
1.32k
  void setCodeCompletionReached() {
1625
1.32k
    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1626
1.32k
    CodeCompletionReached = true;
1627
1.32k
    // Silence any diagnostics that occur after we hit the code-completion.
1628
1.32k
    getDiagnostics().setSuppressAllDiagnostics(true);
1629
1.32k
  }
1630
1631
  /// The location of the currently-active \#pragma clang
1632
  /// arc_cf_code_audited begin.
1633
  ///
1634
  /// Returns an invalid location if there is no such pragma active.
1635
  std::pair<IdentifierInfo *, SourceLocation>
1636
6.89M
  getPragmaARCCFCodeAuditedInfo() const {
1637
6.89M
    return PragmaARCCFCodeAuditedInfo;
1638
6.89M
  }
1639
1640
  /// Set the location of the currently-active \#pragma clang
1641
  /// arc_cf_code_audited begin.  An invalid location ends the pragma.
1642
  void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident,
1643
162k
                                     SourceLocation Loc) {
1644
162k
    PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1645
162k
  }
1646
1647
  /// The location of the currently-active \#pragma clang
1648
  /// assume_nonnull begin.
1649
  ///
1650
  /// Returns an invalid location if there is no such pragma active.
1651
40.6M
  SourceLocation getPragmaAssumeNonNullLoc() const {
1652
40.6M
    return PragmaAssumeNonNullLoc;
1653
40.6M
  }
1654
1655
  /// Set the location of the currently-active \#pragma clang
1656
  /// assume_nonnull begin.  An invalid location ends the pragma.
1657
351k
  void setPragmaAssumeNonNullLoc(SourceLocation Loc) {
1658
351k
    PragmaAssumeNonNullLoc = Loc;
1659
351k
  }
1660
1661
  /// Set the directory in which the main file should be considered
1662
  /// to have been found, if it is not a real file.
1663
1.85k
  void setMainFileDir(const DirectoryEntry *Dir) {
1664
1.85k
    MainFileDir = Dir;
1665
1.85k
  }
1666
1667
  /// Instruct the preprocessor to skip part of the main source file.
1668
  ///
1669
  /// \param Bytes The number of bytes in the preamble to skip.
1670
  ///
1671
  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1672
  /// start of a line.
1673
59.3k
  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1674
59.3k
    SkipMainFilePreamble.first = Bytes;
1675
59.3k
    SkipMainFilePreamble.second = StartOfLine;
1676
59.3k
  }
1677
1678
  /// Forwarding function for diagnostics.  This emits a diagnostic at
1679
  /// the specified Token's location, translating the token's start
1680
  /// position in the current buffer into a SourcePosition object for rendering.
1681
1.07M
  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1682
1.07M
    return Diags->Report(Loc, DiagID);
1683
1.07M
  }
1684
1685
1.31M
  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1686
1.31M
    return Diags->Report(Tok.getLocation(), DiagID);
1687
1.31M
  }
1688
1689
  /// Return the 'spelling' of the token at the given
1690
  /// location; does not go up to the spelling location or down to the
1691
  /// expansion location.
1692
  ///
1693
  /// \param buffer A buffer which will be used only if the token requires
1694
  ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
1695
  /// \param invalid If non-null, will be set \c true if an error occurs.
1696
  StringRef getSpelling(SourceLocation loc,
1697
                        SmallVectorImpl<char> &buffer,
1698
73
                        bool *invalid = nullptr) const {
1699
73
    return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1700
73
  }
1701
1702
  /// Return the 'spelling' of the Tok token.
1703
  ///
1704
  /// The spelling of a token is the characters used to represent the token in
1705
  /// the source file after trigraph expansion and escaped-newline folding.  In
1706
  /// particular, this wants to get the true, uncanonicalized, spelling of
1707
  /// things like digraphs, UCNs, etc.
1708
  ///
1709
  /// \param Invalid If non-null, will be set \c true if an error occurs.
1710
2.08M
  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1711
2.08M
    return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1712
2.08M
  }
1713
1714
  /// Get the spelling of a token into a preallocated buffer, instead
1715
  /// of as an std::string.
1716
  ///
1717
  /// The caller is required to allocate enough space for the token, which is
1718
  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1719
  /// actual result is returned.
1720
  ///
1721
  /// Note that this method may do two possible things: it may either fill in
1722
  /// the buffer specified with characters, or it may *change the input pointer*
1723
  /// to point to a constant buffer with the data already in it (avoiding a
1724
  /// copy).  The caller is not allowed to modify the returned buffer pointer
1725
  /// if an internal buffer is returned.
1726
  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1727
47.9M
                       bool *Invalid = nullptr) const {
1728
47.9M
    return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1729
47.9M
  }
1730
1731
  /// Get the spelling of a token into a SmallVector.
1732
  ///
1733
  /// Note that the returned StringRef may not point to the
1734
  /// supplied buffer if a copy can be avoided.
1735
  StringRef getSpelling(const Token &Tok,
1736
                        SmallVectorImpl<char> &Buffer,
1737
                        bool *Invalid = nullptr) const;
1738
1739
  /// Relex the token at the specified location.
1740
  /// \returns true if there was a failure, false on success.
1741
  bool getRawToken(SourceLocation Loc, Token &Result,
1742
117
                   bool IgnoreWhiteSpace = false) {
1743
117
    return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1744
117
  }
1745
1746
  /// Given a Token \p Tok that is a numeric constant with length 1,
1747
  /// return the character.
1748
  char
1749
  getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
1750
3.69M
                                              bool *Invalid = nullptr) const {
1751
3.69M
    assert(Tok.is(tok::numeric_constant) &&
1752
3.69M
           Tok.getLength() == 1 && "Called on unsupported token");
1753
3.69M
    assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1754
3.69M
1755
3.69M
    // If the token is carrying a literal data pointer, just use it.
1756
3.69M
    if (const char *D = Tok.getLiteralData())
1757
3.69M
      return *D;
1758
2.31k
1759
2.31k
    // Otherwise, fall back on getCharacterData, which is slower, but always
1760
2.31k
    // works.
1761
2.31k
    return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1762
2.31k
  }
1763
1764
  /// Retrieve the name of the immediate macro expansion.
1765
  ///
1766
  /// This routine starts from a source location, and finds the name of the
1767
  /// macro responsible for its immediate expansion. It looks through any
1768
  /// intervening macro argument expansions to compute this. It returns a
1769
  /// StringRef that refers to the SourceManager-owned buffer of the source
1770
  /// where that macro name is spelled. Thus, the result shouldn't out-live
1771
  /// the SourceManager.
1772
32
  StringRef getImmediateMacroName(SourceLocation Loc) {
1773
32
    return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1774
32
  }
1775
1776
  /// Plop the specified string into a scratch buffer and set the
1777
  /// specified token's location and length to it.
1778
  ///
1779
  /// If specified, the source location provides a location of the expansion
1780
  /// point of the token.
1781
  void CreateString(StringRef Str, Token &Tok,
1782
                    SourceLocation ExpansionLocStart = SourceLocation(),
1783
                    SourceLocation ExpansionLocEnd = SourceLocation());
1784
1785
  /// Split the first Length characters out of the token starting at TokLoc
1786
  /// and return a location pointing to the split token. Re-lexing from the
1787
  /// split token will return the split token rather than the original.
1788
  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1789
1790
  /// Computes the source location just past the end of the
1791
  /// token at this source location.
1792
  ///
1793
  /// This routine can be used to produce a source location that
1794
  /// points just past the end of the token referenced by \p Loc, and
1795
  /// is generally used when a diagnostic needs to point just after a
1796
  /// token where it expected something different that it received. If
1797
  /// the returned source location would not be meaningful (e.g., if
1798
  /// it points into a macro), this routine returns an invalid
1799
  /// source location.
1800
  ///
1801
  /// \param Offset an offset from the end of the token, where the source
1802
  /// location should refer to. The default offset (0) produces a source
1803
  /// location pointing just past the end of the token; an offset of 1 produces
1804
  /// a source location pointing to the last character in the token, etc.
1805
6.80k
  SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
1806
6.80k
    return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1807
6.80k
  }
1808
1809
  /// Returns true if the given MacroID location points at the first
1810
  /// token of the macro expansion.
1811
  ///
1812
  /// \param MacroBegin If non-null and function returns true, it is set to
1813
  /// begin location of the macro.
1814
  bool isAtStartOfMacroExpansion(SourceLocation loc,
1815
29
                                 SourceLocation *MacroBegin = nullptr) const {
1816
29
    return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1817
29
                                            MacroBegin);
1818
29
  }
1819
1820
  /// Returns true if the given MacroID location points at the last
1821
  /// token of the macro expansion.
1822
  ///
1823
  /// \param MacroEnd If non-null and function returns true, it is set to
1824
  /// end location of the macro.
1825
  bool isAtEndOfMacroExpansion(SourceLocation loc,
1826
30
                               SourceLocation *MacroEnd = nullptr) const {
1827
30
    return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1828
30
  }
1829
1830
  /// Print the token to stderr, used for debugging.
1831
  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1832
  void DumpLocation(SourceLocation Loc) const;
1833
  void DumpMacro(const MacroInfo &MI) const;
1834
  void dumpMacroInfo(const IdentifierInfo *II);
1835
1836
  /// Given a location that specifies the start of a
1837
  /// token, return a new location that specifies a character within the token.
1838
  SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
1839
3.01k
                                         unsigned Char) const {
1840
3.01k
    return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1841
3.01k
  }
1842
1843
  /// Increment the counters for the number of token paste operations
1844
  /// performed.
1845
  ///
1846
  /// If fast was specified, this is a 'fast paste' case we handled.
1847
14.3M
  void IncrementPasteCounter(bool isFast) {
1848
14.3M
    if (isFast)
1849
13.6M
      ++NumFastTokenPaste;
1850
719k
    else
1851
719k
      ++NumTokenPaste;
1852
14.3M
  }
1853
1854
  void PrintStats();
1855
1856
  size_t getTotalMemory() const;
1857
1858
  /// When the macro expander pastes together a comment (/##/) in Microsoft
1859
  /// mode, this method handles updating the current state, returning the
1860
  /// token on the next source line.
1861
  void HandleMicrosoftCommentPaste(Token &Tok);
1862
1863
  //===--------------------------------------------------------------------===//
1864
  // Preprocessor callback methods.  These are invoked by a lexer as various
1865
  // directives and events are found.
1866
1867
  /// Given a tok::raw_identifier token, look up the
1868
  /// identifier information for the token and install it into the token,
1869
  /// updating the token kind accordingly.
1870
  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1871
1872
private:
1873
  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1874
1875
public:
1876
  /// Specifies the reason for poisoning an identifier.
1877
  ///
1878
  /// If that identifier is accessed while poisoned, then this reason will be
1879
  /// used instead of the default "poisoned" diagnostic.
1880
  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1881
1882
  /// Display reason for poisoned identifier.
1883
  void HandlePoisonedIdentifier(Token & Identifier);
1884
1885
0
  void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1886
0
    if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1887
0
      if(II->isPoisoned()) {
1888
0
        HandlePoisonedIdentifier(Identifier);
1889
0
      }
1890
0
    }
1891
0
  }
1892
1893
private:
1894
  /// Identifiers used for SEH handling in Borland. These are only
1895
  /// allowed in particular circumstances
1896
  // __except block
1897
  IdentifierInfo *Ident__exception_code,
1898
                 *Ident___exception_code,
1899
                 *Ident_GetExceptionCode;
1900
  // __except filter expression
1901
  IdentifierInfo *Ident__exception_info,
1902
                 *Ident___exception_info,
1903
                 *Ident_GetExceptionInfo;
1904
  // __finally
1905
  IdentifierInfo *Ident__abnormal_termination,
1906
                 *Ident___abnormal_termination,
1907
                 *Ident_AbnormalTermination;
1908
1909
  const char *getCurLexerEndPos();
1910
  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1911
1912
public:
1913
  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1914
1915
  /// Callback invoked when the lexer reads an identifier and has
1916
  /// filled in the tokens IdentifierInfo member.
1917
  ///
1918
  /// This callback potentially macro expands it or turns it into a named
1919
  /// token (like 'for').
1920
  ///
1921
  /// \returns true if we actually computed a token, false if we need to
1922
  /// lex again.
1923
  bool HandleIdentifier(Token &Identifier);
1924
1925
  /// Callback invoked when the lexer hits the end of the current file.
1926
  ///
1927
  /// This either returns the EOF token and returns true, or
1928
  /// pops a level off the include stack and returns false, at which point the
1929
  /// client should call lex again.
1930
  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1931
1932
  /// Callback invoked when the current TokenLexer hits the end of its
1933
  /// token stream.
1934
  bool HandleEndOfTokenLexer(Token &Result);
1935
1936
  /// Callback invoked when the lexer sees a # token at the start of a
1937
  /// line.
1938
  ///
1939
  /// This consumes the directive, modifies the lexer/preprocessor state, and
1940
  /// advances the lexer(s) so that the next token read is the correct one.
1941
  void HandleDirective(Token &Result);
1942
1943
  /// Ensure that the next token is a tok::eod token.
1944
  ///
1945
  /// If not, emit a diagnostic and consume up until the eod.
1946
  /// If \p EnableMacros is true, then we consider macros that expand to zero
1947
  /// tokens as being ok.
1948
  ///
1949
  /// \return The location of the end of the directive (the terminating
1950
  /// newline).
1951
  SourceLocation CheckEndOfDirective(const char *DirType,
1952
                                     bool EnableMacros = false);
1953
1954
  /// Read and discard all tokens remaining on the current line until
1955
  /// the tok::eod token is found. Returns the range of the skipped tokens.
1956
  SourceRange DiscardUntilEndOfDirective();
1957
1958
  /// Returns true if the preprocessor has seen a use of
1959
  /// __DATE__ or __TIME__ in the file so far.
1960
4.35k
  bool SawDateOrTime() const {
1961
4.35k
    return DATELoc != SourceLocation() || 
TIMELoc != SourceLocation()4.35k
;
1962
4.35k
  }
1963
4.35k
  unsigned getCounterValue() const { return CounterValue; }
1964
142
  void setCounterValue(unsigned V) { CounterValue = V; }
1965
1966
  /// Retrieves the module that we're currently building, if any.
1967
  Module *getCurrentModule();
1968
1969
  /// Allocate a new MacroInfo object with the provided SourceLocation.
1970
  MacroInfo *AllocateMacroInfo(SourceLocation L);
1971
1972
  /// Turn the specified lexer token into a fully checked and spelled
1973
  /// filename, e.g. as an operand of \#include.
1974
  ///
1975
  /// The caller is expected to provide a buffer that is large enough to hold
1976
  /// the spelling of the filename, but is also expected to handle the case
1977
  /// when this method decides to use a different buffer.
1978
  ///
1979
  /// \returns true if the input filename was in <>'s or false if it was
1980
  /// in ""'s.
1981
  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1982
1983
  /// Given a "foo" or \<foo> reference, look up the indicated file.
1984
  ///
1985
  /// Returns None on failure.  \p isAngled indicates whether the file
1986
  /// reference is for system \#include's or not (i.e. using <> instead of "").
1987
  Optional<FileEntryRef>
1988
  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
1989
             const DirectoryLookup *FromDir, const FileEntry *FromFile,
1990
             const DirectoryLookup *&CurDir, SmallVectorImpl<char> *SearchPath,
1991
             SmallVectorImpl<char> *RelativePath,
1992
             ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
1993
             bool *IsFrameworkFound, bool SkipCache = false);
1994
1995
  /// Get the DirectoryLookup structure used to find the current
1996
  /// FileEntry, if CurLexer is non-null and if applicable.
1997
  ///
1998
  /// This allows us to implement \#include_next and find directory-specific
1999
  /// properties.
2000
4.52k
  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
2001
2002
  /// Return true if we're in the top-level file, not in a \#include.
2003
  bool isInPrimaryFile() const;
2004
2005
  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
2006
  /// followed by EOD.  Return true if the token is not a valid on-off-switch.
2007
  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
2008
2009
  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
2010
                      bool *ShadowFlag = nullptr);
2011
2012
  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
2013
  Module *LeaveSubmodule(bool ForPragma);
2014
2015
private:
2016
  friend void TokenLexer::ExpandFunctionArguments();
2017
2018
123M
  void PushIncludeMacroStack() {
2019
123M
    assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
2020
123M
    IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2021
123M
                                   std::move(CurLexer), CurPPLexer,
2022
123M
                                   std::move(CurTokenLexer), CurDirLookup);
2023
123M
    CurPPLexer = nullptr;
2024
123M
  }
2025
2026
123M
  void PopIncludeMacroStack() {
2027
123M
    CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2028
123M
    CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2029
123M
    CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2030
123M
    CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
2031
123M
    CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2032
123M
    CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2033
123M
    IncludeMacroStack.pop_back();
2034
123M
  }
2035
2036
  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2037
2038
  /// Determine whether we need to create module macros for #defines in the
2039
  /// current context.
2040
  bool needModuleMacros() const;
2041
2042
  /// Update the set of active module macros and ambiguity flag for a module
2043
  /// macro name.
2044
  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2045
2046
  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2047
                                               SourceLocation Loc);
2048
  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2049
  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2050
                                                             bool isPublic);
2051
2052
  /// Lex and validate a macro name, which occurs after a
2053
  /// \#define or \#undef.
2054
  ///
2055
  /// \param MacroNameTok Token that represents the name defined or undefined.
2056
  /// \param IsDefineUndef Kind if preprocessor directive.
2057
  /// \param ShadowFlag Points to flag that is set if macro name shadows
2058
  ///                   a keyword.
2059
  ///
2060
  /// This emits a diagnostic, sets the token kind to eod,
2061
  /// and discards the rest of the macro line if the macro name is invalid.
2062
  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2063
                     bool *ShadowFlag = nullptr);
2064
2065
  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2066
  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2067
  /// doing so performs certain validity checks including (but not limited to):
2068
  ///   - # (stringization) is followed by a macro parameter
2069
  /// \param MacroNameTok - Token that represents the macro name
2070
  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2071
  ///
2072
  ///  Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2073
  ///  returns a nullptr if an invalid sequence of tokens is encountered.
2074
  MacroInfo *ReadOptionalMacroParameterListAndBody(
2075
      const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2076
2077
  /// The ( starting an argument list of a macro definition has just been read.
2078
  /// Lex the rest of the parameters and the closing ), updating \p MI with
2079
  /// what we learn and saving in \p LastTok the last token read.
2080
  /// Return true if an error occurs parsing the arg list.
2081
  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2082
2083
  /// We just read a \#if or related directive and decided that the
2084
  /// subsequent tokens are in the \#if'd out portion of the
2085
  /// file.  Lex the rest of the file, until we see an \#endif.  If \p
2086
  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2087
  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2088
  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2089
  /// already seen one so a \#else directive is a duplicate.  When this returns,
2090
  /// the caller can lex the first valid token.
2091
  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2092
                                    SourceLocation IfTokenLoc,
2093
                                    bool FoundNonSkipPortion, bool FoundElse,
2094
                                    SourceLocation ElseLoc = SourceLocation());
2095
2096
  /// Information about the result for evaluating an expression for a
2097
  /// preprocessor directive.
2098
  struct DirectiveEvalResult {
2099
    /// Whether the expression was evaluated as true or not.
2100
    bool Conditional;
2101
2102
    /// True if the expression contained identifiers that were undefined.
2103
    bool IncludedUndefinedIds;
2104
2105
    /// The source range for the expression.
2106
    SourceRange ExprRange;
2107
  };
2108
2109
  /// Evaluate an integer constant expression that may occur after a
2110
  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2111
  ///
2112
  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2113
  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2114
2115
  /// Install the standard preprocessor pragmas:
2116
  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2117
  void RegisterBuiltinPragmas();
2118
2119
  /// Register builtin macros such as __LINE__ with the identifier table.
2120
  void RegisterBuiltinMacros();
2121
2122
  /// If an identifier token is read that is to be expanded as a macro, handle
2123
  /// it and return the next token as 'Tok'.  If we lexed a token, return true;
2124
  /// otherwise the caller should lex again.
2125
  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2126
2127
  /// Cache macro expanded tokens for TokenLexers.
2128
  //
2129
  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2130
  /// going to lex in the cache and when it finishes the tokens are removed
2131
  /// from the end of the cache.
2132
  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2133
                                  ArrayRef<Token> tokens);
2134
2135
  void removeCachedMacroExpandedTokensOfLastLexer();
2136
2137
  /// Determine whether the next preprocessor token to be
2138
  /// lexed is a '('.  If so, consume the token and return true, if not, this
2139
  /// method should have no observable side-effect on the lexed tokens.
2140
  bool isNextPPTokenLParen();
2141
2142
  /// After reading "MACRO(", this method is invoked to read all of the formal
2143
  /// arguments specified for the macro invocation.  Returns null on error.
2144
  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2145
                                       SourceLocation &MacroEnd);
2146
2147
  /// If an identifier token is read that is to be expanded
2148
  /// as a builtin macro, handle it and return the next token as 'Tok'.
2149
  void ExpandBuiltinMacro(Token &Tok);
2150
2151
  /// Read a \c _Pragma directive, slice it up, process it, then
2152
  /// return the first token after the directive.
2153
  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2154
  void Handle_Pragma(Token &Tok);
2155
2156
  /// Like Handle_Pragma except the pragma text is not enclosed within
2157
  /// a string literal.
2158
  void HandleMicrosoft__pragma(Token &Tok);
2159
2160
  /// Add a lexer to the top of the include stack and
2161
  /// start lexing tokens from it instead of the current buffer.
2162
  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2163
2164
  /// Set the FileID for the preprocessor predefines.
2165
59.2k
  void setPredefinesFileID(FileID FID) {
2166
59.2k
    assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2167
59.2k
    PredefinesFileID = FID;
2168
59.2k
  }
2169
2170
  /// Set the FileID for the PCH through header.
2171
  void setPCHThroughHeaderFileID(FileID FID);
2172
2173
  /// Returns true if we are lexing from a file and not a
2174
  /// pragma or a macro.
2175
2.45M
  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2176
2.45M
    return L ? 
!L->isPragmaLexer()2.45M
:
P != nullptr15
;
2177
2.45M
  }
2178
2179
43.5k
  static bool IsFileLexer(const IncludeStackInfo& I) {
2180
43.5k
    return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2181
43.5k
  }
2182
2183
2.41M
  bool IsFileLexer() const {
2184
2.41M
    return IsFileLexer(CurLexer.get(), CurPPLexer);
2185
2.41M
  }
2186
2187
  //===--------------------------------------------------------------------===//
2188
  // Caching stuff.
2189
  void CachingLex(Token &Result);
2190
2191
275M
  bool InCachingLexMode() const {
2192
275M
    // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2193
275M
    // that we are past EOF, not that we are in CachingLex mode.
2194
275M
    return !CurPPLexer && 
!CurTokenLexer189M
&&
!IncludeMacroStack.empty()172M
;
2195
275M
  }
2196
2197
  void EnterCachingLexMode();
2198
  void EnterCachingLexModeUnchecked();
2199
2200
100M
  void ExitCachingLexMode() {
2201
100M
    if (InCachingLexMode())
2202
58.8M
      RemoveTopOfLexerStack();
2203
100M
  }
2204
2205
  const Token &PeekAhead(unsigned N);
2206
  void AnnotatePreviousCachedTokens(const Token &Tok);
2207
2208
  //===--------------------------------------------------------------------===//
2209
  /// Handle*Directive - implement the various preprocessor directives.  These
2210
  /// should side-effect the current preprocessor object so that the next call
2211
  /// to Lex() will return the appropriate token next.
2212
  void HandleLineDirective();
2213
  void HandleDigitDirective(Token &Tok);
2214
  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2215
  void HandleIdentSCCSDirective(Token &Tok);
2216
  void HandleMacroPublicDirective(Token &Tok);
2217
  void HandleMacroPrivateDirective();
2218
2219
  /// An additional notification that can be produced by a header inclusion or
2220
  /// import to tell the parser what happened.
2221
  struct ImportAction {
2222
    enum ActionKind {
2223
      None,
2224
      ModuleBegin,
2225
      ModuleImport,
2226
      SkippedModuleImport,
2227
      Failure,
2228
    } Kind;
2229
    Module *ModuleForHeader = nullptr;
2230
2231
    ImportAction(ActionKind AK, Module *Mod = nullptr)
2232
2.17M
        : Kind(AK), ModuleForHeader(Mod) {
2233
2.17M
      assert((AK == None || Mod || AK == Failure) &&
2234
2.17M
             "no module for module action");
2235
2.17M
    }
2236
  };
2237
2238
  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
2239
      const DirectoryLookup *&CurDir, StringRef Filename,
2240
      SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2241
      const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2242
      bool &IsMapped, const DirectoryLookup *LookupFrom,
2243
      const FileEntry *LookupFromFile, StringRef LookupFilename,
2244
      SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2245
      ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2246
2247
  // File inclusion.
2248
  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2249
                              const DirectoryLookup *LookupFrom = nullptr,
2250
                              const FileEntry *LookupFromFile = nullptr);
2251
  ImportAction
2252
  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2253
                              Token &FilenameTok, SourceLocation EndLoc,
2254
                              const DirectoryLookup *LookupFrom = nullptr,
2255
                              const FileEntry *LookupFromFile = nullptr);
2256
  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2257
  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2258
  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2259
  void HandleMicrosoftImportDirective(Token &Tok);
2260
2261
public:
2262
  /// Check that the given module is available, producing a diagnostic if not.
2263
  /// \return \c true if the check failed (because the module is not available).
2264
  ///         \c false if the module appears to be usable.
2265
  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2266
                                     const TargetInfo &TargetInfo,
2267
                                     DiagnosticsEngine &Diags, Module *M);
2268
2269
  // Module inclusion testing.
2270
  /// Find the module that owns the source or header file that
2271
  /// \p Loc points to. If the location is in a file that was included
2272
  /// into a module, or is outside any module, returns nullptr.
2273
  Module *getModuleForLocation(SourceLocation Loc);
2274
2275
  /// We want to produce a diagnostic at location IncLoc concerning a
2276
  /// missing module import.
2277
  ///
2278
  /// \param IncLoc The location at which the missing import was detected.
2279
  /// \param M The desired module.
2280
  /// \param MLoc A location within the desired module at which some desired
2281
  ///        effect occurred (eg, where a desired entity was declared).
2282
  ///
2283
  /// \return A file that can be #included to import a module containing MLoc.
2284
  ///         Null if no such file could be determined or if a #include is not
2285
  ///         appropriate.
2286
  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2287
                                                          Module *M,
2288
                                                          SourceLocation MLoc);
2289
2290
1.54M
  bool isRecordingPreamble() const {
2291
1.54M
    return PreambleConditionalStack.isRecording();
2292
1.54M
  }
2293
2294
89
  bool hasRecordedPreamble() const {
2295
89
    return PreambleConditionalStack.hasRecordedPreamble();
2296
89
  }
2297
2298
7
  ArrayRef<PPConditionalInfo> getPreambleConditionalStack() const {
2299
7
      return PreambleConditionalStack.getStack();
2300
7
  }
2301
2302
89
  void setRecordedPreambleConditionalStack(ArrayRef<PPConditionalInfo> s) {
2303
89
    PreambleConditionalStack.setStack(s);
2304
89
  }
2305
2306
  void setReplayablePreambleConditionalStack(ArrayRef<PPConditionalInfo> s,
2307
28
                                             llvm::Optional<PreambleSkipInfo> SkipInfo) {
2308
28
    PreambleConditionalStack.startReplaying();
2309
28
    PreambleConditionalStack.setStack(s);
2310
28
    PreambleConditionalStack.SkipInfo = SkipInfo;
2311
28
  }
2312
2313
7
  llvm::Optional<PreambleSkipInfo> getPreambleSkipInfo() const {
2314
7
    return PreambleConditionalStack.SkipInfo;
2315
7
  }
2316
2317
private:
2318
  /// After processing predefined file, initialize the conditional stack from
2319
  /// the preamble.
2320
  void replayPreambleConditionalStack();
2321
2322
  // Macro handling.
2323
  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2324
  void HandleUndefDirective();
2325
2326
  // Conditional Inclusion.
2327
  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2328
                            bool isIfndef, bool ReadAnyTokensBeforeDirective);
2329
  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2330
                         bool ReadAnyTokensBeforeDirective);
2331
  void HandleEndifDirective(Token &EndifToken);
2332
  void HandleElseDirective(Token &Result, const Token &HashToken);
2333
  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2334
2335
  // Pragmas.
2336
  void HandlePragmaDirective(PragmaIntroducer Introducer);
2337
2338
public:
2339
  void HandlePragmaOnce(Token &OnceTok);
2340
  void HandlePragmaMark();
2341
  void HandlePragmaPoison();
2342
  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2343
  void HandlePragmaDependency(Token &DependencyTok);
2344
  void HandlePragmaPushMacro(Token &Tok);
2345
  void HandlePragmaPopMacro(Token &Tok);
2346
  void HandlePragmaIncludeAlias(Token &Tok);
2347
  void HandlePragmaModuleBuild(Token &Tok);
2348
  void HandlePragmaHdrstop(Token &Tok);
2349
  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2350
2351
  // Return true and store the first token only if any CommentHandler
2352
  // has inserted some tokens and getCommentRetentionState() is false.
2353
  bool HandleComment(Token &result, SourceRange Comment);
2354
2355
  /// A macro is used, update information about macros that need unused
2356
  /// warnings.
2357
  void markMacroAsUsed(MacroInfo *MI);
2358
2359
private:
2360
  Optional<unsigned>
2361
  getSkippedRangeForExcludedConditionalBlock(SourceLocation HashLoc);
2362
2363
  /// Contains the currently active skipped range mappings for skipping excluded
2364
  /// conditional directives.
2365
  ExcludedPreprocessorDirectiveSkipMapping
2366
      *ExcludedConditionalDirectiveSkipMappings;
2367
};
2368
2369
/// Abstract base class that describes a handler that will receive
2370
/// source ranges for each of the comments encountered in the source file.
2371
class CommentHandler {
2372
public:
2373
  virtual ~CommentHandler();
2374
2375
  // The handler shall return true if it has pushed any tokens
2376
  // to be read using e.g. EnterToken or EnterTokenStream.
2377
  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2378
};
2379
2380
/// Registry of pragma handlers added by plugins
2381
using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2382
2383
} // namespace clang
2384
2385
#endif // LLVM_CLANG_LEX_PREPROCESSOR_H